1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
/*
* Copyright (C) 2008 Advanced Micro Devices, Inc.
*
* Author: Joerg Roedel <joerg.roedel@amd.com>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 as published
* by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <linux/dma-debug.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/list.h>
#define HASH_SIZE 1024ULL
#define HASH_FN_SHIFT 13
#define HASH_FN_MASK (HASH_SIZE - 1)
enum {
dma_debug_single,
dma_debug_page,
dma_debug_sg,
dma_debug_coherent,
};
struct dma_debug_entry {
struct list_head list;
struct device *dev;
int type;
phys_addr_t paddr;
u64 dev_addr;
u64 size;
int direction;
int sg_call_ents;
int sg_mapped_ents;
};
struct hash_bucket {
struct list_head list;
spinlock_t lock;
} __cacheline_aligned_in_smp;
/* Hash list to save the allocated dma addresses */
static struct hash_bucket dma_entry_hash[HASH_SIZE];
/* List of pre-allocated dma_debug_entry's */
static LIST_HEAD(free_entries);
/* Lock for the list above */
static DEFINE_SPINLOCK(free_entries_lock);
/* Global disable flag - will be set in case of an error */
static bool global_disable __read_mostly;
static u32 num_free_entries;
static u32 min_free_entries;
/*
* Hash related functions
*
* Every DMA-API request is saved into a struct dma_debug_entry. To
* have quick access to these structs they are stored into a hash.
*/
static int hash_fn(struct dma_debug_entry *entry)
{
/*
* Hash function is based on the dma address.
* We use bits 20-27 here as the index into the hash
*/
return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
}
/*
* Request exclusive access to a hash bucket for a given dma_debug_entry.
*/
static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
unsigned long *flags)
{
int idx = hash_fn(entry);
unsigned long __flags;
spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
*flags = __flags;
return &dma_entry_hash[idx];
}
/*
* Give up exclusive access to the hash bucket
*/
static void put_hash_bucket(struct hash_bucket *bucket,
unsigned long *flags)
{
unsigned long __flags = *flags;
spin_unlock_irqrestore(&bucket->lock, __flags);
}
/*
* Search a given entry in the hash bucket list
*/
static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket,
struct dma_debug_entry *ref)
{
struct dma_debug_entry *entry;
list_for_each_entry(entry, &bucket->list, list) {
if ((entry->dev_addr == ref->dev_addr) &&
(entry->dev == ref->dev))
return entry;
}
return NULL;
}
/*
* Add an entry to a hash bucket
*/
static void hash_bucket_add(struct hash_bucket *bucket,
struct dma_debug_entry *entry)
{
list_add_tail(&entry->list, &bucket->list);
}
/*
* Remove entry from a hash bucket list
*/
static void hash_bucket_del(struct dma_debug_entry *entry)
{
list_del(&entry->list);
}
/*
* Wrapper function for adding an entry to the hash.
* This function takes care of locking itself.
*/
static void add_dma_entry(struct dma_debug_entry *entry)
{
struct hash_bucket *bucket;
unsigned long flags;
bucket = get_hash_bucket(entry, &flags);
hash_bucket_add(bucket, entry);
put_hash_bucket(bucket, &flags);
}
/* struct dma_entry allocator
*
* The next two functions implement the allocator for
* struct dma_debug_entries.
*/
static struct dma_debug_entry *dma_entry_alloc(void)
{
struct dma_debug_entry *entry = NULL;
unsigned long flags;
spin_lock_irqsave(&free_entries_lock, flags);
if (list_empty(&free_entries)) {
printk(KERN_ERR "DMA-API: debugging out of memory "
"- disabling\n");
global_disable = true;
goto out;
}
entry = list_entry(free_entries.next, struct dma_debug_entry, list);
list_del(&entry->list);
memset(entry, 0, sizeof(*entry));
num_free_entries -= 1;
if (num_free_entries < min_free_entries)
min_free_entries = num_free_entries;
out:
spin_unlock_irqrestore(&free_entries_lock, flags);
return entry;
}
static void dma_entry_free(struct dma_debug_entry *entry)
{
unsigned long flags;
/*
* add to beginning of the list - this way the entries are
* more likely cache hot when they are reallocated.
*/
spin_lock_irqsave(&free_entries_lock, flags);
list_add(&entry->list, &free_entries);
num_free_entries += 1;
spin_unlock_irqrestore(&free_entries_lock, flags);
}
|
