1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
|
#include <inttypes.h>
#include "util/u_inlines.h"
#include "util/u_memory.h"
#include "util/u_double_list.h"
#include "nouveau_screen.h"
#include "nouveau_mm.h"
#include "nouveau/nouveau_bo.h"
#define MM_MIN_ORDER 7
#define MM_MAX_ORDER 20
#define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1)
#define MM_MIN_SIZE (1 << MM_MIN_ORDER)
#define MM_MAX_SIZE (1 << MM_MAX_ORDER)
struct mm_bucket {
struct list_head free;
struct list_head used;
struct list_head full;
int num_free;
};
struct nouveau_mman {
struct nouveau_device *dev;
struct mm_bucket bucket[MM_NUM_BUCKETS];
uint32_t storage_type;
uint32_t domain;
uint64_t allocated;
};
struct mm_slab {
struct list_head head;
struct nouveau_bo *bo;
struct nouveau_mman *cache;
int order;
int count;
int free;
uint32_t bits[0];
};
static int
mm_slab_alloc(struct mm_slab *slab)
{
int i, n, b;
if (slab->free == 0)
return -1;
for (i = 0; i < (slab->count + 31) / 32; ++i) {
b = ffs(slab->bits[i]) - 1;
if (b >= 0) {
n = i * 32 + b;
assert(n < slab->count);
slab->free--;
slab->bits[i] &= ~(1 << b);
return n;
}
}
return -1;
}
static INLINE void
mm_slab_free(struct mm_slab *slab, int i)
{
assert(i < slab->count);
slab->bits[i / 32] |= 1 << (i % 32);
slab->free++;
assert(slab->free <= slab->count);
}
static INLINE int
mm_get_order(uint32_t size)
{
int s = __builtin_clz(size) ^ 31;
if (size > (1 << s))
s += 1;
return s;
}
static struct mm_bucket *
mm_bucket_by_order(struct nouveau_mman *cache, int order)
{
if (order > MM_MAX_ORDER)
return NULL;
return &cache->bucket[MAX2(order, MM_MIN_ORDER) - MM_MIN_ORDER];
}
static struct mm_bucket *
mm_bucket_by_size(struct nouveau_mman *cache, unsigned size)
{
return mm_bucket_by_order(cache, mm_get_order(size));
}
/* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */
static INLINE uint32_t
mm_default_slab_size(unsigned chunk_order)
{
static const int8_t slab_order[MM_MAX_ORDER - MM_MIN_ORDER + 1] =
{
12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22
};
assert(chunk_order <= MM_MAX_ORDER && chunk_order >= MM_MIN_ORDER);
return 1 << slab_order[chunk_order - MM_MIN_ORDER];
}
static int
mm_slab_new(struct nouveau_mman *cache, int chunk_order)
{
struct mm_slab *slab;
int words, ret;
const uint32_t size = mm_default_slab_size(chunk_order);
words = ((size >> chunk_order) + 31) / 32;
assert(words);
slab = MALLOC(sizeof(struct mm_slab) + words * 4);
if (!slab)
return PIPE_ERROR_OUT_OF_MEMORY;
memset(&slab->bits[0], ~0, words * 4);
slab->bo = NULL;
ret = nouveau_bo_new_tile(cache->dev, cache->domain, 0, size,
0, cache->storage_type, &slab->bo);
if (ret) {
FREE(slab);
return PIPE_ERROR_OUT_OF_MEMORY;
}
LIST_INITHEAD(&slab->head);
slab->cache = cache;
slab->order = chunk_order;
slab->count = slab->free = size >> chunk_order;
LIST_ADD(&slab->head, &mm_bucket_by_order(cache, chunk_order)->free);
cache->allocated += size;
if (nouveau_mesa_debug)
debug_printf("MM: new slab, total memory = %"PRIu64" KiB\n",
cache->allocated / 1024);
return PIPE_OK;
}
/* @return token to identify slab or NULL if we just allocated a new bo */
struct nouveau_mm_allocation *
nouveau_mm_allocate(struct nouveau_mman *cache,
uint32_t size, struct nouveau_bo **bo, uint32_t *offset)
{
struct mm_bucket *bucket;
struct mm_slab *slab;
struct nouveau_mm_allocation *alloc;
int ret;
bucket = mm_bucket_by_size(cache, size);
if (!bucket) {
ret = nouveau_bo_new_tile(cache->dev, cache->domain, 0, size,
0, cache->storage_type, bo);
if (ret)
debug_printf("bo_new(%x, %x): %i\n", size, cache->storage_type, ret);
*offset = 0;
return NULL;
}
if (!LIST_IS_EMPTY(&bucket->used)) {
slab = LIST_ENTRY(struct mm_slab, bucket->used.next, head);
} else {
if (LIST_IS_EMPTY(&bucket->free)) {
mm_slab_new(cache, MAX2(mm_get_order(size), MM_MIN_ORDER));
}
slab = LIST_ENTRY(struct mm_slab, bucket->free.next, head);
LIST_DEL(&slab->head);
LIST_ADD(&slab->head, &bucket->used);
}
*offset = mm_slab_alloc(slab) << slab->order;
alloc = MALLOC_STRUCT(nouveau_mm_allocation);
if (!alloc)
return NULL;
nouveau_bo_ref(slab->bo, bo);
if (slab->free == 0) {
LIST_DEL(&slab->head);
LIST_ADD(&slab->head, &bucket->full);
}
alloc->next = NULL;
alloc->offset = *offset;
alloc->priv = (void *)slab;
return alloc;
}
void
nouveau_mm_free(struct nouveau_mm_allocation *alloc)
{
struct mm_slab *slab = (struct mm_slab *)alloc->priv;
struct mm_bucket *bucket = mm_bucket_by_order(slab->cache, slab->order);
mm_slab_free(slab, alloc->offset >> slab->order);
if (slab->free == slab->count) {
LIST_DEL(&slab->head);
LIST_ADDTAIL(&slab->head, &bucket->free);
} else
if (slab->free == 1) {
LIST_DEL(&slab->head);
LIST_ADDTAIL(&slab->head, &bucket->used);
}
FREE(alloc);
}
void
nouveau_mm_free_work(void *data)
{
nouveau_mm_free(data);
}
struct nouveau_mman *
nouveau_mm_create(struct nouveau_device *dev, uint32_t domain,
uint32_t storage_type)
{
struct nouveau_mman *cache = MALLOC_STRUCT(nouveau_mman);
int i;
if (!cache)
return NULL;
cache->dev = dev;
cache->domain = domain;
cache->storage_type = storage_type;
cache->allocated = 0;
for (i = 0; i < MM_NUM_BUCKETS; ++i) {
LIST_INITHEAD(&cache->bucket[i].free);
LIST_INITHEAD(&cache->bucket[i].used);
LIST_INITHEAD(&cache->bucket[i].full);
}
return cache;
}
static INLINE void
nouveau_mm_free_slabs(struct list_head *head)
{
struct mm_slab *slab, *next;
LIST_FOR_EACH_ENTRY_SAFE(slab, next, head, head) {
LIST_DEL(&slab->head);
nouveau_bo_ref(NULL, &slab->bo);
FREE(slab);
}
}
void
nouveau_mm_destroy(struct nouveau_mman *cache)
{
int i;
if (!cache)
return;
for (i = 0; i < MM_NUM_BUCKETS; ++i) {
if (!LIST_IS_EMPTY(&cache->bucket[i].used) ||
!LIST_IS_EMPTY(&cache->bucket[i].full))
debug_printf("WARNING: destroying GPU memory cache "
"with some buffers still in use\n");
nouveau_mm_free_slabs(&cache->bucket[i].free);
nouveau_mm_free_slabs(&cache->bucket[i].used);
nouveau_mm_free_slabs(&cache->bucket[i].full);
}
FREE(cache);
}
|