1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
|
#include "linux-kmem.h"
/*
* Memory allocation interfaces
*/
#ifdef DEBUG_KMEM
/* Shim layer memory accounting */
atomic_t kmem_alloc_used;
unsigned int kmem_alloc_max;
#endif
/*
* Slab allocation interfaces
*
* While the linux slab implementation was inspired by solaris they
* have made some changes to the API which complicates this shim
* layer. For one thing the same symbol names are used with different
* arguments for the prototypes. To deal with this we must use the
* preprocessor to re-order arguments. Happily for us standard C says,
* "Macro's appearing in their own expansion are not reexpanded" so
* this does not result in an infinite recursion. Additionally the
* function pointers registered by solarias differ from those used
* by linux so a lookup and mapping from linux style callback to a
* solaris style callback is needed. There is some overhead in this
* operation which isn't horibile but it needs to be kept in mind.
*/
typedef struct kmem_cache_cb {
struct list_head kcc_list;
kmem_cache_t * kcc_cache;
kmem_constructor_t kcc_constructor;
kmem_destructor_t kcc_destructor;
kmem_reclaim_t kcc_reclaim;
void * kcc_private;
void * kcc_vmp;
} kmem_cache_cb_t;
static spinlock_t kmem_cache_cb_lock = SPIN_LOCK_UNLOCKED;
//static spinlock_t kmem_cache_cb_lock = (spinlock_t) { 1 SPINLOCK_MAGIC_INIT };
static LIST_HEAD(kmem_cache_cb_list);
static struct shrinker *kmem_cache_shrinker;
/* Function must be called while holding the kmem_cache_cb_lock
* Because kmem_cache_t is an opaque datatype we're forced to
* match pointers to identify specific cache entires.
*/
static kmem_cache_cb_t *
kmem_cache_find_cache_cb(kmem_cache_t *cache)
{
kmem_cache_cb_t *kcc;
list_for_each_entry(kcc, &kmem_cache_cb_list, kcc_list)
if (cache == kcc->kcc_cache)
return kcc;
return NULL;
}
static kmem_cache_cb_t *
kmem_cache_add_cache_cb(kmem_cache_t *cache,
kmem_constructor_t constructor,
kmem_destructor_t destructor,
kmem_reclaim_t reclaim,
void *priv, void *vmp)
{
kmem_cache_cb_t *kcc;
kcc = (kmem_cache_cb_t *)kmalloc(sizeof(*kcc), GFP_KERNEL);
if (kcc) {
kcc->kcc_cache = cache;
kcc->kcc_constructor = constructor;
kcc->kcc_destructor = destructor;
kcc->kcc_reclaim = reclaim;
kcc->kcc_private = priv;
kcc->kcc_vmp = vmp;
spin_lock(&kmem_cache_cb_lock);
list_add(&kcc->kcc_list, &kmem_cache_cb_list);
spin_unlock(&kmem_cache_cb_lock);
}
return kcc;
}
static void
kmem_cache_remove_cache_cb(kmem_cache_cb_t *kcc)
{
spin_lock(&kmem_cache_cb_lock);
list_del(&kcc->kcc_list);
spin_unlock(&kmem_cache_cb_lock);
if (kcc)
kfree(kcc);
}
static void
kmem_cache_generic_constructor(void *ptr, kmem_cache_t *cache, unsigned long flags)
{
kmem_cache_cb_t *kcc;
spin_lock(&kmem_cache_cb_lock);
/* Callback list must be in sync with linux slab caches */
kcc = kmem_cache_find_cache_cb(cache);
BUG_ON(!kcc);
kcc->kcc_constructor(ptr, kcc->kcc_private, (int)flags);
spin_unlock(&kmem_cache_cb_lock);
/* Linux constructor has no return code, silently eat it */
}
static void
kmem_cache_generic_destructor(void *ptr, kmem_cache_t *cache, unsigned long flags)
{
kmem_cache_cb_t *kcc;
spin_lock(&kmem_cache_cb_lock);
/* Callback list must be in sync with linux slab caches */
kcc = kmem_cache_find_cache_cb(cache);
BUG_ON(!kcc);
/* Solaris destructor takes no flags, silently eat them */
kcc->kcc_destructor(ptr, kcc->kcc_private);
spin_unlock(&kmem_cache_cb_lock);
}
/* XXX - Arguments are ignored */
static int
kmem_cache_generic_shrinker(int nr_to_scan, unsigned int gfp_mask)
{
kmem_cache_cb_t *kcc;
int total = 0;
/* Under linux a shrinker is not tightly coupled with a slab
* cache. In fact linux always systematically trys calling all
* registered shrinker callbacks until its target reclamation level
* is reached. Because of this we only register one shrinker
* function in the shim layer for all slab caches. And we always
* attempt to shrink all caches when this generic shrinker is called.
*/
spin_lock(&kmem_cache_cb_lock);
list_for_each_entry(kcc, &kmem_cache_cb_list, kcc_list) {
/* Under linux the desired number and gfp type of objects
* is passed to the reclaiming function as a sugested reclaim
* target. I do not pass these args on because reclaim
* policy is entirely up to the owner under solaris. We only
* pass on the pre-registered private data.
*/
if (kcc->kcc_reclaim)
kcc->kcc_reclaim(kcc->kcc_private);
total += 1;
}
/* Under linux we should return the remaining number of entires in
* the cache. Unfortunately, I don't see an easy way to safely
* emulate this behavior so I'm returning one entry per cache which
* was registered with the generic shrinker. This should fake out
* the linux VM when it attempts to shrink caches.
*/
spin_unlock(&kmem_cache_cb_lock);
return total;
}
/* Ensure the __kmem_cache_create/__kmem_cache_destroy macros are
* removed here to prevent a recursive substitution, we want to call
* the native linux version.
*/
#undef kmem_cache_create
#undef kmem_cache_destroy
kmem_cache_t *
__kmem_cache_create(char *name, size_t size, size_t align,
kmem_constructor_t constructor,
kmem_destructor_t destructor,
kmem_reclaim_t reclaim,
void *priv, void *vmp, int flags)
{
kmem_cache_t *cache;
kmem_cache_cb_t *kcc;
int shrinker_flag = 0;
/* FIXME: - Option currently unsupported by shim layer */
BUG_ON(vmp);
cache = kmem_cache_create(name, size, align, flags,
kmem_cache_generic_constructor,
kmem_cache_generic_destructor);
if (cache == NULL)
return NULL;
/* Register shared shrinker function on initial cache create */
spin_lock(&kmem_cache_cb_lock);
if (list_empty(&kmem_cache_cb_list)) {
kmem_cache_shrinker = set_shrinker(KMC_DEFAULT_SEEKS,
kmem_cache_generic_shrinker);
if (kmem_cache_shrinker == NULL) {
kmem_cache_destroy(cache);
spin_unlock(&kmem_cache_cb_lock);
return NULL;
}
}
spin_unlock(&kmem_cache_cb_lock);
kcc = kmem_cache_add_cache_cb(cache, constructor, destructor,
reclaim, priv, vmp);
if (kcc == NULL) {
if (shrinker_flag) /* New shrinker registered must be removed */
remove_shrinker(kmem_cache_shrinker);
kmem_cache_destroy(cache);
return NULL;
}
return cache;
}
EXPORT_SYMBOL(__kmem_cache_create);
/* Return codes discarded because Solaris implementation has void return */
void
__kmem_cache_destroy(kmem_cache_t *cache)
{
kmem_cache_cb_t *kcc;
spin_lock(&kmem_cache_cb_lock);
kcc = kmem_cache_find_cache_cb(cache);
spin_unlock(&kmem_cache_cb_lock);
if (kcc == NULL)
return;
kmem_cache_destroy(cache);
kmem_cache_remove_cache_cb(kcc);
/* Unregister generic shrinker on removal of all caches */
spin_lock(&kmem_cache_cb_lock);
if (list_empty(&kmem_cache_cb_list))
remove_shrinker(kmem_cache_shrinker);
spin_unlock(&kmem_cache_cb_lock);
}
EXPORT_SYMBOL(__kmem_cache_destroy);
void
__kmem_reap(void) {
/* Since there's no easy hook in to linux to force all the registered
* shrinkers to run we just run the ones registered for this shim */
kmem_cache_generic_shrinker(KMC_REAP_CHUNK, GFP_KERNEL);
}
EXPORT_SYMBOL(__kmem_reap);
|
