1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
/*
* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
* Copyright (C) 2007 The Regents of the University of California.
* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
* UCRL-CODE-235197
*
* This file is part of the SPL, Solaris Porting Layer.
* For details, see <http://zfsonlinux.org/>.
*
* The SPL is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation; either version 2 of the License, or (at your
* option) any later version.
*
* The SPL is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*
* You should have received a copy of the GNU General Public License along
* with the SPL. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _SPL_KMEM_H
#define _SPL_KMEM_H
#include <sys/debug.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
extern int kmem_debugging(void);
extern char *kmem_vasprintf(const char *fmt, va_list ap);
extern char *kmem_asprintf(const char *fmt, ...);
extern char *kmem_strdup(const char *str);
extern void kmem_strfree(char *str);
/*
* Memory allocation interfaces
*/
#define KM_SLEEP 0x0000 /* can block for memory; success guaranteed */
#define KM_NOSLEEP 0x0001 /* cannot block for memory; may fail */
#define KM_PUSHPAGE 0x0004 /* can block for memory; may use reserve */
#define KM_ZERO 0x1000 /* zero the allocation */
#define KM_VMEM 0x2000 /* caller is vmem_* wrapper */
#define KM_PUBLIC_MASK (KM_SLEEP | KM_NOSLEEP | KM_PUSHPAGE)
static int spl_fstrans_check(void);
void *spl_kvmalloc(size_t size, gfp_t flags);
/*
* Convert a KM_* flags mask to its Linux GFP_* counterpart. The conversion
* function is context aware which means that KM_SLEEP allocations can be
* safely used in syncing contexts which have set PF_FSTRANS.
*/
static inline gfp_t
kmem_flags_convert(int flags)
{
gfp_t lflags = __GFP_NOWARN | __GFP_COMP;
if (flags & KM_NOSLEEP) {
lflags |= GFP_ATOMIC | __GFP_NORETRY;
} else {
lflags |= GFP_KERNEL;
if (spl_fstrans_check())
lflags &= ~(__GFP_IO|__GFP_FS);
}
if (flags & KM_PUSHPAGE)
lflags |= __GFP_HIGH;
if (flags & KM_ZERO)
lflags |= __GFP_ZERO;
return (lflags);
}
typedef struct {
struct task_struct *fstrans_thread;
unsigned int saved_flags;
} fstrans_cookie_t;
/*
* Introduced in Linux 3.9, however this cannot be solely relied on before
* Linux 3.18 as it doesn't turn off __GFP_FS as it should.
*/
#ifdef PF_MEMALLOC_NOIO
#define __SPL_PF_MEMALLOC_NOIO (PF_MEMALLOC_NOIO)
#else
#define __SPL_PF_MEMALLOC_NOIO (0)
#endif
/*
* PF_FSTRANS is removed from Linux 4.12
*/
#ifdef PF_FSTRANS
#define __SPL_PF_FSTRANS (PF_FSTRANS)
#else
#define __SPL_PF_FSTRANS (0)
#endif
#define SPL_FSTRANS (__SPL_PF_FSTRANS|__SPL_PF_MEMALLOC_NOIO)
static inline fstrans_cookie_t
spl_fstrans_mark(void)
{
fstrans_cookie_t cookie;
BUILD_BUG_ON(SPL_FSTRANS == 0);
cookie.fstrans_thread = current;
cookie.saved_flags = current->flags & SPL_FSTRANS;
current->flags |= SPL_FSTRANS;
return (cookie);
}
static inline void
spl_fstrans_unmark(fstrans_cookie_t cookie)
{
ASSERT3P(cookie.fstrans_thread, ==, current);
ASSERT((current->flags & SPL_FSTRANS) == SPL_FSTRANS);
current->flags &= ~SPL_FSTRANS;
current->flags |= cookie.saved_flags;
}
static inline int
spl_fstrans_check(void)
{
return (current->flags & SPL_FSTRANS);
}
/*
* specifically used to check PF_FSTRANS flag, cannot be relied on for
* checking spl_fstrans_mark().
*/
static inline int
__spl_pf_fstrans_check(void)
{
return (current->flags & __SPL_PF_FSTRANS);
}
/*
* Kernel compatibility for GFP flags
*/
/* < 4.13 */
#ifndef __GFP_RETRY_MAYFAIL
#define __GFP_RETRY_MAYFAIL __GFP_REPEAT
#endif
/* < 4.4 */
#ifndef __GFP_RECLAIM
#define __GFP_RECLAIM __GFP_WAIT
#endif
#ifdef HAVE_ATOMIC64_T
#define kmem_alloc_used_add(size) atomic64_add(size, &kmem_alloc_used)
#define kmem_alloc_used_sub(size) atomic64_sub(size, &kmem_alloc_used)
#define kmem_alloc_used_read() atomic64_read(&kmem_alloc_used)
#define kmem_alloc_used_set(size) atomic64_set(&kmem_alloc_used, size)
extern atomic64_t kmem_alloc_used;
extern unsigned long long kmem_alloc_max;
#else /* HAVE_ATOMIC64_T */
#define kmem_alloc_used_add(size) atomic_add(size, &kmem_alloc_used)
#define kmem_alloc_used_sub(size) atomic_sub(size, &kmem_alloc_used)
#define kmem_alloc_used_read() atomic_read(&kmem_alloc_used)
#define kmem_alloc_used_set(size) atomic_set(&kmem_alloc_used, size)
extern atomic_t kmem_alloc_used;
extern unsigned long long kmem_alloc_max;
#endif /* HAVE_ATOMIC64_T */
extern unsigned int spl_kmem_alloc_warn;
extern unsigned int spl_kmem_alloc_max;
#define kmem_alloc(sz, fl) spl_kmem_alloc((sz), (fl), __func__, __LINE__)
#define kmem_zalloc(sz, fl) spl_kmem_zalloc((sz), (fl), __func__, __LINE__)
#define kmem_free(ptr, sz) spl_kmem_free((ptr), (sz))
#define kmem_cache_reap_active spl_kmem_cache_reap_active
extern void *spl_kmem_alloc(size_t sz, int fl, const char *func, int line);
extern void *spl_kmem_zalloc(size_t sz, int fl, const char *func, int line);
extern void spl_kmem_free(const void *ptr, size_t sz);
/*
* 5.8 API change, pgprot_t argument removed.
*/
#ifdef HAVE_VMALLOC_PAGE_KERNEL
#define spl_vmalloc(size, flags) __vmalloc(size, flags, PAGE_KERNEL)
#else
#define spl_vmalloc(size, flags) __vmalloc(size, flags)
#endif
/*
* The following functions are only available for internal use.
*/
extern void *spl_kmem_alloc_impl(size_t size, int flags, int node);
extern void *spl_kmem_alloc_debug(size_t size, int flags, int node);
extern void *spl_kmem_alloc_track(size_t size, int flags,
const char *func, int line, int node);
extern void spl_kmem_free_impl(const void *buf, size_t size);
extern void spl_kmem_free_debug(const void *buf, size_t size);
extern void spl_kmem_free_track(const void *buf, size_t size);
extern int spl_kmem_init(void);
extern void spl_kmem_fini(void);
extern int spl_kmem_cache_reap_active(void);
#endif /* _SPL_KMEM_H */
|
