aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/os/linux/spl/sys/kmem_cache.h6
-rw-r--r--include/sys/zfs_context.h1
-rw-r--r--module/os/linux/spl/spl-kmem-cache.c18
-rw-r--r--module/os/linux/spl/spl-proc.c27
-rw-r--r--module/os/linux/spl/spl-zlib.c2
5 files changed, 43 insertions, 11 deletions
diff --git a/include/os/linux/spl/sys/kmem_cache.h b/include/os/linux/spl/sys/kmem_cache.h
index 3584eefdf..5667382f7 100644
--- a/include/os/linux/spl/sys/kmem_cache.h
+++ b/include/os/linux/spl/sys/kmem_cache.h
@@ -43,8 +43,9 @@ typedef enum kmc_bit {
KMC_BIT_QCACHE = 4, /* XXX: Unsupported */
KMC_BIT_KMEM = 5, /* Use kmem cache */
KMC_BIT_VMEM = 6, /* Use vmem cache */
- KMC_BIT_SLAB = 7, /* Use Linux slab cache */
- KMC_BIT_OFFSLAB = 8, /* Objects not on slab */
+ KMC_BIT_KVMEM = 7, /* Use kvmalloc linux allocator */
+ KMC_BIT_SLAB = 8, /* Use Linux slab cache */
+ KMC_BIT_OFFSLAB = 9, /* Objects not on slab */
KMC_BIT_DEADLOCKED = 14, /* Deadlock detected */
KMC_BIT_GROWING = 15, /* Growing in progress */
KMC_BIT_REAPING = 16, /* Reaping in progress */
@@ -70,6 +71,7 @@ typedef enum kmem_cbrc {
#define KMC_QCACHE (1 << KMC_BIT_QCACHE)
#define KMC_KMEM (1 << KMC_BIT_KMEM)
#define KMC_VMEM (1 << KMC_BIT_VMEM)
+#define KMC_KVMEM (1 << KMC_BIT_KVMEM)
#define KMC_SLAB (1 << KMC_BIT_SLAB)
#define KMC_OFFSLAB (1 << KMC_BIT_OFFSLAB)
#define KMC_DEADLOCKED (1 << KMC_BIT_DEADLOCKED)
diff --git a/include/sys/zfs_context.h b/include/sys/zfs_context.h
index e915f0a8f..f221091d8 100644
--- a/include/sys/zfs_context.h
+++ b/include/sys/zfs_context.h
@@ -406,6 +406,7 @@ void procfs_list_add(procfs_list_t *procfs_list, void *p);
#define KMC_NODEBUG UMC_NODEBUG
#define KMC_KMEM 0x0
#define KMC_VMEM 0x0
+#define KMC_KVMEM 0x0
#define kmem_alloc(_s, _f) umem_alloc(_s, _f)
#define kmem_zalloc(_s, _f) umem_zalloc(_s, _f)
#define kmem_free(_b, _s) umem_free(_b, _s)
diff --git a/module/os/linux/spl/spl-kmem-cache.c b/module/os/linux/spl/spl-kmem-cache.c
index 7e423100d..46cf2f288 100644
--- a/module/os/linux/spl/spl-kmem-cache.c
+++ b/module/os/linux/spl/spl-kmem-cache.c
@@ -202,6 +202,8 @@ kv_alloc(spl_kmem_cache_t *skc, int size, int flags)
if (skc->skc_flags & KMC_KMEM) {
ASSERT(ISP2(size));
ptr = (void *)__get_free_pages(lflags, get_order(size));
+ } else if (skc->skc_flags & KMC_KVMEM) {
+ ptr = spl_kvmalloc(size, lflags);
} else {
/*
* GFP_KERNEL allocations can safely use kvmalloc which may
@@ -890,6 +892,7 @@ spl_magazine_destroy(spl_kmem_cache_t *skc)
* flags
* KMC_KMEM Force SPL kmem backed cache
* KMC_VMEM Force SPL vmem backed cache
+ * KMC_KVMEM Force kvmem backed cache
* KMC_SLAB Force Linux slab backed cache
* KMC_OFFSLAB Locate objects off the slab
* KMC_NOTOUCH Disable cache object aging (unsupported)
@@ -977,8 +980,7 @@ spl_kmem_cache_create(char *name, size_t size, size_t align,
* linuxslab) then select a cache type based on the object size
* and default tunables.
*/
- if (!(skc->skc_flags & (KMC_KMEM | KMC_VMEM | KMC_SLAB))) {
-
+ if (!(skc->skc_flags & (KMC_KMEM | KMC_VMEM | KMC_SLAB | KMC_KVMEM))) {
if (spl_kmem_cache_slab_limit &&
size <= (size_t)spl_kmem_cache_slab_limit) {
/*
@@ -996,16 +998,16 @@ spl_kmem_cache_create(char *name, size_t size, size_t align,
} else {
/*
* All other objects are considered large and are
- * placed on vmem backed slabs.
+ * placed on kvmem backed slabs.
*/
- skc->skc_flags |= KMC_VMEM;
+ skc->skc_flags |= KMC_KVMEM;
}
}
/*
* Given the type of slab allocate the required resources.
*/
- if (skc->skc_flags & (KMC_KMEM | KMC_VMEM)) {
+ if (skc->skc_flags & (KMC_KMEM | KMC_VMEM | KMC_KVMEM)) {
rc = spl_slab_size(skc,
&skc->skc_slab_objs, &skc->skc_slab_size);
if (rc)
@@ -1089,7 +1091,7 @@ spl_kmem_cache_destroy(spl_kmem_cache_t *skc)
taskqid_t id;
ASSERT(skc->skc_magic == SKC_MAGIC);
- ASSERT(skc->skc_flags & (KMC_KMEM | KMC_VMEM | KMC_SLAB));
+ ASSERT(skc->skc_flags & (KMC_KMEM | KMC_VMEM | KMC_KVMEM | KMC_SLAB));
down_write(&spl_kmem_cache_sem);
list_del_init(&skc->skc_list);
@@ -1111,7 +1113,7 @@ spl_kmem_cache_destroy(spl_kmem_cache_t *skc)
*/
wait_event(wq, atomic_read(&skc->skc_ref) == 0);
- if (skc->skc_flags & (KMC_KMEM | KMC_VMEM)) {
+ if (skc->skc_flags & (KMC_KMEM | KMC_VMEM | KMC_KVMEM)) {
spl_magazine_destroy(skc);
spl_slab_reclaim(skc);
} else {
@@ -1267,7 +1269,7 @@ spl_cache_grow(spl_kmem_cache_t *skc, int flags, void **obj)
* However, this can't be applied to KVM_VMEM due to a bug that
* __vmalloc() doesn't honor gfp flags in page table allocation.
*/
- if (!(skc->skc_flags & KMC_VMEM)) {
+ if (!(skc->skc_flags & KMC_VMEM) && !(skc->skc_flags & KMC_KVMEM)) {
rc = __spl_cache_grow(skc, flags | KM_NOSLEEP);
if (rc == 0)
return (0);
diff --git a/module/os/linux/spl/spl-proc.c b/module/os/linux/spl/spl-proc.c
index 2dce8cd70..f4e0e0594 100644
--- a/module/os/linux/spl/spl-proc.c
+++ b/module/os/linux/spl/spl-proc.c
@@ -662,6 +662,33 @@ static struct ctl_table spl_kmem_table[] = {
.mode = 0444,
.proc_handler = &proc_doslab,
},
+ {
+ .procname = "slab_kvmem_total",
+ .data = (void *)(KMC_KVMEM | KMC_TOTAL),
+ .maxlen = sizeof (unsigned long),
+ .extra1 = &table_min,
+ .extra2 = &table_max,
+ .mode = 0444,
+ .proc_handler = &proc_doslab,
+ },
+ {
+ .procname = "slab_kvmem_alloc",
+ .data = (void *)(KMC_KVMEM | KMC_ALLOC),
+ .maxlen = sizeof (unsigned long),
+ .extra1 = &table_min,
+ .extra2 = &table_max,
+ .mode = 0444,
+ .proc_handler = &proc_doslab,
+ },
+ {
+ .procname = "slab_kvmem_max",
+ .data = (void *)(KMC_KVMEM | KMC_MAX),
+ .maxlen = sizeof (unsigned long),
+ .extra1 = &table_min,
+ .extra2 = &table_max,
+ .mode = 0444,
+ .proc_handler = &proc_doslab,
+ },
{},
};
diff --git a/module/os/linux/spl/spl-zlib.c b/module/os/linux/spl/spl-zlib.c
index 84026d710..a1c6abecf 100644
--- a/module/os/linux/spl/spl-zlib.c
+++ b/module/os/linux/spl/spl-zlib.c
@@ -202,7 +202,7 @@ spl_zlib_init(void)
zlib_workspace_cache = kmem_cache_create(
"spl_zlib_workspace_cache",
size, 0, NULL, NULL, NULL, NULL, NULL,
- KMC_VMEM);
+ KMC_KVMEM);
if (!zlib_workspace_cache)
return (1);
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-29 14:50:09 +0000