From ea3e6ca9e595ebfba82b964ee2eaf1ddd7076f0f Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 30 Jan 2009 20:54:49 -0800 Subject: kmem_cache hardening and performance improvements - Added slab work queue task which gradually ages and free's slabs from the cache which have not been used recently. - Optimized slab packing algorithm to ensure each slab contains the maximum number of objects without create to large a slab. - Fix deadlock, we can never call kv_free() under the skc_lock. We now unlink the objects and slabs from the cache itself and attach them to a private work list. The contents of the list are then subsequently freed outside the spin lock. - Move magazine create/destroy operation on to local cpu. - Further performace optimizations by minimize the usage of the large per-cache skc_lock. This includes the addition of KMC_BIT_REAPING bit mask which is used to prevent concurrent reaping, and to defer new slab creation when reaping is occuring. - Add KMC_BIT_DESTROYING bit mask which is set when the cache is being destroyed, this is used to catch any task accessing the cache while it is being destroyed. - Add comments to all the functions and additional comments to try and make everything as clear as possible. - Major cleanup and additions to the SPLAT kmem tests to more rigerously stress the cache implementation and look for any problems. This includes correctness and performance tests. - Updated portable work queue interfaces --- include/sys/kmem.h | 73 +++++++++++++++++++++++++++++++------------------ include/sys/sysmacros.h | 12 -------- include/sys/vmsystm.h | 3 +- 3 files changed, 48 insertions(+), 40 deletions(-) (limited to 'include') diff --git a/include/sys/kmem.h b/include/sys/kmem.h index ef5876312..4f939e0fc 100644 --- a/include/sys/kmem.h +++ b/include/sys/kmem.h @@ -45,6 +45,7 @@ extern "C" { #include #include #include +#include /* * Memory allocation interfaces @@ -161,17 +162,32 @@ kmem_alloc_tryhard(size_t size, size_t *alloc_size, int kmflags) /* * Slab allocation interfaces */ -#define KMC_NOTOUCH 0x00000001 -#define KMC_NODEBUG 0x00000002 /* Default behavior */ -#define KMC_NOMAGAZINE 0x00000004 /* XXX: No disable support available */ -#define KMC_NOHASH 0x00000008 /* XXX: No hash available */ -#define KMC_QCACHE 0x00000010 /* XXX: Unsupported */ -#define KMC_KMEM 0x00000100 /* Use kmem cache */ -#define KMC_VMEM 0x00000200 /* Use vmem cache */ -#define KMC_OFFSLAB 0x00000400 /* Objects not on slab */ - -#define KMC_REAP_CHUNK 256 -#define KMC_DEFAULT_SEEKS DEFAULT_SEEKS +enum { + KMC_BIT_NOTOUCH = 0, /* Don't update ages */ + KMC_BIT_NODEBUG = 1, /* Default behavior */ + KMC_BIT_NOMAGAZINE = 2, /* XXX: Unsupported */ + KMC_BIT_NOHASH = 3, /* XXX: Unsupported */ + KMC_BIT_QCACHE = 4, /* XXX: Unsupported */ + KMC_BIT_KMEM = 5, /* Use kmem cache */ + KMC_BIT_VMEM = 6, /* Use vmem cache */ + KMC_BIT_OFFSLAB = 7, /* Objects not on slab */ + KMC_BIT_REAPING = 16, /* Reaping in progress */ + KMC_BIT_DESTROY = 17, /* Destroy in progress */ +}; + +#define KMC_NOTOUCH (1 << KMC_BIT_NOTOUCH) +#define KMC_NODEBUG (1 << KMC_BIT_NODEBUG) +#define KMC_NOMAGAZINE (1 << KMC_BIT_NOMAGAZINE) +#define KMC_NOHASH (1 << KMC_BIT_NOHASH) +#define KMC_QCACHE (1 << KMC_BIT_QCACHE) +#define KMC_KMEM (1 << KMC_BIT_KMEM) +#define KMC_VMEM (1 << KMC_BIT_VMEM) +#define KMC_OFFSLAB (1 << KMC_BIT_OFFSLAB) +#define KMC_REAPING (1 << KMC_BIT_REAPING) +#define KMC_DESTROY (1 << KMC_BIT_DESTROY) + +#define KMC_REAP_CHUNK INT_MAX +#define KMC_DEFAULT_SEEKS 1 #ifdef DEBUG_KMEM_UNIMPLEMENTED static __inline__ void kmem_init(void) { @@ -223,9 +239,10 @@ extern struct rw_semaphore spl_kmem_cache_sem; #define SKS_MAGIC 0x22222222 #define SKC_MAGIC 0x2c2c2c2c -#define SPL_KMEM_CACHE_DELAY 5 -#define SPL_KMEM_CACHE_OBJ_PER_SLAB 32 -#define SPL_KMEM_CACHE_ALIGN 8 +#define SPL_KMEM_CACHE_DELAY 5 /* Minimum slab release age */ +#define SPL_KMEM_CACHE_OBJ_PER_SLAB 32 /* Target objects per slab */ +#define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN 8 /* Minimum objects per slab */ +#define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */ typedef int (*spl_kmem_ctor_t)(void *, void *, int); typedef void (*spl_kmem_dtor_t)(void *, void *); @@ -258,24 +275,28 @@ typedef struct spl_kmem_slab { } spl_kmem_slab_t; typedef struct spl_kmem_cache { - uint32_t skc_magic; /* Sanity magic */ - uint32_t skc_name_size; /* Name length */ - char *skc_name; /* Name string */ + uint32_t skc_magic; /* Sanity magic */ + uint32_t skc_name_size; /* Name length */ + char *skc_name; /* Name string */ spl_kmem_magazine_t *skc_mag[NR_CPUS]; /* Per-CPU warm cache */ uint32_t skc_mag_size; /* Magazine size */ uint32_t skc_mag_refill; /* Magazine refill count */ - spl_kmem_ctor_t skc_ctor; /* Constructor */ - spl_kmem_dtor_t skc_dtor; /* Destructor */ - spl_kmem_reclaim_t skc_reclaim; /* Reclaimator */ - void *skc_private; /* Private data */ - void *skc_vmp; /* Unused */ + spl_kmem_ctor_t skc_ctor; /* Constructor */ + spl_kmem_dtor_t skc_dtor; /* Destructor */ + spl_kmem_reclaim_t skc_reclaim; /* Reclaimator */ + void *skc_private; /* Private data */ + void *skc_vmp; /* Unused */ uint32_t skc_flags; /* Flags */ uint32_t skc_obj_size; /* Object size */ uint32_t skc_obj_align; /* Object alignment */ uint32_t skc_slab_objs; /* Objects per slab */ - uint32_t skc_slab_size; /* Slab size */ - uint32_t skc_delay; /* slab reclaim interval */ - struct list_head skc_list; /* List of caches linkage */ + uint32_t skc_slab_size; /* Slab size */ + uint32_t skc_delay; /* Slab reclaim interval */ + atomic_t skc_ref; /* Ref count callers */ + struct delayed_work skc_work; /* Slab reclaim work */ + struct work_struct work; + struct timer_list timer; + struct list_head skc_list; /* List of caches linkage */ struct list_head skc_complete_list;/* Completely alloc'ed */ struct list_head skc_partial_list; /* Partially alloc'ed */ spinlock_t skc_lock; /* Cache lock */ @@ -283,7 +304,7 @@ typedef struct spl_kmem_cache { uint64_t skc_slab_create;/* Slab creates */ uint64_t skc_slab_destroy;/* Slab destroys */ uint64_t skc_slab_total; /* Slab total current */ - uint64_t skc_slab_alloc; /* Slab alloc current */ + uint64_t skc_slab_alloc; /* Slab alloc current */ uint64_t skc_slab_max; /* Slab max historic */ uint64_t skc_obj_total; /* Obj total current */ uint64_t skc_obj_alloc; /* Obj alloc current */ diff --git a/include/sys/sysmacros.h b/include/sys/sysmacros.h index 94ff3f84e..b82812385 100644 --- a/include/sys/sysmacros.h +++ b/include/sys/sysmacros.h @@ -203,18 +203,6 @@ extern int ddi_strtoul(const char *str, char **nptr, #define offsetof(s, m) ((size_t)(&(((s *)0)->m))) #endif -#ifdef HAVE_3ARGS_INIT_WORK - -#define spl_init_work(wq,cb,d) INIT_WORK((wq), (void *)(cb), (void *)(d)) -#define spl_get_work_data(type,field,data) (data) - -#else - -#define spl_init_work(wq,cb,d) INIT_WORK((wq), (void *)(cb)); -#define spl_get_work_data(type,field,data) container_of(data,type,field) - -#endif - #ifdef __cplusplus } #endif diff --git a/include/sys/vmsystm.h b/include/sys/vmsystm.h index e92c17bdd..1cb716f13 100644 --- a/include/sys/vmsystm.h +++ b/include/sys/vmsystm.h @@ -35,8 +35,7 @@ extern vmem_t *zio_alloc_arena; /* arena for zio caches */ #define physmem num_physpages -#define freemem nr_free_pages() // Expensive on linux, - // cheap on solaris +#define freemem nr_free_pages() #define minfree 0 #define needfree 0 /* # of needed pages */ #define ptob(pages) (pages * PAGE_SIZE) -- cgit v1.2.3