diff options
| -rw-r--r-- | config/spl-build.m4 | 22 | ||||
| -rwxr-xr-x | configure | 104 | ||||
| -rw-r--r-- | include/sys/atomic.h | 146 | ||||
| -rw-r--r-- | module/spl/spl-atomic.c | 6 | ||||
| -rw-r--r-- | spl_config.h.in | 3 |
5 files changed, 202 insertions, 79 deletions
diff --git a/config/spl-build.m4 b/config/spl-build.m4 index e6a626108..15d10841b 100644 --- a/config/spl-build.m4 +++ b/config/spl-build.m4 @@ -25,6 +25,7 @@ AC_DEFUN([SPL_AC_CONFIG_KERNEL], [ SPL_AC_DEBUG_KMEM SPL_AC_DEBUG_KSTAT SPL_AC_DEBUG_CALLB + SPL_AC_ATOMIC_SPINLOCK SPL_AC_TYPE_UINTPTR_T SPL_AC_TYPE_ATOMIC64_T SPL_AC_3ARGS_INIT_WORK @@ -303,6 +304,27 @@ AC_DEFUN([SPL_AC_DEBUG_CALLB], [ ]) dnl # +dnl # Use the atomic implemenation based on global spinlocks. This +dnl # should never be needed, however it has been left in place as +dnl # a fallback option in case problems are observed with directly +dnl # mapping to the native Linux atomic operations. +dnl # +AC_DEFUN([SPL_AC_ATOMIC_SPINLOCK], [ + AC_ARG_ENABLE([atomic-spinlocks], + [AS_HELP_STRING([--enable-atomic-spinlocks], + [Atomic types use spinlocks @<:@default=no@:>@])], + [], + [enable_atomic_spinlocks=no]) + + AS_IF([test "x$enable_atomic_spinlocks" = xyes], + [AC_DEFINE([ATOMIC_SPINLOCK], [1], + [Atomic types use spinlocks])]) + + AC_MSG_CHECKING([whether atomic types use spinlocks]) + AC_MSG_RESULT([$enable_atomic_spinlocks]) +]) + +dnl # dnl # SPL_LINUX_CONFTEST dnl # AC_DEFUN([SPL_LINUX_CONFTEST], [ @@ -1038,6 +1038,8 @@ Optional Features: --enable-debug-kmem Enable kmem debug support (default off) --enable-debug-kstat Enable kstat debug support (default off) --enable-debug-callb Enable callb debug support (default off) + --enable-atomic-spinlocks + Atomic types use spinlocks [default=no] Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] @@ -3986,7 +3988,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 3989 "configure"' > conftest.$ac_ext + echo '#line 3991 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -5585,7 +5587,7 @@ fi # Provide some information about the compiler. -echo "$as_me:5588:" \ +echo "$as_me:5590:" \ "checking for Fortran 77 compiler version" >&5 ac_compiler=`set X $ac_compile; echo $2` { (eval echo "$as_me:$LINENO: \"$ac_compiler --version </dev/null >&5\"") >&5 @@ -6648,11 +6650,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6651: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6653: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6655: \$? = $ac_status" >&5 + echo "$as_me:6657: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -6916,11 +6918,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6919: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6921: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6923: \$? = $ac_status" >&5 + echo "$as_me:6925: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7020,11 +7022,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7023: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7025: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7027: \$? = $ac_status" >&5 + echo "$as_me:7029: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -8489,7 +8491,7 @@ linux*) libsuff= case "$host_cpu" in x86_64*|s390x*|powerpc64*) - echo '#line 8492 "configure"' > conftest.$ac_ext + echo '#line 8494 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -9386,7 +9388,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<EOF -#line 9389 "configure" +#line 9391 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9486,7 +9488,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<EOF -#line 9489 "configure" +#line 9491 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -11829,11 +11831,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:11832: $lt_compile\"" >&5) + (eval echo "\"\$as_me:11834: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:11836: \$? = $ac_status" >&5 + echo "$as_me:11838: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -11933,11 +11935,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:11936: $lt_compile\"" >&5) + (eval echo "\"\$as_me:11938: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:11940: \$? = $ac_status" >&5 + echo "$as_me:11942: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -12469,7 +12471,7 @@ linux*) libsuff= case "$host_cpu" in x86_64*|s390x*|powerpc64*) - echo '#line 12472 "configure"' > conftest.$ac_ext + echo '#line 12474 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -13527,11 +13529,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:13530: $lt_compile\"" >&5) + (eval echo "\"\$as_me:13532: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:13534: \$? = $ac_status" >&5 + echo "$as_me:13536: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -13631,11 +13633,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:13634: $lt_compile\"" >&5) + (eval echo "\"\$as_me:13636: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:13638: \$? = $ac_status" >&5 + echo "$as_me:13640: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -15080,7 +15082,7 @@ linux*) libsuff= case "$host_cpu" in x86_64*|s390x*|powerpc64*) - echo '#line 15083 "configure"' > conftest.$ac_ext + echo '#line 15085 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -15858,11 +15860,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:15861: $lt_compile\"" >&5) + (eval echo "\"\$as_me:15863: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:15865: \$? = $ac_status" >&5 + echo "$as_me:15867: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -16126,11 +16128,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:16129: $lt_compile\"" >&5) + (eval echo "\"\$as_me:16131: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:16133: \$? = $ac_status" >&5 + echo "$as_me:16135: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -16230,11 +16232,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:16233: $lt_compile\"" >&5) + (eval echo "\"\$as_me:16235: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:16237: \$? = $ac_status" >&5 + echo "$as_me:16239: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -17699,7 +17701,7 @@ linux*) libsuff= case "$host_cpu" in x86_64*|s390x*|powerpc64*) - echo '#line 17702 "configure"' > conftest.$ac_ext + echo '#line 17704 "configure"' > conftest.$ac_ext if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -19203,6 +19205,29 @@ _ACEOF echo "${ECHO_T}no" >&6 fi + + # Check whether --enable-atomic-spinlocks or --disable-atomic-spinlocks was given. +if test "${enable_atomic_spinlocks+set}" = set; then + enableval="$enable_atomic_spinlocks" + +else + enable_atomic_spinlocks=no +fi; + + if test "x$enable_atomic_spinlocks" = xyes; then + +cat >>confdefs.h <<\_ACEOF +#define ATOMIC_SPINLOCK 1 +_ACEOF + +fi + + + echo "$as_me:$LINENO: checking whether atomic types use spinlocks" >&5 +echo $ECHO_N "checking whether atomic types use spinlocks... $ECHO_C" >&6 + echo "$as_me:$LINENO: result: $enable_atomic_spinlocks" >&5 +echo "${ECHO_T}$enable_atomic_spinlocks" >&6 + echo "$as_me:$LINENO: checking whether kernel defines uintptr_t" >&5 echo $ECHO_N "checking whether kernel defines uintptr_t... $ECHO_C" >&6 @@ -22351,6 +22376,29 @@ _ACEOF echo "${ECHO_T}no" >&6 fi + + # Check whether --enable-atomic-spinlocks or --disable-atomic-spinlocks was given. +if test "${enable_atomic_spinlocks+set}" = set; then + enableval="$enable_atomic_spinlocks" + +else + enable_atomic_spinlocks=no +fi; + + if test "x$enable_atomic_spinlocks" = xyes; then + +cat >>confdefs.h <<\_ACEOF +#define ATOMIC_SPINLOCK 1 +_ACEOF + +fi + + + echo "$as_me:$LINENO: checking whether atomic types use spinlocks" >&5 +echo $ECHO_N "checking whether atomic types use spinlocks... $ECHO_C" >&6 + echo "$as_me:$LINENO: result: $enable_atomic_spinlocks" >&5 +echo "${ECHO_T}$enable_atomic_spinlocks" >&6 + echo "$as_me:$LINENO: checking whether kernel defines uintptr_t" >&5 echo $ECHO_N "checking whether kernel defines uintptr_t... $ECHO_C" >&6 diff --git a/include/sys/atomic.h b/include/sys/atomic.h index cd0eb3b0a..4f4a1e058 100644 --- a/include/sys/atomic.h +++ b/include/sys/atomic.h @@ -27,31 +27,94 @@ #ifndef _SPL_ATOMIC_H #define _SPL_ATOMIC_H -#ifdef __cplusplus -extern "C" { -#endif - #include <linux/module.h> #include <linux/spinlock.h> -#include <sys/isa_defs.h> +#include <sys/types.h> -/* XXX: Serialize everything through global locks. This is - * going to be bad for performance, but for now it's the easiest - * way to ensure correct behavior. I don't like it at all. - * It would be nicer to make these function to the atomic linux - * functions, but the normal uint64_t type complicates this. +/* + * Two approaches to atomic operations are implemented each with its + * own benefits are drawbacks imposed by the Solaris API. Neither + * approach handles the issue of word breaking when using a 64-bit + * atomic variable on a 32-bit arch. The Solaris API would need to + * add an atomic read call to correctly support this. + * + * When ATOMIC_SPINLOCK is defined all atomic operations will be + * serialized through global spin locks. This is bad for performance + * but it does allow a simple generic implementation. + * + * When ATOMIC_SPINLOCK is not defined the Linux atomic operations + * are used. This is safe as long as the core Linux implementation + * doesn't change because we are relying on the fact that an atomic + * type is really just a uint32 or uint64. If this changes at some + * point in the future we need to fall-back to the spin approach. */ -extern spinlock_t atomic64_lock; +#ifdef ATOMIC_SPINLOCK extern spinlock_t atomic32_lock; +extern spinlock_t atomic64_lock; -static __inline__ uint32_t +static __inline__ void +atomic_inc_32(volatile uint32_t *target) +{ + spin_lock(&atomic32_lock); + (*target)++; + spin_unlock(&atomic32_lock); +} + +static __inline__ void +atomic_dec_32(volatile uint32_t *target) +{ + spin_lock(&atomic32_lock); + (*target)--; + spin_unlock(&atomic32_lock); +} + +static __inline__ void atomic_add_32(volatile uint32_t *target, int32_t delta) { + spin_lock(&atomic32_lock); + *target += delta; + spin_unlock(&atomic32_lock); +} + +static __inline__ void +atomic_sub_32(volatile uint32_t *target, int32_t delta) +{ + spin_lock(&atomic32_lock); + *target -= delta; + spin_unlock(&atomic32_lock); +} + +static __inline__ uint32_t +atomic_add_32_nv(volatile uint32_t *target, uint32_t delta) +{ + spin_lock(&atomic32_lock); + *target += delta; + spin_unlock(&atomic32_lock); + + return *target; +} + +static __inline__ uint32_t +atomic_sub_32_nv(volatile uint32_t *target, uint32_t delta) +{ + spin_lock(&atomic32_lock); + *target -= delta; + spin_unlock(&atomic32_lock); + + return *target; +} + +static __inline__ uint32_t +atomic_cas_32(volatile uint32_t *target, uint32_t cmp, + uint32_t newval) +{ uint32_t rc; spin_lock(&atomic32_lock); rc = *target; - *target += delta; + if (*target == cmp) + *target = newval; + spin_unlock(&atomic32_lock); return rc; @@ -73,30 +136,20 @@ atomic_dec_64(volatile uint64_t *target) spin_unlock(&atomic64_lock); } -static __inline__ uint64_t +static __inline__ void atomic_add_64(volatile uint64_t *target, uint64_t delta) { - uint64_t rc; - spin_lock(&atomic64_lock); - rc = *target; *target += delta; spin_unlock(&atomic64_lock); - - return rc; } -static __inline__ uint64_t +static __inline__ void atomic_sub_64(volatile uint64_t *target, uint64_t delta) { - uint64_t rc; - spin_lock(&atomic64_lock); - rc = *target; *target -= delta; spin_unlock(&atomic64_lock); - - return rc; } static __inline__ uint64_t @@ -121,7 +174,7 @@ atomic_sub_64_nv(volatile uint64_t *target, uint64_t delta) static __inline__ uint64_t atomic_cas_64(volatile uint64_t *target, uint64_t cmp, - uint64_t newval) + uint64_t newval) { uint64_t rc; @@ -134,45 +187,40 @@ atomic_cas_64(volatile uint64_t *target, uint64_t cmp, return rc; } -static __inline__ uint32_t -atomic_cas_32(volatile uint32_t *target, uint32_t cmp, - uint32_t newval) -{ - uint32_t rc; - spin_lock(&atomic32_lock); - rc = *target; - if (*target == cmp) - *target = newval; +#else /* ATOMIC_SPINLOCK */ - spin_unlock(&atomic32_lock); +#define atomic_inc_32(v) atomic_inc((atomic_t *)(v)) +#define atomic_dec_32(v) atomic_dec((atomic_t *)(v)) +#define atomic_add_32(v, i) atomic_add((i), (atomic_t *)(v)) +#define atomic_sub_32(v, i) atomic_sub((i), (atomic_t *)(v)) +#define atomic_add_32_nv(v, i) atomic_add_return((i), (atomic_t *)(v)) +#define atomic_sub_32_nv(v, i) atomic_sub_return((i), (atomic_t *)(v)) +#define atomic_cas_32(v, x, y) atomic_cmpxchg((atomic_t *)(v), x, y) +#define atomic_inc_64(v) atomic64_inc((atomic64_t *)(v)) +#define atomic_dec_64(v) atomic64_dec((atomic64_t *)(v)) +#define atomic_add_64(v, i) atomic64_add((i), (atomic64_t *)(v)) +#define atomic_sub_64(v, i) atomic64_sub((i), (atomic64_t *)(v)) +#define atomic_add_64_nv(v, i) atomic64_add_return((i), (atomic64_t *)(v)) +#define atomic_sub_64_nv(v, i) atomic64_sub_return((i), (atomic64_t *)(v)) +#define atomic_cas_64(v, x, y) atomic64_cmpxchg((atomic64_t *)(v), x, y) - return rc; -} +#endif /* ATOMIC_SPINLOCK */ #ifdef _LP64 -/* XXX: Implement atomic_cas_ptr() in terms of uint64'ts. This - * is of course only safe and correct for 64 bit arches... but - * for now I'm OK with that. - */ static __inline__ void * atomic_cas_ptr(volatile void *target, void *cmp, void *newval) { return (void *)atomic_cas_64((volatile uint64_t *)target, (uint64_t)cmp, (uint64_t)newval); } -#else +#else /* _LP64 */ static __inline__ void * atomic_cas_ptr(volatile void *target, void *cmp, void *newval) { return (void *)atomic_cas_32((volatile uint32_t *)target, (uint32_t)cmp, (uint32_t)newval); } -#endif - -#ifdef __cplusplus -} -#endif +#endif /* _LP64 */ #endif /* _SPL_ATOMIC_H */ - diff --git a/module/spl/spl-atomic.c b/module/spl/spl-atomic.c index 40cdb06cc..decf9515e 100644 --- a/module/spl/spl-atomic.c +++ b/module/spl/spl-atomic.c @@ -32,9 +32,11 @@ #define DEBUG_SUBSYSTEM S_ATOMIC +#ifdef ATOMIC_SPINLOCK /* Global atomic lock declarations */ -spinlock_t atomic64_lock = SPIN_LOCK_UNLOCKED; spinlock_t atomic32_lock = SPIN_LOCK_UNLOCKED; +spinlock_t atomic64_lock = SPIN_LOCK_UNLOCKED; -EXPORT_SYMBOL(atomic64_lock); EXPORT_SYMBOL(atomic32_lock); +EXPORT_SYMBOL(atomic64_lock); +#endif /* ATOMIC_SPINLOCK */ diff --git a/spl_config.h.in b/spl_config.h.in index 176da466b..feb09a865 100644 --- a/spl_config.h.in +++ b/spl_config.h.in @@ -1,5 +1,8 @@ /* spl_config.h.in. Generated from configure.ac by autoheader. */ +/* Atomic types use spinlocks */ +#undef ATOMIC_SPINLOCK + /* Define to 1 to enable callb debugging */ #undef DEBUG_CALLB |