diff options
| author | Chunwei Chen <[email protected]> | 2016-05-25 16:35:42 -0700 |
|---|---|---|
| committer | Brian Behlendorf <[email protected]> | 2016-05-31 11:44:15 -0700 |
| commit | f58040c0fc8bc6490fcc75db7fc3e709dfc3c656 (patch) | |
| tree | 03aaebb8e0e152ade6edb7a9ae14f4b430314797 /include | |
| parent | c60a51b640bab61c54f370752750841675730899 (diff) | |
Implement a proper rw_tryupgrade
Current rw_tryupgrade does rw_exit and then rw_tryenter(RW_RWITER), and then
does rw_enter(RW_READER) if it fails. This violate the assumption that
rw_tryupgrade should be atomic and could cause extra contention or even lock
inversion.
This patch we implement a proper rw_tryupgrade. For rwsem-spinlock, we take
the spinlock to check rwsem->count and rwsem->wait_list. For normal rwsem, we
use cmpxchg on rwsem->count to change the value from single reader to single
writer.
Signed-off-by: Chunwei Chen <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Signed-off-by: Tim Chase <[email protected]>
Closes zfsonlinux/zfs#4692
Closes #554
Diffstat (limited to 'include')
| -rw-r--r-- | include/linux/rwsem_compat.h | 17 | ||||
| -rw-r--r-- | include/sys/rwlock.h | 11 |
2 files changed, 21 insertions, 7 deletions
diff --git a/include/linux/rwsem_compat.h b/include/linux/rwsem_compat.h index 5841d7c28..9a4df2673 100644 --- a/include/linux/rwsem_compat.h +++ b/include/linux/rwsem_compat.h @@ -27,6 +27,23 @@ #include <linux/rwsem.h> +#ifdef CONFIG_RWSEM_GENERIC_SPINLOCK +#define SPL_RWSEM_SINGLE_READER_VALUE (1) +#define SPL_RWSEM_SINGLE_WRITER_VALUE (-1) +#else +#define SPL_RWSEM_SINGLE_READER_VALUE (RWSEM_ACTIVE_READ_BIAS) +#define SPL_RWSEM_SINGLE_WRITER_VALUE (RWSEM_ACTIVE_WRITE_BIAS) +#endif + +/* Linux 3.16 change activity to count for rwsem-spinlock */ +#ifdef HAVE_RWSEM_ACTIVITY +#define RWSEM_COUNT(sem) sem->activity +#else +#define RWSEM_COUNT(sem) sem->count +#endif + +int rwsem_tryupgrade(struct rw_semaphore *rwsem); + #if defined(RWSEM_SPINLOCK_IS_RAW) #define spl_rwsem_lock_irqsave(lk, fl) raw_spin_lock_irqsave(lk, fl) #define spl_rwsem_unlock_irqrestore(lk, fl) raw_spin_unlock_irqrestore(lk, fl) diff --git a/include/sys/rwlock.h b/include/sys/rwlock.h index 14d097b01..facebe3ba 100644 --- a/include/sys/rwlock.h +++ b/include/sys/rwlock.h @@ -223,13 +223,10 @@ RW_LOCK_HELD(krwlock_t *rwp) if (RW_WRITE_HELD(rwp)) { \ _rc_ = 1; \ } else { \ - rw_exit(rwp); \ - if (rw_tryenter(rwp, RW_WRITER)) { \ - _rc_ = 1; \ - } else { \ - rw_enter(rwp, RW_READER); \ - _rc_ = 0; \ - } \ + spl_rw_lockdep_off_maybe(rwp); \ + if ((_rc_ = rwsem_tryupgrade(SEM(rwp)))) \ + spl_rw_set_owner(rwp); \ + spl_rw_lockdep_on_maybe(rwp); \ } \ _rc_; \ }) |