aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBrian Behlendorf <behlendorf1@llnl.gov>2016-03-09 14:20:48 -0800
committerBrian Behlendorf <behlendorf1@llnl.gov>2016-03-10 13:05:25 -0800
commita6ae97caed620b0e9e0575346062c751f6f15483 (patch)
tree15c8b923a044403a5bda9c8e207ebe48f8af5902
parent47f98247814fff50d02ddc0772197842c57cdc3a (diff)
Add rw_tryupgrade()
This implementation of rw_tryupgrade() behaves slightly differently from its counterparts on other platforms. It drops the RW_READER lock and then acquires the RW_WRITER lock leaving a small window where no lock is held. On other platforms the lock is never released during the upgrade process. This is necessary under Linux because the kernel does not provide an upgrade function. There are currently no callers in the ZFS code where this change in behavior is a problem. In fact, in most cases the code is already written such that if the upgrade fails the RW_READER lock is dropped and the caller blocks waiting to acquire the lock as RW_WRITER. Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: Tim Chase <tim@chase2k.com> Signed-off-by: Matthew Thode <prometheanfire@gentoo.org> Closes zfsonlinux/zfs#4388 Closes #534
-rw-r--r--include/sys/rwlock.h50
-rw-r--r--module/spl/spl-rwlock.c60
-rw-r--r--module/splat/splat-rwlock.c6
3 files changed, 16 insertions, 100 deletions
diff --git a/include/sys/rwlock.h b/include/sys/rwlock.h
index c82764ce9..14d097b01 100644
--- a/include/sys/rwlock.h
+++ b/include/sys/rwlock.h
@@ -208,49 +208,31 @@ RW_LOCK_HELD(krwlock_t *rwp)
spl_rw_lockdep_on_maybe(rwp); \
})
-#if defined(CONFIG_RWSEM_GENERIC_SPINLOCK)
-#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
-#error spinlock rwsem should not have spin on owner
-#endif
/*
- * For the generic implementations of rw-semaphores the following is
- * true. If your semaphore implementation internally represents the
- * semaphore state differently then special case handling is required.
- * - if activity/count is 0 then there are no active readers or writers
- * - if activity/count is +ve then that is the number of active readers
- * - if activity/count is -1 then there is one active writer
+ * This implementation of rw_tryupgrade() behaves slightly differently
+ * from its counterparts on other platforms. It drops the RW_READER lock
+ * and then acquires the RW_WRITER lock leaving a small window where no
+ * lock is held. On other platforms the lock is never released during
+ * the upgrade process. This is necessary under Linux because the kernel
+ * does not provide an upgrade function.
*/
-
-extern void __up_read_locked(struct rw_semaphore *);
-extern int __down_write_trylock_locked(struct rw_semaphore *);
-
#define rw_tryupgrade(rwp) \
({ \
- unsigned long _flags_; \
int _rc_ = 0; \
\
- spl_rw_lockdep_off_maybe(rwp); \
- spl_rwsem_lock_irqsave(&SEM(rwp)->wait_lock, _flags_); \
- if ((list_empty(&SEM(rwp)->wait_list)) && \
- (SEM(rwp)->activity == 1)) { \
- __up_read_locked(SEM(rwp)); \
- VERIFY(_rc_ = __down_write_trylock_locked(SEM(rwp))); \
- (rwp)->rw_owner = current; \
+ if (RW_WRITE_HELD(rwp)) { \
+ _rc_ = 1; \
+ } else { \
+ rw_exit(rwp); \
+ if (rw_tryenter(rwp, RW_WRITER)) { \
+ _rc_ = 1; \
+ } else { \
+ rw_enter(rwp, RW_READER); \
+ _rc_ = 0; \
+ } \
} \
- spl_rwsem_unlock_irqrestore(&SEM(rwp)->wait_lock, _flags_); \
- spl_rw_lockdep_on_maybe(rwp); \
_rc_; \
})
-#else
-/*
- * rw_tryupgrade() can be implemented correctly but for each supported
- * arch we will need a custom implementation. For the x86 implementation
- * it looks like a custom cmpxchg() to atomically check and promote the
- * rwsem would be safe. For now that's not worth the trouble so in this
- * case rw_tryupgrade() has just been disabled.
- */
-#define rw_tryupgrade(rwp) ({ 0; })
-#endif
int spl_rw_init(void);
void spl_rw_fini(void);
diff --git a/module/spl/spl-rwlock.c b/module/spl/spl-rwlock.c
index 462a6f0de..98251c011 100644
--- a/module/spl/spl-rwlock.c
+++ b/module/spl/spl-rwlock.c
@@ -32,65 +32,5 @@
#define DEBUG_SUBSYSTEM S_RWLOCK
-#ifdef CONFIG_RWSEM_GENERIC_SPINLOCK
-
-/*
- * From lib/rwsem-spinlock.c but modified such that the caller is
- * responsible for acquiring and dropping the sem->wait_lock.
- */
-struct rwsem_waiter {
- struct list_head list;
- struct task_struct *task;
- unsigned int flags;
-#define RWSEM_WAITING_FOR_READ 0x00000001
-#define RWSEM_WAITING_FOR_WRITE 0x00000002
-};
-
-/* wake a single writer */
-static struct rw_semaphore *
-__rwsem_wake_one_writer_locked(struct rw_semaphore *sem)
-{
- struct rwsem_waiter *waiter;
- struct task_struct *tsk;
-
- sem->activity = -1;
-
- waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
- list_del(&waiter->list);
-
- tsk = waiter->task;
- smp_mb();
- waiter->task = NULL;
- wake_up_process(tsk);
- put_task_struct(tsk);
- return sem;
-}
-
-/* release a read lock on the semaphore */
-void
-__up_read_locked(struct rw_semaphore *sem)
-{
- if (--sem->activity == 0 && !list_empty(&sem->wait_list))
- (void)__rwsem_wake_one_writer_locked(sem);
-}
-EXPORT_SYMBOL(__up_read_locked);
-
-/* trylock for writing -- returns 1 if successful, 0 if contention */
-int
-__down_write_trylock_locked(struct rw_semaphore *sem)
-{
- int ret = 0;
-
- if (sem->activity == 0 && list_empty(&sem->wait_list)) {
- sem->activity = -1;
- ret = 1;
- }
-
- return ret;
-}
-EXPORT_SYMBOL(__down_write_trylock_locked);
-
-#endif
-
int spl_rw_init(void) { return 0; }
void spl_rw_fini(void) { }
diff --git a/module/splat/splat-rwlock.c b/module/splat/splat-rwlock.c
index 284f77370..abd6a0e60 100644
--- a/module/splat/splat-rwlock.c
+++ b/module/splat/splat-rwlock.c
@@ -588,7 +588,6 @@ splat_rwlock_test6(struct file *file, void *arg)
goto out;
}
-#if defined(CONFIG_RWSEM_GENERIC_SPINLOCK)
/* With one reader upgrade should never fail. */
rc = rw_tryupgrade(&rwp->rw_rwlock);
if (!rc) {
@@ -610,11 +609,6 @@ splat_rwlock_test6(struct file *file, void *arg)
rc = 0;
splat_vprint(file, SPLAT_RWLOCK_TEST6_NAME, "%s",
"rwlock properly upgraded\n");
-#else
- rc = 0;
- splat_vprint(file, SPLAT_RWLOCK_TEST6_NAME, "%s",
- "rw_tryupgrade() is disabled for this arch\n");
-#endif
out:
rw_exit(&rwp->rw_rwlock);
rw_destroy(&rwp->rw_rwlock);