Use tsd to store tq for taskq_member

To prevent taskq_member holding tq_lock and doing linear search, thus causing
contention. We store the taskq pointer to which the thread belongs in tsd.
This way taskq_member will not need to touch tq_lock, and tsd has per slot
spinlock. So the contention should be reduced greatly.

Signed-off-by: Chunwei Chen <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #500
Closes #504
Closes #505

5 files changed, 63 insertions, 57 deletions

diff --git a/include/sys/taskq.h b/include/sys/taskq.h
index 544dbb2bb..e7661f7ce 100644
--- a/include/sys/taskq.h
+++ b/include/sys/taskq.h
@@ -140,7 +140,7 @@ extern void taskq_wait_id(taskq_t *, taskqid_t);
 extern void taskq_wait_outstanding(taskq_t *, taskqid_t);
 extern void taskq_wait(taskq_t *);
 extern int taskq_cancel_id(taskq_t *, taskqid_t);
-extern int taskq_member(taskq_t *, void *);
+extern int taskq_member(taskq_t *, kthread_t *);
 
 #define	taskq_create_proc(name, nthreads, pri, min, max, proc, flags) \
     taskq_create(name, nthreads, pri, min, max, flags)
diff --git a/include/sys/tsd.h b/include/sys/tsd.h
index ebc55b09b..1894a8232 100644
--- a/include/sys/tsd.h
+++ b/include/sys/tsd.h
@@ -35,6 +35,7 @@ typedef void (*dtor_func_t)(void *);
 
 extern int tsd_set(uint_t, void *);
 extern void *tsd_get(uint_t);
+extern void *tsd_get_by_thread(uint_t, kthread_t *);
 extern void tsd_create(uint_t *, dtor_func_t);
 extern void tsd_destroy(uint_t *);
 extern void tsd_exit(void);
diff --git a/module/spl/spl-generic.c b/module/spl/spl-generic.c
index 4d9846cf5..dc3e74aa5 100644
--- a/module/spl/spl-generic.c
+++ b/module/spl/spl-generic.c
@@ -491,29 +491,20 @@ spl_kvmem_init(void)
 
 	rc = spl_kmem_init();
 	if (rc)
-		goto out1;
+		return (rc);
 
 	rc = spl_vmem_init();
-	if (rc)
-		goto out2;
-
-	rc = spl_kmem_cache_init();
-	if (rc)
-		goto out3;
+	if (rc) {
+		spl_kmem_fini();
+		return (rc);
+	}
 
 	return (rc);
-out3:
-	spl_vmem_fini();
-out2:
-	spl_kmem_fini();
-out1:
-	return (rc);
 }
 
 static void
 spl_kvmem_fini(void)
 {
-	spl_kmem_cache_fini();
 	spl_vmem_fini();
 	spl_kmem_fini();
 }
@@ -532,38 +523,43 @@ spl_init(void)
 	if ((rc = spl_rw_init()))
 		goto out3;
 
-	if ((rc = spl_taskq_init()))
+	if ((rc = spl_tsd_init()))
 		goto out4;
 
-	if ((rc = spl_vn_init()))
+	if ((rc = spl_taskq_init()))
 		goto out5;
 
-	if ((rc = spl_proc_init()))
+	if ((rc = spl_kmem_cache_init()))
 		goto out6;
 
-	if ((rc = spl_kstat_init()))
+	if ((rc = spl_vn_init()))
 		goto out7;
 
-	if ((rc = spl_tsd_init()))
+	if ((rc = spl_proc_init()))
 		goto out8;
 
-	if ((rc = spl_zlib_init()))
+	if ((rc = spl_kstat_init()))
 		goto out9;
 
+	if ((rc = spl_zlib_init()))
+		goto out10;
+
 	printk(KERN_NOTICE "SPL: Loaded module v%s-%s%s\n", SPL_META_VERSION,
 	       SPL_META_RELEASE, SPL_DEBUG_STR);
 	return (rc);
 
+out10:
+	spl_kstat_fini();
 out9:
-	spl_tsd_fini();
+	spl_proc_fini();
 out8:
-	spl_kstat_fini();
+	spl_vn_fini();
 out7:
-	spl_proc_fini();
+	spl_kmem_cache_fini();
 out6:
-	spl_vn_fini();
-out5:
 	spl_taskq_fini();
+out5:
+	spl_tsd_fini();
 out4:
 	spl_rw_fini();
 out3:
@@ -584,11 +580,12 @@ spl_fini(void)
 	printk(KERN_NOTICE "SPL: Unloaded module v%s-%s%s\n",
 	       SPL_META_VERSION, SPL_META_RELEASE, SPL_DEBUG_STR);
 	spl_zlib_fini();
-	spl_tsd_fini();
 	spl_kstat_fini();
 	spl_proc_fini();
 	spl_vn_fini();
+	spl_kmem_cache_fini();
 	spl_taskq_fini();
+	spl_tsd_fini();
 	spl_rw_fini();
 	spl_mutex_fini();
 	spl_kvmem_fini();
diff --git a/module/spl/spl-taskq.c b/module/spl/spl-taskq.c
index 5f2b725c9..2b3f3f4bc 100644
--- a/module/spl/spl-taskq.c
+++ b/module/spl/spl-taskq.c
@@ -26,6 +26,7 @@
 
 #include <sys/taskq.h>
 #include <sys/kmem.h>
+#include <sys/tsd.h>
 
 int spl_taskq_thread_bind = 0;
 module_param(spl_taskq_thread_bind, int, 0644);
@@ -57,6 +58,7 @@ static taskq_thread_t *taskq_thread_create(taskq_t *);
 /* List of all taskqs */
 LIST_HEAD(tq_list);
 DECLARE_RWSEM(tq_list_sem);
+static uint_t taskq_tsd;
 
 static int
 task_km_flags(uint_t flags)
@@ -474,38 +476,10 @@ taskq_wait(taskq_t *tq)
 }
 EXPORT_SYMBOL(taskq_wait);
 
-static int
-taskq_member_impl(taskq_t *tq, void *t)
-{
-	struct list_head *l;
-	taskq_thread_t *tqt;
-	int found = 0;
-
-	ASSERT(tq);
-	ASSERT(t);
-	ASSERT(spin_is_locked(&tq->tq_lock));
-
-	list_for_each(l, &tq->tq_thread_list) {
-		tqt = list_entry(l, taskq_thread_t, tqt_thread_list);
-		if (tqt->tqt_thread == (struct task_struct *)t) {
-			found = 1;
-			break;
-		}
-	}
-	return (found);
-}
-
 int
-taskq_member(taskq_t *tq, void *t)
+taskq_member(taskq_t *tq, kthread_t *t)
 {
-	int found;
-	unsigned long flags;
-
-	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
-	found = taskq_member_impl(tq, t);
-	spin_unlock_irqrestore(&tq->tq_lock, flags);
-
-	return (found);
+	return (tq == (taskq_t *)tsd_get_by_thread(taskq_tsd, t));
 }
 EXPORT_SYMBOL(taskq_member);
 
@@ -855,6 +829,7 @@ taskq_thread(void *args)
 	sigprocmask(SIG_BLOCK, &blocked, NULL);
 	flush_signals(current);
 
+	tsd_set(taskq_tsd, tq);
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 
 	/* Immediately exit if more threads than allowed were created. */
@@ -959,6 +934,8 @@ error:
 	kmem_free(tqt, sizeof (taskq_thread_t));
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 
+	tsd_set(taskq_tsd, NULL);
+
 	return (0);
 }
 
@@ -1160,6 +1137,8 @@ EXPORT_SYMBOL(taskq_destroy);
 int
 spl_taskq_init(void)
 {
+	tsd_create(&taskq_tsd, NULL);
+
 	system_taskq = taskq_create("spl_system_taskq", MAX(boot_ncpus, 64),
 	    maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC);
 	if (system_taskq == NULL)
@@ -1190,4 +1169,6 @@ spl_taskq_fini(void)
 
 	taskq_destroy(system_taskq);
 	system_taskq = NULL;
+
+	tsd_destroy(&taskq_tsd);
 }
diff --git a/module/spl/spl-tsd.c b/module/spl/spl-tsd.c
index 4d0800e5a..bf8235063 100644
--- a/module/spl/spl-tsd.c
+++ b/module/spl/spl-tsd.c
@@ -528,6 +528,33 @@ tsd_get(uint_t key)
 EXPORT_SYMBOL(tsd_get);
 
 /*
+ * tsd_get_by_thread - get thread specific data for specified thread
+ * @key: lookup key
+ * @thread: thread to lookup
+ *
+ * Caller must prevent racing tsd_create() or tsd_destroy().  This
+ * implementation is designed to be fast and scalable, it does not
+ * lock the entire table only a single hash bin.
+ */
+void *
+tsd_get_by_thread(uint_t key, kthread_t *thread)
+{
+	tsd_hash_entry_t *entry;
+
+	ASSERT3P(tsd_hash_table, !=, NULL);
+
+	if ((key == 0) || (key > TSD_KEYS_MAX))
+		return (NULL);
+
+	entry = tsd_hash_search(tsd_hash_table, key, thread->pid);
+	if (entry == NULL)
+		return (NULL);
+
+	return (entry->he_value);
+}
+EXPORT_SYMBOL(tsd_get_by_thread);
+
+/*
  * tsd_create - create thread specific data key
  * @keyp: lookup key address
  * @dtor: destructor called during tsd_destroy() or tsd_exit()