Improve performance of zio_taskq_member

__zio_execute() calls zio_taskq_member() to determine if we are running
in a zio interrupt taskq, in which case we may need to switch to
processing this zio in a zio issue taskq.  The call to
zio_taskq_member() can become a performance bottleneck when we are
processing a high rate of zio's.

zio_taskq_member() calls taskq_member() on each of the zio interrupt
taskqs, of which there are 21.  This is slow because each call to
taskq_member() does tsd_get(taskq_tsd), which on Linux is relatively
slow.

This commit improves the performance of zio_taskq_member() by having it
cache the value of tsd_get(taskq_tsd), reducing the number of those
calls to 1/21th of the current behavior.

In a test case running `zfs send -c >/dev/null` of a filesystem with
small blocks (average 2.5KB/block), zio_taskq_member() was using 6.7% of
one CPU, and with this change it is reduced to 1.3%.  Overall time to
perform the `zfs send` reduced by 10% (~150,000 block/sec to ~165,000
blocks/sec).

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Serapheim Dimitropoulos <[email protected]>
Reviewed-by: Ryan Moeller <[email protected]>
Reviewed-by: Tony Nguyen <[email protected]>
Signed-off-by: Matthew Ahrens <[email protected]>
Closes #10070 

5 files changed, 24 insertions, 2 deletions

diff --git a/include/os/linux/spl/sys/taskq.h b/include/os/linux/spl/sys/taskq.h
index 7353367a2..7a1ee9ec4 100644
--- a/include/os/linux/spl/sys/taskq.h
+++ b/include/os/linux/spl/sys/taskq.h
@@ -151,6 +151,7 @@ extern void taskq_wait_outstanding(taskq_t *, taskqid_t);
 extern void taskq_wait(taskq_t *);
 extern int taskq_cancel_id(taskq_t *, taskqid_t);
 extern int taskq_member(taskq_t *, kthread_t *);
+extern taskq_t *taskq_of_curthread(void);
 
 #define	taskq_create_proc(name, nthreads, pri, min, max, proc, flags) \
     taskq_create(name, nthreads, pri, min, max, flags)
diff --git a/include/sys/zfs_context.h b/include/sys/zfs_context.h
index 5a67ca677..39c778b1e 100644
--- a/include/sys/zfs_context.h
+++ b/include/sys/zfs_context.h
@@ -494,6 +494,7 @@ extern void	taskq_wait(taskq_t *);
 extern void	taskq_wait_id(taskq_t *, taskqid_t);
 extern void	taskq_wait_outstanding(taskq_t *, taskqid_t);
 extern int	taskq_member(taskq_t *, kthread_t *);
+extern taskq_t	*taskq_of_curthread(void);
 extern int	taskq_cancel_id(taskq_t *, taskqid_t);
 extern void	system_taskq_init(void);
 extern void	system_taskq_fini(void);
diff --git a/lib/libzpool/taskq.c b/lib/libzpool/taskq.c
index ae67906fe..456080f7f 100644
--- a/lib/libzpool/taskq.c
+++ b/lib/libzpool/taskq.c
@@ -34,6 +34,8 @@ int taskq_now;
 taskq_t *system_taskq;
 taskq_t *system_delay_taskq;
 
+static pthread_key_t taskq_tsd;
+
 #define	TASKQ_ACTIVE	0x00010000
 
 static taskq_ent_t *
@@ -213,6 +215,8 @@ taskq_thread(void *arg)
 	taskq_ent_t *t;
 	boolean_t prealloc;
 
+	VERIFY0(pthread_setspecific(taskq_tsd, tq));
+
 	mutex_enter(&tq->tq_lock);
 	while (tq->tq_flags & TASKQ_ACTIVE) {
 		if ((t = tq->tq_task.tqent_next) == &tq->tq_task) {
@@ -343,6 +347,12 @@ taskq_member(taskq_t *tq, kthread_t *t)
 	return (0);
 }
 
+taskq_t *
+taskq_of_curthread(void)
+{
+	return (pthread_getspecific(taskq_tsd));
+}
+
 int
 taskq_cancel_id(taskq_t *tq, taskqid_t id)
 {
@@ -352,6 +362,7 @@ taskq_cancel_id(taskq_t *tq, taskqid_t id)
 void
 system_taskq_init(void)
 {
+	VERIFY0(pthread_key_create(&taskq_tsd, NULL));
 	system_taskq = taskq_create("system_taskq", 64, maxclsyspri, 4, 512,
 	    TASKQ_DYNAMIC | TASKQ_PREPOPULATE);
 	system_delay_taskq = taskq_create("delay_taskq", 4, maxclsyspri, 4,
@@ -365,4 +376,5 @@ system_taskq_fini(void)
 	system_taskq = NULL; /* defensive */
 	taskq_destroy(system_delay_taskq);
 	system_delay_taskq = NULL;
+	VERIFY0(pthread_key_delete(taskq_tsd));
 }
diff --git a/module/os/linux/spl/spl-taskq.c b/module/os/linux/spl/spl-taskq.c
index 4df44ade8..9cbf3e381 100644
--- a/module/os/linux/spl/spl-taskq.c
+++ b/module/os/linux/spl/spl-taskq.c
@@ -491,6 +491,13 @@ taskq_member(taskq_t *tq, kthread_t *t)
 }
 EXPORT_SYMBOL(taskq_member);
 
+taskq_t *
+taskq_of_curthread(void)
+{
+	return (tsd_get(taskq_tsd));
+}
+EXPORT_SYMBOL(taskq_of_curthread);
+
 /*
  * Cancel an already dispatched task given the task id.  Still pending tasks
  * will be immediately canceled, and if the task is active the function will
diff --git a/module/zfs/zio.c b/module/zfs/zio.c
index cbe6fc751..bf6b5afab 100644
--- a/module/zfs/zio.c
+++ b/module/zfs/zio.c
@@ -1865,14 +1865,15 @@ zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline)
 static boolean_t
 zio_taskq_member(zio_t *zio, zio_taskq_type_t q)
 {
-	kthread_t *executor = zio->io_executor;
 	spa_t *spa = zio->io_spa;
 
+	taskq_t *tq = taskq_of_curthread();
+
 	for (zio_type_t t = 0; t < ZIO_TYPES; t++) {
 		spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
 		uint_t i;
 		for (i = 0; i < tqs->stqs_count; i++) {
-			if (taskq_member(tqs->stqs_taskq[i], executor))
+			if (tqs->stqs_taskq[i] == tq)
 				return (B_TRUE);
 		}
 	}