Better control the thread pool size when mounting datasets

Ever since a10d50f999, ZFS has mounted file systems in parallel when
importing a pool.  It uses a fixed size of 512 for the thread pool.  But
since c183d164aa1, it has also imported pools in parallel.  So the total
number of threads at one time is 513 * npools + 1.  That can easily
exceed the system's limit on the number of threads per process, which
will cause one or more pools to be unable to allocate any worker
threads, forcing them to fallback to slow serial mounting .  To
forestall that, manage the threadpool size in /sbin/zpool, not libzfs.
Use the same size (512), but divided by the number of pools.

This is a backwards-incompatible change to the libzfs abi.

Sponsored by: Axcient
Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: George Wilson <[email protected]>
Signed-off-by: Alan Somers <[email protected]>
Closes #16178

2 files changed, 15 insertions, 13 deletions

diff --git a/lib/libzfs/libzfs.abi b/lib/libzfs/libzfs.abi
index 2bbaae634..c3efb2984 100644
--- a/lib/libzfs/libzfs.abi
+++ b/lib/libzfs/libzfs.abi
@@ -5532,13 +5532,14 @@
       <parameter type-id='b59d7dce' name='num_handles'/>
       <parameter type-id='d8e49ab9' name='func'/>
       <parameter type-id='eaa32e2f' name='data'/>
-      <parameter type-id='c19b74c3' name='parallel'/>
+      <parameter type-id='3502e3ff' name='nthr'/>
       <return type-id='48b5725f'/>
     </function-decl>
     <function-decl name='zpool_enable_datasets' mangled-name='zpool_enable_datasets' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_enable_datasets'>
       <parameter type-id='4c81de99' name='zhp'/>
       <parameter type-id='80f4b756' name='mntopts'/>
       <parameter type-id='95e97e5e' name='flags'/>
+      <parameter type-id='3502e3ff' name='nthr'/>
       <return type-id='95e97e5e'/>
     </function-decl>
     <function-decl name='zpool_disable_datasets' mangled-name='zpool_disable_datasets' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_disable_datasets'>
diff --git a/lib/libzfs/libzfs_mount.c b/lib/libzfs/libzfs_mount.c
index 3084e05e4..42988bf9c 100644
--- a/lib/libzfs/libzfs_mount.c
+++ b/lib/libzfs/libzfs_mount.c
@@ -83,8 +83,6 @@
 #include <sys/systeminfo.h>
 #define	MAXISALEN	257	/* based on sysinfo(2) man page */
 
-static int mount_tp_nthr = 512;	/* tpool threads for multi-threaded mounting */
-
 static void zfs_mount_task(void *);
 
 static const proto_table_t proto_table[SA_PROTOCOL_COUNT] = {
@@ -1205,19 +1203,20 @@ out:
  *
  * Callbacks are issued in one of two ways:
  *
- * 1. Sequentially: If the parallel argument is B_FALSE or the ZFS_SERIAL_MOUNT
+ * 1. Sequentially: If the nthr argument is <= 1 or the ZFS_SERIAL_MOUNT
  *    environment variable is set, then we issue callbacks sequentially.
  *
- * 2. In parallel: If the parallel argument is B_TRUE and the ZFS_SERIAL_MOUNT
+ * 2. In parallel: If the nthr argument is > 1 and the ZFS_SERIAL_MOUNT
  *    environment variable is not set, then we use a tpool to dispatch threads
  *    to mount filesystems in parallel. This function dispatches tasks to mount
  *    the filesystems at the top-level mountpoints, and these tasks in turn
  *    are responsible for recursively mounting filesystems in their children
- *    mountpoints.
+ *    mountpoints.  The value of the nthr argument will be the number of worker
+ *    threads for the thread pool.
  */
 void
 zfs_foreach_mountpoint(libzfs_handle_t *hdl, zfs_handle_t **handles,
-    size_t num_handles, zfs_iter_f func, void *data, boolean_t parallel)
+    size_t num_handles, zfs_iter_f func, void *data, uint_t nthr)
 {
 	zoneid_t zoneid = getzoneid();
 
@@ -1226,7 +1225,7 @@ zfs_foreach_mountpoint(libzfs_handle_t *hdl, zfs_handle_t **handles,
 	 * variable that can be used as a convenience to do a/b comparison
 	 * of serial vs. parallel mounting.
 	 */
-	boolean_t serial_mount = !parallel ||
+	boolean_t serial_mount = nthr <= 1 ||
 	    (getenv("ZFS_SERIAL_MOUNT") != NULL);
 
 	/*
@@ -1246,7 +1245,7 @@ zfs_foreach_mountpoint(libzfs_handle_t *hdl, zfs_handle_t **handles,
 	 * Issue the callback function for each dataset using a parallel
 	 * algorithm that uses a thread pool to manage threads.
 	 */
-	tpool_t *tp = tpool_create(1, mount_tp_nthr, 0, NULL);
+	tpool_t *tp = tpool_create(1, nthr, 0, NULL);
 
 	/*
 	 * There may be multiple "top level" mountpoints outside of the pool's
@@ -1273,10 +1272,12 @@ zfs_foreach_mountpoint(libzfs_handle_t *hdl, zfs_handle_t **handles,
 
 /*
  * Mount and share all datasets within the given pool.  This assumes that no
- * datasets within the pool are currently mounted.
+ * datasets within the pool are currently mounted.  nthr will be number of
+ * worker threads to use while mounting datasets.
  */
 int
-zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags)
+zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags,
+    uint_t nthr)
 {
 	get_all_cb_t cb = { 0 };
 	mount_state_t ms = { 0 };
@@ -1302,7 +1303,7 @@ zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags)
 	ms.ms_mntopts = mntopts;
 	ms.ms_mntflags = flags;
 	zfs_foreach_mountpoint(zhp->zpool_hdl, cb.cb_handles, cb.cb_used,
-	    zfs_mount_one, &ms, B_TRUE);
+	    zfs_mount_one, &ms, nthr);
 	if (ms.ms_mntstatus != 0)
 		ret = EZFS_MOUNTFAILED;
 
@@ -1313,7 +1314,7 @@ zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags)
 	 */
 	ms.ms_mntstatus = 0;
 	zfs_foreach_mountpoint(zhp->zpool_hdl, cb.cb_handles, cb.cb_used,
-	    zfs_share_one, &ms, B_FALSE);
+	    zfs_share_one, &ms, 1);
 	if (ms.ms_mntstatus != 0)
 		ret = EZFS_SHAREFAILED;
 	else