Better control the thread pool size when mounting datasets

Ever since a10d50f999, ZFS has mounted file systems in parallel when
importing a pool.  It uses a fixed size of 512 for the thread pool.  But
since c183d164aa1, it has also imported pools in parallel.  So the total
number of threads at one time is 513 * npools + 1.  That can easily
exceed the system's limit on the number of threads per process, which
will cause one or more pools to be unable to allocate any worker
threads, forcing them to fallback to slow serial mounting .  To
forestall that, manage the threadpool size in /sbin/zpool, not libzfs.
Use the same size (512), but divided by the number of pools.

This is a backwards-incompatible change to the libzfs abi.

Sponsored by: Axcient
Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: George Wilson <[email protected]>
Signed-off-by: Alan Somers <[email protected]>
Closes #16178

3 files changed, 20 insertions, 8 deletions

diff --git a/cmd/zed/agents/zfs_mod.c b/cmd/zed/agents/zfs_mod.c
index 69163b80b..d0372608c 100644
--- a/cmd/zed/agents/zfs_mod.c
+++ b/cmd/zed/agents/zfs_mod.c
@@ -702,7 +702,7 @@ zfs_enable_ds(void *arg)
 {
 	unavailpool_t *pool = (unavailpool_t *)arg;
 
-	(void) zpool_enable_datasets(pool->uap_zhp, NULL, 0);
+	(void) zpool_enable_datasets(pool->uap_zhp, NULL, 0, 512);
 	zpool_close(pool->uap_zhp);
 	free(pool);
 }
diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c
index 0bbdd5b18..75c0e40b6 100644
--- a/cmd/zfs/zfs_main.c
+++ b/cmd/zfs/zfs_main.c
@@ -7192,6 +7192,8 @@ share_mount(int op, int argc, char **argv)
 	int c, ret = 0;
 	char *options = NULL;
 	int flags = 0;
+	const uint_t mount_nthr = 512;
+	uint_t nthr;
 
 	/* check options */
 	while ((c = getopt(argc, argv, op == OP_MOUNT ? ":aRlvo:Of" : "al"))
@@ -7310,9 +7312,9 @@ share_mount(int op, int argc, char **argv)
 		 * be serialized so that we can prompt the user for their keys
 		 * in a consistent manner.
 		 */
+		nthr = op == OP_MOUNT && !(flags & MS_CRYPT) ? mount_nthr : 1;
 		zfs_foreach_mountpoint(g_zfs, cb.cb_handles, cb.cb_used,
-		    share_mount_one_cb, &share_mount_state,
-		    op == OP_MOUNT && !(flags & MS_CRYPT));
+		    share_mount_one_cb, &share_mount_state, nthr);
 		zfs_commit_shares(NULL);
 
 		ret = share_mount_state.sm_status;
diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c
index 400f4bf1a..d47e1cda9 100644
--- a/cmd/zpool/zpool_main.c
+++ b/cmd/zpool/zpool_main.c
@@ -80,6 +80,8 @@
 
 libzfs_handle_t *g_zfs;
 
+static int mount_tp_nthr = 512;  /* tpool threads for multi-threaded mounting */
+
 static int zpool_do_create(int, char **);
 static int zpool_do_destroy(int, char **);
 
@@ -3418,7 +3420,7 @@ zfs_force_import_required(nvlist_t *config)
  */
 static int
 do_import(nvlist_t *config, const char *newname, const char *mntopts,
-    nvlist_t *props, int flags)
+    nvlist_t *props, int flags, uint_t mntthreads)
 {
 	int ret = 0;
 	int ms_status = 0;
@@ -3518,7 +3520,7 @@ do_import(nvlist_t *config, const char *newname, const char *mntopts,
 
 	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
 	    !(flags & ZFS_IMPORT_ONLY)) {
-		ms_status = zpool_enable_datasets(zhp, mntopts, 0);
+		ms_status = zpool_enable_datasets(zhp, mntopts, 0, mntthreads);
 		if (ms_status == EZFS_SHAREFAILED) {
 			(void) fprintf(stderr, gettext("Import was "
 			    "successful, but unable to share some datasets\n"));
@@ -3537,6 +3539,7 @@ typedef struct import_parameters {
 	const char *ip_mntopts;
 	nvlist_t *ip_props;
 	int ip_flags;
+	uint_t ip_mntthreads;
 	int *ip_err;
 } import_parameters_t;
 
@@ -3545,7 +3548,7 @@ do_import_task(void *arg)
 {
 	import_parameters_t *ip = arg;
 	*ip->ip_err |= do_import(ip->ip_config, NULL, ip->ip_mntopts,
-	    ip->ip_props, ip->ip_flags);
+	    ip->ip_props, ip->ip_flags, ip->ip_mntthreads);
 	free(ip);
 }
 
@@ -3559,6 +3562,7 @@ import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
 	uint64_t pool_state;
 	boolean_t pool_specified = (import->poolname != NULL ||
 	    import->guid != 0);
+	uint_t npools = 0;
 
 
 	tpool_t *tp = NULL;
@@ -3576,6 +3580,10 @@ import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
 	int err = 0;
 	nvpair_t *elem = NULL;
 	boolean_t first = B_TRUE;
+	if (!pool_specified && import->do_all) {
+		while ((elem = nvlist_next_nvpair(pools, elem)) != NULL)
+			npools++;
+	}
 	while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
 
 		verify(nvpair_value_nvlist(elem, &config) == 0);
@@ -3606,6 +3614,7 @@ import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
 				ip->ip_mntopts = mntopts;
 				ip->ip_props = props;
 				ip->ip_flags = flags;
+				ip->ip_mntthreads = mount_tp_nthr / npools;
 				ip->ip_err = &err;
 
 				(void) tpool_dispatch(tp, do_import_task,
@@ -3673,7 +3682,7 @@ import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
 			err = B_TRUE;
 		} else {
 			err |= do_import(found_config, new_name,
-			    mntopts, props, flags);
+			    mntopts, props, flags, mount_tp_nthr);
 		}
 	}
 
@@ -7217,7 +7226,8 @@ zpool_do_split(int argc, char **argv)
 	}
 
 	if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL) {
-		ms_status = zpool_enable_datasets(zhp, mntopts, 0);
+		ms_status = zpool_enable_datasets(zhp, mntopts, 0,
+		    mount_tp_nthr);
 		if (ms_status == EZFS_SHAREFAILED) {
 			(void) fprintf(stderr, gettext("Split was successful, "
 			    "datasets are mounted but sharing of some datasets "