ddt: add support for prefetching tables into the ARC

This change adds a new `zpool prefetch -t ddt $pool` command which
causes a pool's DDT to be loaded into the ARC. The primary goal is to
remove the need to "warm" a pool's cache before deduplication stops
slowing write performance. It may also provide a way to reload portions
of a DDT if they have been flushed due to inactivity.

Sponsored-by: iXsystems, Inc.
Sponsored-by: Catalogics, Inc.
Sponsored-by: Klara, Inc.
Reviewed-by: Alexander Motin <[email protected]>
Reviewed-by: Tony Hutter <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Allan Jude <[email protected]>
Signed-off-by: Will Andrews <[email protected]>
Signed-off-by: Fred Weigel <[email protected]>
Signed-off-by: Rob Norris <[email protected]>
Signed-off-by: Don Brady <[email protected]>
Co-authored-by: Will Andrews <[email protected]>
Co-authored-by: Don Brady <[email protected]>
Closes #15890 

5 files changed, 132 insertions, 8 deletions

diff --git a/lib/libzfs/libzfs.abi b/lib/libzfs/libzfs.abi
index aee6e59c6..4394c9420 100644
--- a/lib/libzfs/libzfs.abi
+++ b/lib/libzfs/libzfs.abi
@@ -457,6 +457,7 @@
     <elf-symbol name='zfs_wait_status' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_zpl_version_map' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_add' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zpool_add_propname' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_checkpoint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_clear' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_clear_label' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -520,6 +521,7 @@
     <elf-symbol name='zpool_open' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_open_canfail' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_pool_state_to_name' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='zpool_prefetch' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_prepare_and_label_disk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_prepare_disk' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zpool_print_unsup_feat' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -1642,6 +1644,9 @@
     <class-decl name='uu_avl' is-struct='yes' visibility='default' is-declaration-only='yes' id='4af029d1'/>
     <class-decl name='uu_avl_pool' is-struct='yes' visibility='default' is-declaration-only='yes' id='12a530a8'/>
     <class-decl name='uu_avl_walk' is-struct='yes' visibility='default' is-declaration-only='yes' id='e70a39e3'/>
+    <array-type-def dimensions='1' type-id='80f4b756' size-in-bits='256' id='71dc54ac'>
+      <subrange length='4' type-id='7359adad' id='16fe7105'/>
+    </array-type-def>
     <type-decl name='int' size-in-bits='32' id='95e97e5e'/>
     <type-decl name='long int' size-in-bits='64' id='bd54fe1a'/>
     <type-decl name='long long int' size-in-bits='64' id='1eb56b1e'/>
@@ -2096,7 +2101,7 @@
         <var-decl name='zfs_props_table' type-id='ae3e8ca6' visibility='default'/>
       </data-member>
     </class-decl>
-    <class-decl name='zpool_handle' size-in-bits='2560' is-struct='yes' visibility='default' id='67002a8a'>
+    <class-decl name='zpool_handle' size-in-bits='2816' is-struct='yes' visibility='default' id='67002a8a'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='zpool_hdl' type-id='b0382bb3' visibility='default'/>
       </data-member>
@@ -2109,19 +2114,25 @@
       <data-member access='public' layout-offset-in-bits='2176'>
         <var-decl name='zpool_state' type-id='95e97e5e' visibility='default'/>
       </data-member>
+      <data-member access='public' layout-offset-in-bits='2208'>
+        <var-decl name='zpool_n_propnames' type-id='f0981eeb' visibility='default'/>
+      </data-member>
       <data-member access='public' layout-offset-in-bits='2240'>
+        <var-decl name='zpool_propnames' type-id='71dc54ac' visibility='default'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='2496'>
         <var-decl name='zpool_config_size' type-id='b59d7dce' visibility='default'/>
       </data-member>
-      <data-member access='public' layout-offset-in-bits='2304'>
+      <data-member access='public' layout-offset-in-bits='2560'>
         <var-decl name='zpool_config' type-id='5ce45b60' visibility='default'/>
       </data-member>
-      <data-member access='public' layout-offset-in-bits='2368'>
+      <data-member access='public' layout-offset-in-bits='2624'>
         <var-decl name='zpool_old_config' type-id='5ce45b60' visibility='default'/>
       </data-member>
-      <data-member access='public' layout-offset-in-bits='2432'>
+      <data-member access='public' layout-offset-in-bits='2688'>
         <var-decl name='zpool_props' type-id='5ce45b60' visibility='default'/>
       </data-member>
-      <data-member access='public' layout-offset-in-bits='2496'>
+      <data-member access='public' layout-offset-in-bits='2752'>
         <var-decl name='zpool_start_block' type-id='804dc465' visibility='default'/>
       </data-member>
     </class-decl>
@@ -2923,7 +2934,8 @@
       <enumerator name='ZPOOL_PROP_BCLONERATIO' value='35'/>
       <enumerator name='ZPOOL_PROP_DEDUP_TABLE_SIZE' value='36'/>
       <enumerator name='ZPOOL_PROP_DEDUP_TABLE_QUOTA' value='37'/>
-      <enumerator name='ZPOOL_NUM_PROPS' value='38'/>
+      <enumerator name='ZPOOL_PROP_DEDUPCACHED' value='38'/>
+      <enumerator name='ZPOOL_NUM_PROPS' value='39'/>
     </enum-decl>
     <typedef-decl name='zpool_prop_t' type-id='af1ba157' id='5d0c23fb'/>
     <typedef-decl name='regoff_t' type-id='95e97e5e' id='54a2a2a8'/>
@@ -5896,6 +5908,7 @@
       <enumerator name='ZFS_IOC_VDEV_GET_PROPS' value='23125'/>
       <enumerator name='ZFS_IOC_VDEV_SET_PROPS' value='23126'/>
       <enumerator name='ZFS_IOC_POOL_SCRUB' value='23127'/>
+      <enumerator name='ZFS_IOC_POOL_PREFETCH' value='23128'/>
       <enumerator name='ZFS_IOC_PLATFORM' value='23168'/>
       <enumerator name='ZFS_IOC_EVENTS_NEXT' value='23169'/>
       <enumerator name='ZFS_IOC_EVENTS_CLEAR' value='23170'/>
@@ -5924,6 +5937,12 @@
       <enumerator name='ZPOOL_WAIT_NUM_ACTIVITIES' value='9'/>
     </enum-decl>
     <typedef-decl name='zpool_wait_activity_t' type-id='849338e3' id='73446457'/>
+    <enum-decl name='zpool_prefetch_type_t' naming-typedef-id='e55ff6bc' id='0299ab50'>
+      <underlying-type type-id='9cac1fee'/>
+      <enumerator name='ZPOOL_PREFETCH_NONE' value='0'/>
+      <enumerator name='ZPOOL_PREFETCH_DDT' value='1'/>
+    </enum-decl>
+    <typedef-decl name='zpool_prefetch_type_t' type-id='0299ab50' id='e55ff6bc'/>
     <enum-decl name='spa_feature' id='33ecb627'>
       <underlying-type type-id='9cac1fee'/>
       <enumerator name='SPA_FEATURE_NONE' value='-1'/>
@@ -5971,6 +5990,8 @@
       <enumerator name='SPA_FEATURES' value='41'/>
     </enum-decl>
     <typedef-decl name='spa_feature_t' type-id='33ecb627' id='d6618c78'/>
+    <qualified-type-def type-id='80f4b756' const='yes' id='b99c00c9'/>
+    <pointer-type-def type-id='b99c00c9' size-in-bits='64' id='13956559'/>
     <qualified-type-def type-id='22cce67b' const='yes' id='d2816df0'/>
     <pointer-type-def type-id='d2816df0' size-in-bits='64' id='3bbfee2e'/>
     <qualified-type-def type-id='b96825af' const='yes' id='2b61797f'/>
@@ -6063,6 +6084,11 @@
       <parameter type-id='37e3bd22'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='lzc_pool_prefetch' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='80f4b756'/>
+      <parameter type-id='e55ff6bc'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='lzc_set_bootenv' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='80f4b756'/>
       <parameter type-id='22cce67b'/>
@@ -6208,6 +6234,13 @@
       <parameter type-id='9da381c4'/>
       <return type-id='48b5725f'/>
     </function-decl>
+    <function-decl name='fnvlist_add_string_array' visibility='default' binding='global' size-in-bits='64'>
+      <parameter type-id='5ce45b60'/>
+      <parameter type-id='80f4b756'/>
+      <parameter type-id='13956559'/>
+      <parameter type-id='3502e3ff'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='fnvlist_add_nvlist_array' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='5ce45b60'/>
       <parameter type-id='80f4b756'/>
@@ -6396,6 +6429,11 @@
       <parameter type-id='4c81de99' name='zhp'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='zpool_prefetch' mangled-name='zpool_prefetch' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_prefetch'>
+      <parameter type-id='4c81de99' name='zhp'/>
+      <parameter type-id='e55ff6bc' name='type'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='zpool_add' mangled-name='zpool_add' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_add'>
       <parameter type-id='4c81de99' name='zhp'/>
       <parameter type-id='5ce45b60' name='nvroot'/>
@@ -6590,6 +6628,11 @@
       <parameter type-id='95e97e5e' name='name_flags'/>
       <return type-id='26a90f95'/>
     </function-decl>
+    <function-decl name='zpool_add_propname' mangled-name='zpool_add_propname' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_add_propname'>
+      <parameter type-id='4c81de99' name='zhp'/>
+      <parameter type-id='80f4b756' name='propname'/>
+      <return type-id='48b5725f'/>
+    </function-decl>
     <function-decl name='zpool_get_errlog' mangled-name='zpool_get_errlog' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_get_errlog'>
       <parameter type-id='4c81de99' name='zhp'/>
       <parameter type-id='857bb57e' name='nverrlistp'/>
@@ -8689,7 +8732,6 @@
     </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='lib/libzutil/zutil_device_path.c' language='LANG_C99'>
-    <pointer-type-def type-id='b99c00c9' size-in-bits='64' id='13956559'/>
     <function-decl name='zpool_default_search_paths' mangled-name='zpool_default_search_paths' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='zpool_default_search_paths'>
       <parameter type-id='78c01427'/>
       <return type-id='13956559'/>
@@ -9151,7 +9193,6 @@
     <array-type-def dimensions='1' type-id='b99c00c9' size-in-bits='2624' id='5ce15418'>
       <subrange length='41' type-id='7359adad' id='cb834f44'/>
     </array-type-def>
-    <qualified-type-def type-id='80f4b756' const='yes' id='b99c00c9'/>
     <pointer-type-def type-id='8f92235e' size-in-bits='64' id='90421557'/>
     <function-decl name='nvpair_value_uint32' visibility='default' binding='global' size-in-bits='64'>
       <parameter type-id='dace003f'/>
diff --git a/lib/libzfs/libzfs_impl.h b/lib/libzfs/libzfs_impl.h
index ef0359f45..e98ede51e 100644
--- a/lib/libzfs/libzfs_impl.h
+++ b/lib/libzfs/libzfs_impl.h
@@ -94,12 +94,15 @@ struct zfs_handle {
  * snapshots of volumes.
  */
 #define	ZFS_IS_VOLUME(zhp) ((zhp)->zfs_head_type == ZFS_TYPE_VOLUME)
+#define	ZHP_MAX_PROPNAMES 4
 
 struct zpool_handle {
 	libzfs_handle_t *zpool_hdl;
 	zpool_handle_t *zpool_next;
 	char zpool_name[ZFS_MAX_DATASET_NAME_LEN];
 	int zpool_state;
+	unsigned int zpool_n_propnames;
+	const char *zpool_propnames[ZHP_MAX_PROPNAMES];
 	size_t zpool_config_size;
 	nvlist_t *zpool_config;
 	nvlist_t *zpool_old_config;
diff --git a/lib/libzfs/libzfs_pool.c b/lib/libzfs/libzfs_pool.c
index 9896948a2..b787d91a2 100644
--- a/lib/libzfs/libzfs_pool.c
+++ b/lib/libzfs/libzfs_pool.c
@@ -79,6 +79,13 @@ zpool_get_all_props(zpool_handle_t *zhp)
 
 	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
 
+	if (zhp->zpool_n_propnames > 0) {
+		nvlist_t *innvl = fnvlist_alloc();
+		fnvlist_add_string_array(innvl, ZPOOL_GET_PROPS_NAMES,
+		    zhp->zpool_propnames, zhp->zpool_n_propnames);
+		zcmd_write_src_nvlist(hdl, &zc, innvl);
+	}
+
 	zcmd_alloc_dst_nvlist(hdl, &zc, 0);
 
 	while (zfs_ioctl(hdl, ZFS_IOC_POOL_GET_PROPS, &zc) != 0) {
@@ -318,6 +325,15 @@ zpool_get_prop(zpool_handle_t *zhp, zpool_prop_t prop, char *buf,
 		return (0);
 	}
 
+	/*
+	 * ZPOOL_PROP_DEDUPCACHED can be fetched by name only using
+	 * the ZPOOL_GET_PROPS_NAMES mechanism
+	 */
+	if (prop == ZPOOL_PROP_DEDUPCACHED) {
+		zpool_add_propname(zhp, ZPOOL_DEDUPCACHED_PROP_NAME);
+		(void) zpool_get_all_props(zhp);
+	}
+
 	if (zhp->zpool_props == NULL && zpool_get_all_props(zhp) &&
 	    prop != ZPOOL_PROP_NAME)
 		return (-1);
@@ -361,6 +377,7 @@ zpool_get_prop(zpool_handle_t *zhp, zpool_prop_t prop, char *buf,
 		case ZPOOL_PROP_BCLONESAVED:
 		case ZPOOL_PROP_BCLONEUSED:
 		case ZPOOL_PROP_DEDUP_TABLE_SIZE:
+		case ZPOOL_PROP_DEDUPCACHED:
 			if (literal)
 				(void) snprintf(buf, len, "%llu",
 				    (u_longlong_t)intval);
@@ -1739,6 +1756,28 @@ zpool_discard_checkpoint(zpool_handle_t *zhp)
 }
 
 /*
+ * Load data type for the given pool.
+ */
+int
+zpool_prefetch(zpool_handle_t *zhp, zpool_prefetch_type_t type)
+{
+	libzfs_handle_t *hdl = zhp->zpool_hdl;
+	char msg[1024];
+	int error;
+
+	error = lzc_pool_prefetch(zhp->zpool_name, type);
+	if (error != 0) {
+		(void) snprintf(msg, sizeof (msg), dgettext(TEXT_DOMAIN,
+		    "cannot prefetch %s in '%s'"),
+		    type == ZPOOL_PREFETCH_DDT ? "ddt" : "", zhp->zpool_name);
+		(void) zpool_standard_error(hdl, error, msg);
+		return (-1);
+	}
+
+	return (0);
+}
+
+/*
  * Add the given vdevs to the pool.  The caller must have already performed the
  * necessary verification to ensure that the vdev specification is well-formed.
  */
@@ -4401,6 +4440,14 @@ zbookmark_mem_compare(const void *a, const void *b)
 	return (memcmp(a, b, sizeof (zbookmark_phys_t)));
 }
 
+void
+zpool_add_propname(zpool_handle_t *zhp, const char *propname)
+{
+	assert(zhp->zpool_n_propnames < ZHP_MAX_PROPNAMES);
+	zhp->zpool_propnames[zhp->zpool_n_propnames] = propname;
+	zhp->zpool_n_propnames++;
+}
+
 /*
  * Retrieve the persistent error log, uniquify the members, and return to the
  * caller.
diff --git a/lib/libzfs_core/libzfs_core.abi b/lib/libzfs_core/libzfs_core.abi
index c20698580..02b2dc718 100644
--- a/lib/libzfs_core/libzfs_core.abi
+++ b/lib/libzfs_core/libzfs_core.abi
@@ -176,6 +176,7 @@
     <elf-symbol name='lzc_load_key' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='lzc_pool_checkpoint' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='lzc_pool_checkpoint_discard' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='lzc_pool_prefetch' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='lzc_promote' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='lzc_receive' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='lzc_receive_one' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -1428,6 +1429,7 @@
       <enumerator name='ZFS_IOC_VDEV_GET_PROPS' value='23125'/>
       <enumerator name='ZFS_IOC_VDEV_SET_PROPS' value='23126'/>
       <enumerator name='ZFS_IOC_POOL_SCRUB' value='23127'/>
+      <enumerator name='ZFS_IOC_POOL_PREFETCH' value='23128'/>
       <enumerator name='ZFS_IOC_PLATFORM' value='23168'/>
       <enumerator name='ZFS_IOC_EVENTS_NEXT' value='23169'/>
       <enumerator name='ZFS_IOC_EVENTS_CLEAR' value='23170'/>
@@ -1462,6 +1464,12 @@
       <enumerator name='ZFS_WAIT_NUM_ACTIVITIES' value='1'/>
     </enum-decl>
     <typedef-decl name='zfs_wait_activity_t' type-id='527d5dc6' id='3024501a'/>
+    <enum-decl name='zpool_prefetch_type_t' naming-typedef-id='e55ff6bc' id='0299ab50'>
+      <underlying-type type-id='9cac1fee'/>
+      <enumerator name='ZPOOL_PREFETCH_NONE' value='0'/>
+      <enumerator name='ZPOOL_PREFETCH_DDT' value='1'/>
+    </enum-decl>
+    <typedef-decl name='zpool_prefetch_type_t' type-id='0299ab50' id='e55ff6bc'/>
     <enum-decl name='data_type_t' naming-typedef-id='8d0687d2' id='aeeae136'>
       <underlying-type type-id='9cac1fee'/>
       <enumerator name='DATA_TYPE_DONTCARE' value='-1'/>
@@ -2892,6 +2900,11 @@
       <parameter type-id='80f4b756' name='pool'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-decl name='lzc_pool_prefetch' mangled-name='lzc_pool_prefetch' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzc_pool_prefetch'>
+      <parameter type-id='80f4b756' name='pool'/>
+      <parameter type-id='e55ff6bc' name='type'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
     <function-decl name='lzc_channel_program_nosync' mangled-name='lzc_channel_program_nosync' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='lzc_channel_program_nosync'>
       <parameter type-id='80f4b756' name='pool'/>
       <parameter type-id='80f4b756' name='program'/>
diff --git a/lib/libzfs_core/libzfs_core.c b/lib/libzfs_core/libzfs_core.c
index 070f8c1be..ec8b0ff4f 100644
--- a/lib/libzfs_core/libzfs_core.c
+++ b/lib/libzfs_core/libzfs_core.c
@@ -1630,6 +1630,26 @@ lzc_pool_checkpoint_discard(const char *pool)
 }
 
 /*
+ * Load the requested data type for the specified pool.
+ */
+int
+lzc_pool_prefetch(const char *pool, zpool_prefetch_type_t type)
+{
+	int error;
+	nvlist_t *result = NULL;
+	nvlist_t *args = fnvlist_alloc();
+
+	fnvlist_add_int32(args, ZPOOL_PREFETCH_TYPE, type);
+
+	error = lzc_ioctl(ZFS_IOC_POOL_PREFETCH, pool, args, &result);
+
+	fnvlist_free(args);
+	fnvlist_free(result);
+
+	return (error);
+}
+
+/*
  * Executes a read-only channel program.
  *
  * A read-only channel program works programmatically the same way as a