Vdev Properties Feature

Add properties, similar to pool properties, to each vdev.
This makes use of the existing per-vdev ZAP that was added as
part of device evacuation/removal.

A large number of read-only properties are exposed,
many of the members of struct vdev_t, that provide useful
statistics.

Adds support for read-only "removing" vdev property.
Adds the "allocating" property that defaults to "on" and
can be set to "off" to prevent future allocations from that
top-level vdev.

Supports user-defined vdev properties.
Includes support for properties.vdev in SYSFS.

Co-authored-by: Allan Jude <[email protected]>
Co-authored-by: Mark Maybee <[email protected]>
Reviewed-by: Matthew Ahrens <[email protected]>
Reviewed-by: Mark Maybee <[email protected]>
Signed-off-by: Allan Jude <[email protected]>
Closes #11711

9 files changed, 141 insertions, 5 deletions

diff --git a/include/libzfs.h b/include/libzfs.h
index c0883a983..53a778f7b 100644
--- a/include/libzfs.h
+++ b/include/libzfs.h
@@ -150,6 +150,7 @@ typedef enum zfs_error {
 	EZFS_NO_RESILVER_DEFER,	/* pool doesn't support resilver_defer */
 	EZFS_EXPORT_IN_PROGRESS,	/* currently exporting the pool */
 	EZFS_REBUILDING,	/* resilvering (sequential reconstrution) */
+	EZFS_VDEV_NOTSUP,	/* ops not supported for this type of vdev */
 	EZFS_UNKNOWN
 } zfs_error_t;
 
@@ -337,6 +338,24 @@ _LIBZFS_H const char *zpool_prop_to_name(zpool_prop_t);
 _LIBZFS_H const char *zpool_prop_values(zpool_prop_t);
 
 /*
+ * Functions to manage vdev properties
+ */
+_LIBZFS_H int zpool_get_vdev_prop_value(nvlist_t *, vdev_prop_t, char *, char *,
+    size_t, zprop_source_t *, boolean_t);
+_LIBZFS_H int zpool_get_vdev_prop(zpool_handle_t *, const char *, vdev_prop_t,
+    char *, char *, size_t, zprop_source_t *, boolean_t);
+_LIBZFS_H int zpool_get_all_vdev_props(zpool_handle_t *, const char *,
+    nvlist_t **);
+_LIBZFS_H int zpool_set_vdev_prop(zpool_handle_t *, const char *, const char *,
+    const char *);
+
+_LIBZFS_H const char *vdev_prop_to_name(vdev_prop_t);
+_LIBZFS_H const char *vdev_prop_values(vdev_prop_t);
+_LIBZFS_H boolean_t vdev_prop_user(const char *name);
+_LIBZFS_H const char *vdev_prop_column_name(vdev_prop_t);
+_LIBZFS_H boolean_t vdev_prop_align_right(vdev_prop_t);
+
+/*
  * Pool health statistics.
  */
 typedef enum {
@@ -552,6 +571,8 @@ typedef struct zprop_list {
 _LIBZFS_H int zfs_expand_proplist(zfs_handle_t *, zprop_list_t **, boolean_t,
     boolean_t);
 _LIBZFS_H void zfs_prune_proplist(zfs_handle_t *, uint8_t *);
+_LIBZFS_H int vdev_expand_proplist(zpool_handle_t *, const char *,
+    zprop_list_t **);
 
 #define	ZFS_MOUNTPOINT_NONE	"none"
 #define	ZFS_MOUNTPOINT_LEGACY	"legacy"
@@ -567,7 +588,7 @@ _LIBZFS_H void zfs_prune_proplist(zfs_handle_t *, uint8_t *);
  * zpool property management
  */
 _LIBZFS_H int zpool_expand_proplist(zpool_handle_t *, zprop_list_t **,
-    boolean_t);
+    zfs_type_t, boolean_t);
 _LIBZFS_H int zpool_prop_get_feature(zpool_handle_t *, const char *, char *,
     size_t);
 _LIBZFS_H const char *zpool_prop_default_string(zpool_prop_t);
@@ -598,6 +619,12 @@ typedef enum {
 /*
  * Functions for printing zfs or zpool properties
  */
+typedef struct vdev_cbdata {
+	int cb_name_flags;
+	char **cb_names;
+	unsigned int cb_names_count;
+} vdev_cbdata_t;
+
 typedef struct zprop_get_cbdata {
 	int cb_sources;
 	zfs_get_column_t cb_columns[ZFS_GET_NCOLS];
@@ -607,6 +634,7 @@ typedef struct zprop_get_cbdata {
 	boolean_t cb_first;
 	zprop_list_t *cb_proplist;
 	zfs_type_t cb_type;
+	vdev_cbdata_t cb_vdevs;
 } zprop_get_cbdata_t;
 
 _LIBZFS_H void zprop_print_one_property(const char *, zprop_get_cbdata_t *,
@@ -879,7 +907,7 @@ _LIBZFS_H void zfs_commit_shares(const char *);
 _LIBZFS_H int zfs_nicestrtonum(libzfs_handle_t *, const char *, uint64_t *);
 
 /*
- * Utility functions to run an external process.
+ * Utility functions to run an _LIBZFS_Hal process.
  */
 #define	STDOUT_VERBOSE	0x01
 #define	STDERR_VERBOSE	0x02
diff --git a/include/libzfs_core.h b/include/libzfs_core.h
index 9020d70db..7acc03fc7 100644
--- a/include/libzfs_core.h
+++ b/include/libzfs_core.h
@@ -146,6 +146,10 @@ _LIBZFS_CORE_H int lzc_wait_fs(const char *, zfs_wait_activity_t, boolean_t *);
 
 _LIBZFS_CORE_H int lzc_set_bootenv(const char *, const nvlist_t *);
 _LIBZFS_CORE_H int lzc_get_bootenv(const char *, nvlist_t **);
+
+_LIBZFS_CORE_H int lzc_get_vdev_prop(const char *, nvlist_t *, nvlist_t **);
+_LIBZFS_CORE_H int lzc_set_vdev_prop(const char *, nvlist_t *, nvlist_t **);
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h
index 2af11fc71..287b3beae 100644
--- a/include/sys/fs/zfs.h
+++ b/include/sys/fs/zfs.h
@@ -54,7 +54,8 @@ typedef enum {
 	ZFS_TYPE_SNAPSHOT	= (1 << 1),
 	ZFS_TYPE_VOLUME		= (1 << 2),
 	ZFS_TYPE_POOL		= (1 << 3),
-	ZFS_TYPE_BOOKMARK	= (1 << 4)
+	ZFS_TYPE_BOOKMARK	= (1 << 4),
+	ZFS_TYPE_VDEV		= (1 << 5),
 } zfs_type_t;
 
 /*
@@ -252,6 +253,7 @@ typedef enum {
 
 /* Small enough to not hog a whole line of printout in zpool(8). */
 #define	ZPROP_MAX_COMMENT	32
+#define	ZPROP_BOOLEAN_NA	2
 
 #define	ZPROP_VALUE		"value"
 #define	ZPROP_SOURCE		"source"
@@ -299,6 +301,59 @@ typedef int (*zprop_func)(int, void *);
 #define	ZFS_WRITTEN_PROP_PREFIX_LEN	8
 
 /*
+ * VDEV properties are identified by these constants and must be added to the
+ * end of this list to ensure that external consumers are not affected
+ * by the change. If you make any changes to this list, be sure to update
+ * the property table in usr/src/common/zfs/zpool_prop.c.
+ */
+typedef enum {
+	VDEV_PROP_INVAL = -1,
+#define	VDEV_PROP_USER	VDEV_PROP_INVAL
+	VDEV_PROP_NAME,
+	VDEV_PROP_CAPACITY,
+	VDEV_PROP_STATE,
+	VDEV_PROP_GUID,
+	VDEV_PROP_ASIZE,
+	VDEV_PROP_PSIZE,
+	VDEV_PROP_ASHIFT,
+	VDEV_PROP_SIZE,
+	VDEV_PROP_FREE,
+	VDEV_PROP_ALLOCATED,
+	VDEV_PROP_COMMENT,
+	VDEV_PROP_EXPANDSZ,
+	VDEV_PROP_FRAGMENTATION,
+	VDEV_PROP_BOOTSIZE,
+	VDEV_PROP_PARITY,
+	VDEV_PROP_PATH,
+	VDEV_PROP_DEVID,
+	VDEV_PROP_PHYS_PATH,
+	VDEV_PROP_ENC_PATH,
+	VDEV_PROP_FRU,
+	VDEV_PROP_PARENT,
+	VDEV_PROP_CHILDREN,
+	VDEV_PROP_NUMCHILDREN,
+	VDEV_PROP_READ_ERRORS,
+	VDEV_PROP_WRITE_ERRORS,
+	VDEV_PROP_CHECKSUM_ERRORS,
+	VDEV_PROP_INITIALIZE_ERRORS,
+	VDEV_PROP_OPS_NULL,
+	VDEV_PROP_OPS_READ,
+	VDEV_PROP_OPS_WRITE,
+	VDEV_PROP_OPS_FREE,
+	VDEV_PROP_OPS_CLAIM,
+	VDEV_PROP_OPS_TRIM,
+	VDEV_PROP_BYTES_NULL,
+	VDEV_PROP_BYTES_READ,
+	VDEV_PROP_BYTES_WRITE,
+	VDEV_PROP_BYTES_FREE,
+	VDEV_PROP_BYTES_CLAIM,
+	VDEV_PROP_BYTES_TRIM,
+	VDEV_PROP_REMOVING,
+	VDEV_PROP_ALLOCATING,
+	VDEV_NUM_PROPS
+} vdev_prop_t;
+
+/*
  * Dataset property functions shared between libzfs and kernel.
  */
 _SYS_FS_ZFS_H const char *zfs_prop_default_string(zfs_prop_t);
@@ -338,6 +393,22 @@ _SYS_FS_ZFS_H int zpool_prop_string_to_index(zpool_prop_t, const char *,
 _SYS_FS_ZFS_H uint64_t zpool_prop_random_value(zpool_prop_t, uint64_t seed);
 
 /*
+ * VDEV property functions shared between libzfs and kernel.
+ */
+_SYS_FS_ZFS_H vdev_prop_t vdev_name_to_prop(const char *);
+_SYS_FS_ZFS_H boolean_t vdev_prop_user(const char *name);
+_SYS_FS_ZFS_H const char *vdev_prop_to_name(vdev_prop_t);
+_SYS_FS_ZFS_H const char *vdev_prop_default_string(vdev_prop_t);
+_SYS_FS_ZFS_H uint64_t vdev_prop_default_numeric(vdev_prop_t);
+_SYS_FS_ZFS_H boolean_t vdev_prop_readonly(vdev_prop_t prop);
+_SYS_FS_ZFS_H int vdev_prop_index_to_string(vdev_prop_t, uint64_t,
+    const char **);
+_SYS_FS_ZFS_H int vdev_prop_string_to_index(vdev_prop_t, const char *,
+    uint64_t *);
+_SYS_FS_ZFS_H boolean_t zpool_prop_vdev(const char *name);
+_SYS_FS_ZFS_H uint64_t vdev_prop_random_value(vdev_prop_t prop, uint64_t seed);
+
+/*
  * Definitions for the Delegation.
  */
 typedef enum {
@@ -712,6 +783,7 @@ typedef struct zpool_load_policy {
 #define	ZPOOL_CONFIG_ORIG_GUID		"orig_guid"
 #define	ZPOOL_CONFIG_SPLIT_GUID		"split_guid"
 #define	ZPOOL_CONFIG_SPLIT_LIST		"guid_list"
+#define	ZPOOL_CONFIG_NONALLOCATING	"non_allocating"
 #define	ZPOOL_CONFIG_REMOVING		"removing"
 #define	ZPOOL_CONFIG_RESILVER_TXG	"resilver_txg"
 #define	ZPOOL_CONFIG_REBUILD_TXG	"rebuild_txg"
@@ -1109,6 +1181,7 @@ typedef struct vdev_stat {
 	uint64_t	vs_configured_ashift;   /* TLV vdev_ashift */
 	uint64_t	vs_logical_ashift;	/* vdev_logical_ashift  */
 	uint64_t	vs_physical_ashift;	/* vdev_physical_ashift */
+	uint64_t	vs_noalloc;		/* allocations halted?	*/
 } vdev_stat_t;
 
 /* BEGIN CSTYLED */
@@ -1362,6 +1435,8 @@ typedef enum zfs_ioc {
 	ZFS_IOC_GET_BOOKMARK_PROPS,		/* 0x5a52 */
 	ZFS_IOC_WAIT,				/* 0x5a53 */
 	ZFS_IOC_WAIT_FS,			/* 0x5a54 */
+	ZFS_IOC_VDEV_GET_PROPS,			/* 0x5a55 */
+	ZFS_IOC_VDEV_SET_PROPS,			/* 0x5a56 */
 
 	/*
 	 * Per-platform (Optional) - 8/128 numbers reserved.
@@ -1417,6 +1492,7 @@ typedef enum {
 	ZFS_ERR_RESILVER_IN_PROGRESS,
 	ZFS_ERR_REBUILD_IN_PROGRESS,
 	ZFS_ERR_BADPROP,
+	ZFS_ERR_VDEV_NOTSUP,
 } zfs_errno_t;
 
 /*
@@ -1509,6 +1585,18 @@ typedef enum {
 #define	ZPOOL_WAIT_WAITED		"wait_waited"
 
 /*
+ * The following are names used when invoking ZFS_IOC_VDEV_GET_PROP.
+ */
+#define	ZPOOL_VDEV_PROPS_GET_VDEV	"vdevprops_get_vdev"
+#define	ZPOOL_VDEV_PROPS_GET_PROPS	"vdevprops_get_props"
+
+/*
+ * The following are names used when invoking ZFS_IOC_VDEV_SET_PROP.
+ */
+#define	ZPOOL_VDEV_PROPS_SET_VDEV	"vdevprops_set_vdev"
+#define	ZPOOL_VDEV_PROPS_SET_PROPS	"vdevprops_set_props"
+
+/*
  * The following are names used when invoking ZFS_IOC_WAIT_FS.
  */
 #define	ZFS_WAIT_ACTIVITY		"wait_activity"
diff --git a/include/sys/spa.h b/include/sys/spa.h
index a55dbd66d..2e365eabe 100644
--- a/include/sys/spa.h
+++ b/include/sys/spa.h
@@ -792,7 +792,8 @@ extern int spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot,
     int replacing, int rebuild);
 extern int spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid,
     int replace_done);
-extern int spa_vdev_remove(spa_t *spa, uint64_t guid, boolean_t unspare);
+extern int spa_vdev_alloc(spa_t *spa, uint64_t guid);
+extern int spa_vdev_noalloc(spa_t *spa, uint64_t guid);
 extern boolean_t spa_vdev_remove_active(spa_t *spa);
 extern int spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type,
     nvlist_t *vdev_errlist);
diff --git a/include/sys/spa_impl.h b/include/sys/spa_impl.h
index 9714bbce9..eee4783fe 100644
--- a/include/sys/spa_impl.h
+++ b/include/sys/spa_impl.h
@@ -308,6 +308,7 @@ struct spa {
 	uint64_t	spa_missing_tvds;	/* unopenable tvds on load */
 	uint64_t	spa_missing_tvds_allowed; /* allow loading spa? */
 
+	uint64_t	spa_nonallocating_dspace;
 	spa_removing_phys_t spa_removing_phys;
 	spa_vdev_removal_t *spa_vdev_removal;
 
diff --git a/include/sys/vdev.h b/include/sys/vdev.h
index 0a81713a4..4e507d081 100644
--- a/include/sys/vdev.h
+++ b/include/sys/vdev.h
@@ -219,6 +219,9 @@ typedef enum {
 
 extern int vdev_label_init(vdev_t *vd, uint64_t txg, vdev_labeltype_t reason);
 
+extern int vdev_prop_set(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl);
+extern int vdev_prop_get(vdev_t *vd, nvlist_t *nvprops, nvlist_t *outnvl);
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/include/sys/vdev_impl.h b/include/sys/vdev_impl.h
index 3cfde40a7..86959725a 100644
--- a/include/sys/vdev_impl.h
+++ b/include/sys/vdev_impl.h
@@ -295,6 +295,7 @@ struct vdev {
 	list_node_t	vdev_state_dirty_node; /* state dirty list	*/
 	uint64_t	vdev_deflate_ratio; /* deflation ratio (x512)	*/
 	uint64_t	vdev_islog;	/* is an intent log device	*/
+	uint64_t	vdev_noalloc;	/* device is passivated?	*/
 	uint64_t	vdev_removing;	/* device is being removed?	*/
 	boolean_t	vdev_ishole;	/* is a hole in the namespace	*/
 	uint64_t	vdev_top_zap;
diff --git a/include/sys/zfs_sysfs.h b/include/sys/zfs_sysfs.h
index 14ba61fc4..912ef234f 100644
--- a/include/sys/zfs_sysfs.h
+++ b/include/sys/zfs_sysfs.h
@@ -39,6 +39,7 @@ _SYS_ZFS_SYSFS_H boolean_t zfs_mod_supported(const char *, const char *);
 #endif
 
 #define	ZFS_SYSFS_POOL_PROPERTIES	"properties.pool"
+#define	ZFS_SYSFS_VDEV_PROPERTIES	"properties.vdev"
 #define	ZFS_SYSFS_DATASET_PROPERTIES	"properties.dataset"
 #define	ZFS_SYSFS_KERNEL_FEATURES	"features.kernel"
 #define	ZFS_SYSFS_POOL_FEATURES		"features.pool"
diff --git a/include/zfs_prop.h b/include/zfs_prop.h
index 91b5032e7..8014c757a 100644
--- a/include/zfs_prop.h
+++ b/include/zfs_prop.h
@@ -100,6 +100,13 @@ _ZFS_PROP_H zprop_type_t zpool_prop_get_type(zpool_prop_t);
 _ZFS_PROP_H zprop_desc_t *zpool_prop_get_table(void);
 
 /*
+ * vdev property functions
+ */
+_ZFS_PROP_H void vdev_prop_init(void);
+_ZFS_PROP_H zprop_type_t vdev_prop_get_type(vdev_prop_t prop);
+_ZFS_PROP_H zprop_desc_t *vdev_prop_get_table(void);
+
+/*
  * Common routines to initialize property tables
  */
 _ZFS_PROP_H void zprop_register_impl(int, const char *, zprop_type_t, uint64_t,
@@ -122,11 +129,13 @@ _ZFS_PROP_H int zprop_iter_common(zprop_func, void *, boolean_t, boolean_t,
 _ZFS_PROP_H int zprop_name_to_prop(const char *, zfs_type_t);
 _ZFS_PROP_H int zprop_string_to_index(int, const char *, uint64_t *,
     zfs_type_t);
-_ZFS_PROP_H int zprop_index_to_string(int, uint64_t, const char **, zfs_type_t);
+_ZFS_PROP_H int zprop_index_to_string(int, uint64_t, const char **,
+    zfs_type_t);
 _ZFS_PROP_H uint64_t zprop_random_value(int, uint64_t, zfs_type_t);
 _ZFS_PROP_H const char *zprop_values(int, zfs_type_t);
 _ZFS_PROP_H size_t zprop_width(int, boolean_t *, zfs_type_t);
 _ZFS_PROP_H boolean_t zprop_valid_for_type(int, zfs_type_t, boolean_t);
+_ZFS_PROP_H int zprop_valid_char(char c);
 
 #ifdef	__cplusplus
 }