Add port of FreeBSD 'volmode' property

The volmode property may be set to control the visibility of ZVOL
block devices.

This allow switching ZVOL between three modes:
   full - existing fully functional behaviour (default)
   dev  - hide partitions on ZVOL block devices
   none - not exposing volumes outside ZFS

Additionally the new zvol_volmode module parameter can be used to
control the default behaviour.

This functionality can be used, for instance, on "backup" pools to
avoid cluttering /dev with unneeded zd* devices.

Original-patch-by: mav <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Ported-by: loli10K <[email protected]>
Signed-off-by: loli10K <[email protected]>

FreeBSD-commit: https://github.com/freebsd/freebsd/commit/dd28e6bb
Closes #1796 
Closes #3438 
Closes #6233 

14 files changed, 634 insertions, 83 deletions

diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h
index 0ce35468f..b634635c8 100644
--- a/include/sys/fs/zfs.h
+++ b/include/sys/fs/zfs.h
@@ -154,6 +154,7 @@ typedef enum {
 	ZFS_PROP_LOGICALUSED,
 	ZFS_PROP_LOGICALREFERENCED,
 	ZFS_PROP_INCONSISTENT,		/* not exposed to the user */
+	ZFS_PROP_VOLMODE,
 	ZFS_PROP_FILESYSTEM_LIMIT,
 	ZFS_PROP_SNAPSHOT_LIMIT,
 	ZFS_PROP_FILESYSTEM_COUNT,
@@ -394,6 +395,13 @@ typedef enum {
 	ZFS_REDUNDANT_METADATA_MOST
 } zfs_redundant_metadata_type_t;
 
+typedef enum {
+	ZFS_VOLMODE_DEFAULT = 0,
+	ZFS_VOLMODE_GEOM = 1,
+	ZFS_VOLMODE_DEV = 2,
+	ZFS_VOLMODE_NONE = 3
+} zfs_volmode_t;
+
 /*
  * On-disk version number.
  */
diff --git a/include/sys/zvol.h b/include/sys/zvol.h
index f149da977..e8b084762 100644
--- a/include/sys/zvol.h
+++ b/include/sys/zvol.h
@@ -51,6 +51,7 @@ extern void zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx);
 extern int zvol_set_volsize(const char *, uint64_t);
 extern int zvol_set_volblocksize(const char *, uint64_t);
 extern int zvol_set_snapdev(const char *, zprop_source_t, uint64_t);
+extern int zvol_set_volmode(const char *, zprop_source_t, uint64_t);
 extern zvol_state_t *zvol_suspend(const char *);
 extern int zvol_resume(zvol_state_t *);
 extern void *zvol_tag(zvol_state_t *);
diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5
index 0d2745aec..ab1c15841 100644
--- a/man/man5/zfs-module-parameters.5
+++ b/man/man5/zfs-module-parameters.5
@@ -2079,6 +2079,18 @@ Default value: \fB32\fR.
 .sp
 .ne 2
 .na
+\fBzvol_volmode\fR (uint)
+.ad
+.RS 12n
+Defines zvol block devices behaviour when \fBvolmode\fR is set to \fBdefault\fR.
+Valid values are \fB1\fR (full), \fB2\fR (dev) and \fB3\fR (none).
+.sp
+Default value: \fB1\fR.
+.RE
+
+.sp
+.ne 2
+.na
 \fBzfs_qat_disable\fR (int)
 .ad
 .RS 12n
diff --git a/man/man8/zfs.8 b/man/man8/zfs.8
index 439c21ac4..44180d603 100644
--- a/man/man8/zfs.8
+++ b/man/man8/zfs.8
@@ -1757,6 +1757,35 @@ when the pool is low on space.
 For a sparse volume, changes to
 .Sy volsize
 are not reflected in the reservation.
+.It Sy volmode Ns = Ns Cm default | full | geom | dev | none
+This property specifies how volumes should be exposed to the OS.
+Setting it to
+.Sy full
+exposes volumes as fully fledged block devices, providing maximal
+functionality. The value
+.Sy geom
+is just an alias for
+.Sy full
+and is kept for compatibility.
+Setting it to
+.Sy dev
+hides its partitions.
+Volumes with property set to
+.Sy none
+are not exposed outside ZFS, but can be snapshoted, cloned, replicated, etc,
+that can be suitable for backup purposes.
+Value
+.Sy default
+means that volumes exposition is controlled by system-wide tunable
+.Va zvol_volmode ,
+where
+.Sy full ,
+.Sy dev
+and
+.Sy none
+are encoded as 1, 2 and 3 respectively.
+The default values is
+.Sy full .
 .It Sy vscan Ns = Ns Sy on Ns | Ns Sy off
 Controls whether regular files should be scanned for viruses when a file is
 opened and closed.
diff --git a/module/zcommon/zfs_prop.c b/module/zcommon/zfs_prop.c
index 97736ee27..93c89e4aa 100644
--- a/module/zcommon/zfs_prop.c
+++ b/module/zcommon/zfs_prop.c
@@ -245,6 +245,15 @@ zfs_prop_init(void)
 		{ NULL }
 	};
 
+	static zprop_index_t volmode_table[] = {
+		{ "default",	ZFS_VOLMODE_DEFAULT },
+		{ "full",	ZFS_VOLMODE_GEOM },
+		{ "geom",	ZFS_VOLMODE_GEOM },
+		{ "dev",	ZFS_VOLMODE_DEV },
+		{ "none",	ZFS_VOLMODE_NONE },
+		{ NULL }
+	};
+
 	/* inherit index properties */
 	zprop_register_index(ZFS_PROP_REDUNDANT_METADATA, "redundant_metadata",
 	    ZFS_REDUNDANT_METADATA_ALL,
@@ -302,6 +311,10 @@ zfs_prop_init(void)
 	zprop_register_index(ZFS_PROP_DNODESIZE, "dnodesize",
 	    ZFS_DNSIZE_LEGACY, PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
 	    "legacy | auto | 1k | 2k | 4k | 8k | 16k", "DNSIZE", dnsize_table);
+	zprop_register_index(ZFS_PROP_VOLMODE, "volmode",
+	    ZFS_VOLMODE_DEFAULT, PROP_INHERIT,
+	    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
+	    "default | full | geom | dev | none", "VOLMODE", volmode_table);
 
 	/* inherit index (boolean) properties */
 	zprop_register_index(ZFS_PROP_ATIME, "atime", 1, PROP_INHERIT,
diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c
index d56049931..728b02377 100644
--- a/module/zfs/zfs_ioctl.c
+++ b/module/zfs/zfs_ioctl.c
@@ -2440,6 +2440,9 @@ zfs_prop_set_special(const char *dsname, zprop_source_t source,
 	case ZFS_PROP_SNAPDEV:
 		err = zvol_set_snapdev(dsname, source, intval);
 		break;
+	case ZFS_PROP_VOLMODE:
+		err = zvol_set_volmode(dsname, source, intval);
+		break;
 	case ZFS_PROP_VERSION:
 	{
 		zfsvfs_t *zfsvfs;
diff --git a/module/zfs/zvol.c b/module/zfs/zvol.c
index 4f1601ec6..623fb9b22 100644
--- a/module/zfs/zvol.c
+++ b/module/zfs/zvol.c
@@ -96,6 +96,7 @@ unsigned int zvol_threads = 32;
 unsigned int zvol_request_sync = 0;
 unsigned int zvol_prefetch_bytes = (128 * 1024);
 unsigned long zvol_max_discard_blocks = 16384;
+unsigned int zvol_volmode = ZFS_VOLMODE_GEOM;
 
 static taskq_t *zvol_taskq;
 static kmutex_t zvol_state_lock;
@@ -137,6 +138,7 @@ typedef enum {
 	ZVOL_ASYNC_REMOVE_MINORS,
 	ZVOL_ASYNC_RENAME_MINORS,
 	ZVOL_ASYNC_SET_SNAPDEV,
+	ZVOL_ASYNC_SET_VOLMODE,
 	ZVOL_ASYNC_MAX
 } zvol_async_op_t;
 
@@ -146,7 +148,7 @@ typedef struct {
 	char name1[MAXNAMELEN];
 	char name2[MAXNAMELEN];
 	zprop_source_t source;
-	uint64_t snapdev;
+	uint64_t value;
 } zvol_task_t;
 
 #define	ZVOL_RDONLY	0x1
@@ -1593,6 +1595,13 @@ static zvol_state_t *
 zvol_alloc(dev_t dev, const char *name)
 {
 	zvol_state_t *zv;
+	uint64_t volmode;
+
+	if (dsl_prop_get_integer(name, "volmode", &volmode, NULL) != 0)
+		return (NULL);
+
+	if (volmode == ZFS_VOLMODE_DEFAULT)
+		volmode = zvol_volmode;
 
 	zv = kmem_zalloc(sizeof (zvol_state_t), KM_SLEEP);
 
@@ -1626,6 +1635,22 @@ zvol_alloc(dev_t dev, const char *name)
 	rw_init(&zv->zv_suspend_lock, NULL, RW_DEFAULT, NULL);
 
 	zv->zv_disk->major = zvol_major;
+	if (volmode == ZFS_VOLMODE_DEV) {
+		/*
+		 * ZFS_VOLMODE_DEV disable partitioning on ZVOL devices: set
+		 * gendisk->minors = 1 as noted in include/linux/genhd.h.
+		 * Also disable extended partition numbers (GENHD_FL_EXT_DEVT)
+		 * and suppresses partition scanning (GENHD_FL_NO_PART_SCAN)
+		 * setting gendisk->flags accordingly.
+		 */
+		zv->zv_disk->minors = 1;
+#if defined(GENHD_FL_EXT_DEVT)
+		zv->zv_disk->flags &= ~GENHD_FL_EXT_DEVT;
+#endif
+#if defined(GENHD_FL_NO_PART_SCAN)
+		zv->zv_disk->flags |= GENHD_FL_NO_PART_SCAN;
+#endif
+	}
 	zv->zv_disk->first_minor = (dev & MINORMASK);
 	zv->zv_disk->fops = &zvol_ops;
 	zv->zv_disk->private_data = zv;
@@ -1692,6 +1717,9 @@ zvol_create_minor_impl(const char *name)
 	int idx;
 	uint64_t hash = zvol_name_hash(name);
 
+	if (zvol_inhibit_dev)
+		return (0);
+
 	idx = ida_simple_get(&zvol_ida, 0, 0, kmem_flags_convert(KM_SLEEP));
 	if (idx < 0)
 		return (SET_ERROR(-idx));
@@ -2095,7 +2123,7 @@ zvol_remove_minors_impl(const char *name)
 		taskq_wait_outstanding(system_taskq, tid);
 }
 
-/* Remove minor for this specific snapshot only */
+/* Remove minor for this specific volume only */
 static void
 zvol_remove_minor_impl(const char *name)
 {
@@ -2104,9 +2132,6 @@ zvol_remove_minor_impl(const char *name)
 	if (zvol_inhibit_dev)
 		return;
 
-	if (strchr(name, '@') == NULL)
-		return;
-
 	mutex_enter(&zvol_state_lock);
 
 	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
@@ -2227,9 +2252,50 @@ zvol_set_snapdev_impl(char *name, uint64_t snapdev)
 	spl_fstrans_unmark(cookie);
 }
 
+typedef struct zvol_volmode_cb_arg {
+	uint64_t volmode;
+} zvol_volmode_cb_arg_t;
+
+static void
+zvol_set_volmode_impl(char *name, uint64_t volmode)
+{
+	fstrans_cookie_t cookie = spl_fstrans_mark();
+
+	if (strchr(name, '@') != NULL)
+		return;
+
+	/*
+	 * It's unfortunate we need to remove minors before we create new ones:
+	 * this is necessary because our backing gendisk (zvol_state->zv_disk)
+	 * coule be different when we set, for instance, volmode from "geom"
+	 * to "dev" (or vice versa).
+	 * A possible optimization is to modify our consumers so we don't get
+	 * called when "volmode" does not change.
+	 */
+	switch (volmode) {
+		case ZFS_VOLMODE_NONE:
+			(void) zvol_remove_minor_impl(name);
+			break;
+		case ZFS_VOLMODE_GEOM:
+		case ZFS_VOLMODE_DEV:
+			(void) zvol_remove_minor_impl(name);
+			(void) zvol_create_minor_impl(name);
+			break;
+		case ZFS_VOLMODE_DEFAULT:
+			(void) zvol_remove_minor_impl(name);
+			if (zvol_volmode == ZFS_VOLMODE_NONE)
+				break;
+			else /* if zvol_volmode is invalid defaults to "geom" */
+				(void) zvol_create_minor_impl(name);
+			break;
+	}
+
+	spl_fstrans_unmark(cookie);
+}
+
 static zvol_task_t *
 zvol_task_alloc(zvol_async_op_t op, const char *name1, const char *name2,
-    uint64_t snapdev)
+    uint64_t value)
 {
 	zvol_task_t *task;
 	char *delim;
@@ -2240,7 +2306,7 @@ zvol_task_alloc(zvol_async_op_t op, const char *name1, const char *name2,
 
 	task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP);
 	task->op = op;
-	task->snapdev = snapdev;
+	task->value = value;
 	delim = strchr(name1, '/');
 	strlcpy(task->pool, name1, delim ? (delim - name1 + 1) : MAXNAMELEN);
 
@@ -2276,7 +2342,10 @@ zvol_task_cb(void *param)
 		zvol_rename_minors_impl(task->name1, task->name2);
 		break;
 	case ZVOL_ASYNC_SET_SNAPDEV:
-		zvol_set_snapdev_impl(task->name1, task->snapdev);
+		zvol_set_snapdev_impl(task->name1, task->value);
+		break;
+	case ZVOL_ASYNC_SET_VOLMODE:
+		zvol_set_volmode_impl(task->name1, task->value);
 		break;
 	default:
 		VERIFY(0);
@@ -2286,12 +2355,12 @@ zvol_task_cb(void *param)
 	zvol_task_free(task);
 }
 
-typedef struct zvol_set_snapdev_arg {
+typedef struct zvol_set_prop_int_arg {
 	const char *zsda_name;
 	uint64_t zsda_value;
 	zprop_source_t zsda_source;
 	dmu_tx_t *zsda_tx;
-} zvol_set_snapdev_arg_t;
+} zvol_set_prop_int_arg_t;
 
 /*
  * Sanity check the dataset for safe use by the sync task.  No additional
@@ -2300,7 +2369,7 @@ typedef struct zvol_set_snapdev_arg {
 static int
 zvol_set_snapdev_check(void *arg, dmu_tx_t *tx)
 {
-	zvol_set_snapdev_arg_t *zsda = arg;
+	zvol_set_prop_int_arg_t *zsda = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dir_t *dd;
 	int error;
@@ -2344,7 +2413,7 @@ zvol_set_snapdev_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
 static void
 zvol_set_snapdev_sync(void *arg, dmu_tx_t *tx)
 {
-	zvol_set_snapdev_arg_t *zsda = arg;
+	zvol_set_prop_int_arg_t *zsda = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dir_t *dd;
 	dsl_dataset_t *ds;
@@ -2369,7 +2438,7 @@ zvol_set_snapdev_sync(void *arg, dmu_tx_t *tx)
 int
 zvol_set_snapdev(const char *ddname, zprop_source_t source, uint64_t snapdev)
 {
-	zvol_set_snapdev_arg_t zsda;
+	zvol_set_prop_int_arg_t zsda;
 
 	zsda.zsda_name = ddname;
 	zsda.zsda_source = source;
@@ -2379,6 +2448,93 @@ zvol_set_snapdev(const char *ddname, zprop_source_t source, uint64_t snapdev)
 	    zvol_set_snapdev_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE));
 }
 
+/*
+ * Sanity check the dataset for safe use by the sync task.  No additional
+ * conditions are imposed.
+ */
+static int
+zvol_set_volmode_check(void *arg, dmu_tx_t *tx)
+{
+	zvol_set_prop_int_arg_t *zsda = arg;
+	dsl_pool_t *dp = dmu_tx_pool(tx);
+	dsl_dir_t *dd;
+	int error;
+
+	error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL);
+	if (error != 0)
+		return (error);
+
+	dsl_dir_rele(dd, FTAG);
+
+	return (error);
+}
+
+/* ARGSUSED */
+static int
+zvol_set_volmode_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
+{
+	char dsname[MAXNAMELEN];
+	zvol_task_t *task;
+	uint64_t volmode;
+
+	dsl_dataset_name(ds, dsname);
+	if (dsl_prop_get_int_ds(ds, "volmode", &volmode) != 0)
+		return (0);
+	task = zvol_task_alloc(ZVOL_ASYNC_SET_VOLMODE, dsname, NULL, volmode);
+	if (task == NULL)
+		return (0);
+
+	(void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb,
+	    task, TQ_SLEEP);
+	return (0);
+}
+
+/*
+ * Traverse all child datasets and apply volmode appropriately.
+ * We call dsl_prop_set_sync_impl() here to set the value only on the toplevel
+ * dataset and read the effective "volmode" on every child in the callback
+ * function: this is because the value is not guaranteed to be the same in the
+ * whole dataset hierarchy.
+ */
+static void
+zvol_set_volmode_sync(void *arg, dmu_tx_t *tx)
+{
+	zvol_set_prop_int_arg_t *zsda = arg;
+	dsl_pool_t *dp = dmu_tx_pool(tx);
+	dsl_dir_t *dd;
+	dsl_dataset_t *ds;
+	int error;
+
+	VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL));
+	zsda->zsda_tx = tx;
+
+	error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds);
+	if (error == 0) {
+		dsl_prop_set_sync_impl(ds, zfs_prop_to_name(ZFS_PROP_VOLMODE),
+		    zsda->zsda_source, sizeof (zsda->zsda_value), 1,
+		    &zsda->zsda_value, zsda->zsda_tx);
+		dsl_dataset_rele(ds, FTAG);
+	}
+
+	dmu_objset_find_dp(dp, dd->dd_object, zvol_set_volmode_sync_cb,
+	    zsda, DS_FIND_CHILDREN);
+
+	dsl_dir_rele(dd, FTAG);
+}
+
+int
+zvol_set_volmode(const char *ddname, zprop_source_t source, uint64_t volmode)
+{
+	zvol_set_prop_int_arg_t zsda;
+
+	zsda.zsda_name = ddname;
+	zsda.zsda_source = source;
+	zsda.zsda_value = volmode;
+
+	return (dsl_sync_task(ddname, zvol_set_volmode_check,
+	    zvol_set_volmode_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE));
+}
+
 void
 zvol_create_minors(spa_t *spa, const char *name, boolean_t async)
 {
@@ -2510,4 +2666,7 @@ MODULE_PARM_DESC(zvol_max_discard_blocks, "Max number of blocks to discard");
 
 module_param(zvol_prefetch_bytes, uint, 0644);
 MODULE_PARM_DESC(zvol_prefetch_bytes, "Prefetch N bytes at zvol start+end");
+
+module_param(zvol_volmode, uint, 0644);
+MODULE_PARM_DESC(zvol_volmode, "Default volmode property value");
 /* END CSTYLED */
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index c7eb9cf81..9f195628c 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -569,7 +569,7 @@ tests = ['zvol_cli_001_pos', 'zvol_cli_002_pos', 'zvol_cli_003_neg']
 [tests/functional/zvol/zvol_misc]
 tests = ['zvol_misc_001_neg', 'zvol_misc_002_pos', 'zvol_misc_003_neg',
     'zvol_misc_004_pos', 'zvol_misc_005_neg', 'zvol_misc_006_pos',
-    'zvol_misc_snapdev']
+    'zvol_misc_snapdev', 'zvol_misc_volmode']
 
 [tests/functional/zvol/zvol_swap]
 tests = ['zvol_swap_001_pos', 'zvol_swap_002_pos', 'zvol_swap_003_pos',
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index 034b36691..ddfe550bf 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -748,7 +748,7 @@ function zero_partitions #<whole_disk_name>
 	else
 		for i in 0 1 3 4 5 6 7
 		do
-			set_partition $i "" 0mb $diskname
+			log_must set_partition $i "" 0mb $diskname
 		done
 	fi
 
@@ -788,7 +788,11 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
 		parted $DEV_DSKDIR/$disk -s -- print 1 >/dev/null
 		typeset ret_val=$?
 		if [[ $slicenum -eq 0 || $ret_val -ne 0 ]]; then
-			log_must parted $DEV_DSKDIR/$disk -s -- mklabel gpt
+			parted $DEV_DSKDIR/$disk -s -- mklabel gpt
+			if [[ $? -ne 0 ]]; then
+				log_note "Failed to create GPT partition table on $disk"
+				return 1
+			fi
 		fi
 
 		# When no start is given align on the first cylinder.
@@ -804,8 +808,12 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
 			awk -F '[:k.]' '{print $4}')
 		((end = (size_mb * 1024 / cly_size_kb) + start))
 
-		log_must parted $DEV_DSKDIR/$disk -s -- \
+		parted $DEV_DSKDIR/$disk -s -- \
 		    mkpart part$slicenum ${start}cyl ${end}cyl
+		if [[ $? -ne 0 ]]; then
+			log_note "Failed to create partition $slicenum on $disk"
+			return 1
+		fi
 
 		blockdev --rereadpt $DEV_DSKDIR/$disk 2>/dev/null
 		block_device_wait
@@ -828,8 +836,10 @@ function set_partition #<slice_num> <slice_start> <size_plus_units>  <whole_disk
 
 	typeset ret_val=$?
 	rm -f $format_file
-	[[ $ret_val -ne 0 ]] && \
-	    log_fail "Unable to format $disk slice $slicenum to $size"
+	if [[ $ret_val -ne 0 ]]; then
+		log_note "Unable to format $disk slice $slicenum to $size"
+		return 1
+	fi
 	return 0
 }
 
@@ -959,7 +969,7 @@ function partition_disk	#<slice_size> <whole_disk_name>	<total_slices>
 				continue
 			fi
 		fi
-		set_partition $i "$cyl" $slice_size $disk_name
+		log_must set_partition $i "$cyl" $slice_size $disk_name
 		cyl=$(get_endslice $disk_name $i)
 		((i = i+1))
 	done
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_011_neg.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_011_neg.ksh
index 8ea6aa374..8ade2561f 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_011_neg.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_011_neg.ksh
@@ -81,13 +81,13 @@ vfstab_dev=$(find_vfstab_dev)
 if is_linux; then
 	partition_disk $SIZE $disk 7
 	cyl=$(get_endslice $disk $SLICE5)
-	set_partition $SLICE6 "$cyl" $SIZE1 $disk
+	log_must set_partition $SLICE6 "$cyl" $SIZE1 $disk
 else
 	specified_dump_dev=${disk}${SLICE_PREFIX}${SLICE0}
 	saved_dump_dev=$(save_dump_dev)
 
 	cyl=$(get_endslice $disk $SLICE6)
-	set_partition $SLICE7 "$cyl" $SIZE1 $disk
+	log_must set_partition $SLICE7 "$cyl" $SIZE1 $disk
 fi
 create_pool "$TESTPOOL" "$pooldev1"
 
diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/Makefile.am b/tests/zfs-tests/tests/functional/zvol/zvol_misc/Makefile.am
index f72970490..30ef9acbd 100644
--- a/tests/zfs-tests/tests/functional/zvol/zvol_misc/Makefile.am
+++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/Makefile.am
@@ -1,5 +1,6 @@
 pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/zvol/zvol_misc
 dist_pkgdata_SCRIPTS = \
+	zvol_misc_common.kshlib \
 	cleanup.ksh \
 	setup.ksh \
 	zvol_misc_001_neg.ksh \
@@ -8,4 +9,5 @@ dist_pkgdata_SCRIPTS = \
 	zvol_misc_004_pos.ksh \
 	zvol_misc_005_neg.ksh \
 	zvol_misc_006_pos.ksh \
-	zvol_misc_snapdev.ksh
+	zvol_misc_snapdev.ksh \
+	zvol_misc_volmode.ksh
diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib
new file mode 100755
index 000000000..5c8c9847a
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib
@@ -0,0 +1,141 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2017, loli10K <[email protected]>. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
+. $STF_SUITE/tests/functional/zvol/zvol_common.shlib
+
+#
+# Wait for udev to settle, completely.
+# This is quite discomforting, but there's a race condition here
+# (Amazon 2015.09 x86_64 Release (TEST) is good at triggering this)  where the
+# kernel tries to remove zvol device nodes while they're open by [blkid],
+# [zvol_id] or other udev related processes.
+# Calling 'udevadm settle' is not enough: wait for those processes "manually".
+#
+function udev_wait
+{
+	sleep 1
+	udevadm trigger --action=change
+	udevadm settle
+	for i in {1..3}; do
+		blkid="$(pgrep blkid | wc -l)"
+		zvol_id="$(pgrep zvol_id | wc -l)"
+		[[ "0" == "$zvol_id" && "0" == "$blkid" ]] && return
+		udevadm settle
+	done
+	log_fail "Wait timeout reached for udev_wait"
+}
+
+#
+# Clean up udev status
+# This is also a problem on "Amazon 2015.09 x86_64 Release (TEST)" where udev,
+# sometimes, does not clean up /dev/zvol symlinks correctly for removed ZVOLs.
+# Prune those links manually, then tell udev to forget them.
+#
+function udev_cleanup
+{
+	log_note "Pruning broken ZVOL symlinks ..."
+	udevadm settle
+	# find all dangling links and delete them
+	find -L "${ZVOL_DEVDIR}" -type l -print -delete
+	# purge those links from udev database
+	udevadm info --cleanup-db
+}
+
+#
+# Verify $device exists and is a block device
+#
+function blockdev_exists # device
+{
+	typeset device="$1"
+
+	# we wait here instead of doing it in a wrapper around 'zfs set snapdev'
+	# because there are other commands (zfs snap, zfs inherit, zfs destroy)
+	# that can affect device nodes
+	for i in {1..3}; do
+		udev_wait
+		[[ -b "$device" ]] && return 0
+	done
+	log_fail "$device does not exist as a block device"
+}
+
+#
+# Verify $device does not exist
+#
+function blockdev_missing # device
+{
+	typeset device="$1"
+
+	# we wait here instead of doing it in a wrapper around 'zfs set snapdev'
+	# because there are other commands (zfs snap, zfs inherit, zfs destroy)
+	# that can affect device nodes
+	for i in {1..3}; do
+		udev_wait
+		[[ ! -e "$device" ]] && return 0
+	done
+	log_fail "$device exists when not expected"
+}
+
+#
+# Verify $property on $dataset is inherited by $parent and is set to $value
+#
+function verify_inherited # property value dataset parent
+{
+	typeset property="$1"
+	typeset value="$2"
+	typeset dataset="$3"
+	typeset parent="$4"
+
+	typeset val=$(get_prop "$property" "$dataset")
+	typeset src=$(get_source "$property" "$dataset")
+	if [[ "$val" != "$value" || "$src" != "inherited from $parent" ]]; then
+		log_fail "Dataset $dataset did not inherit $property properly:"\
+		    "expected=$value, value=$val, source=$src."
+	fi
+}
+
+#
+# Create a small partition on $device, then verify if we can access it
+#
+function verify_partition # device
+{
+	typeset device="$1"
+
+	if [[ ! -b "$device" ]]; then
+		log_fail "$device is not a block device"
+	fi
+	# create a small dummy partition
+	set_partition 0 1 1m $device
+	# verify we can access the partition on the device
+	devname="$(readlink -f "$device")"
+	if is_linux; then
+		[[ -b "$devname""p1" ]]
+	else
+		[[ -b "$devname""s0" ]]
+	fi
+	return $?
+}
diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev.ksh b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev.ksh
index 57002fe65..8d95bfa39 100755
--- a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev.ksh
+++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_snapdev.ksh
@@ -27,6 +27,7 @@
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
 . $STF_SUITE/tests/functional/zvol/zvol_common.shlib
+. $STF_SUITE/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib
 
 #
 # DESCRIPTION:
@@ -46,60 +47,7 @@ function cleanup
 	datasetexists $ZVOL && log_must zfs destroy -r $ZVOL
 	log_must zfs inherit snapdev $TESTPOOL
 	block_device_wait
-}
-
-#
-# Verify $device exists and is a block device
-#
-function blockdev_exists # device
-{
-	typeset device="$1"
-
-	# we wait here instead of doing it in a wrapper around 'zfs set snapdev'
-	# because there are other commands (zfs snap, zfs inherit, zfs destroy)
-	# that can affect device nodes
-	block_device_wait
-
-	if [[ ! -b "$device" ]]; then
-		log_fail "$device does not exist as a block device"
-	fi
-}
-
-#
-# Verify $device does not exist
-#
-function check_missing # device
-{
-	typeset device="$1"
-
-	# we wait here instead of doing it in a wrapper around 'zfs set snapdev'
-	# because there are other commands (zfs snap, zfs inherit, zfs destroy)
-	# that can affect device nodes
-	block_device_wait
-
-	if [[ -e "$device" ]]; then
-		log_fail "$device exists when not expected"
-	fi
-}
-
-#
-# Verify $property on $dataset is inherited by $parent and is set to $value
-#
-function verify_inherited # property value dataset parent
-{
-	typeset property="$1"
-	typeset value="$2"
-	typeset dataset="$3"
-	typeset parent="$4"
-
-	typeset val=$(get_prop "$property" "$dataset")
-	typeset src=$(get_source "$property" "$dataset")
-	if [[ "$val" != "$value" || "$src" != "inherited from $parent" ]]
-	then
-		log_fail "Dataset $dataset did not inherit $property properly:"\
-		    "expected=$value, value=$val, source=$src."
-	fi
-
+	udev_cleanup
 }
 
 log_assert "Verify that ZFS volume property 'snapdev' works as expected."
@@ -130,14 +78,14 @@ log_must zfs snapshot $SNAP
 log_must zfs set snapdev=visible $ZVOL
 blockdev_exists $SNAPDEV
 log_must zfs set snapdev=hidden $ZVOL
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 log_must zfs destroy $SNAP
 # 2.2 First set snapdev property then create a snapshot
 log_must zfs set snapdev=visible $ZVOL
 log_must zfs snapshot $SNAP
 blockdev_exists $SNAPDEV
 log_must zfs destroy $SNAP
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 # 2.3 Verify setting to the same value multiple times does not lead to issues
 log_must zfs snapshot $SNAP
 log_must zfs set snapdev=visible $ZVOL
@@ -145,9 +93,9 @@ blockdev_exists $SNAPDEV
 log_must zfs set snapdev=visible $ZVOL
 blockdev_exists $SNAPDEV
 log_must zfs set snapdev=hidden $ZVOL
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 log_must zfs set snapdev=hidden $ZVOL
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 log_must zfs destroy $SNAP
 
 # 3. Verify "snapdev" is inherited correctly
@@ -160,14 +108,14 @@ blockdev_exists $SNAPDEV
 # 3.2 Check snapdev=hidden case
 log_must zfs set snapdev=hidden $TESTPOOL
 verify_inherited 'snapdev' 'hidden' $ZVOL $TESTPOOL
-check_missing $SNAPDEV
+blockdev_missing $SNAPDEV
 # 3.3 Check inheritance on multiple levels
 log_must zfs snapshot $SUBSNAP
 log_must zfs inherit snapdev $SUBZVOL
 log_must zfs set snapdev=hidden $VOLFS
 log_must zfs set snapdev=visible $TESTPOOL
 verify_inherited 'snapdev' 'hidden' $SUBZVOL $VOLFS
-check_missing $SUBSNAPDEV
+blockdev_missing $SUBSNAPDEV
 blockdev_exists $SNAPDEV
 
 log_pass "ZFS volume property 'snapdev' works as expected"
diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh
new file mode 100755
index 000000000..5cf6a60b3
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_volmode.ksh
@@ -0,0 +1,225 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2017, loli10K <[email protected]>. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
+. $STF_SUITE/tests/functional/zvol/zvol_common.shlib
+. $STF_SUITE/tests/functional/zvol/zvol_misc/zvol_misc_common.kshlib
+
+#
+# DESCRIPTION:
+# Verify that ZFS volume property "volmode" works as intended.
+#
+# STRATEGY:
+# 1. Verify "volmode" property does not accept invalid values
+# 2. Verify "volmode=none" hides ZVOL device nodes
+# 3. Verify "volmode=full" exposes a fully functional device
+# 4. Verify "volmode=dev" hides partition info on the device
+# 5. Verify "volmode=default" behaves accordingly to "volmode" module parameter
+# 6. Verify "volmode" property is inherited correctly
+# 7. Verify "volmode" behaves accordingly to zvol_inhibit_dev (Linux only)
+#
+# NOTE: changing volmode may need to remove minors, which could be open, so call
+#       udev_wait() before we "zfs set volmode=<value>".
+
+verify_runnable "global"
+
+function cleanup
+{
+	datasetexists $VOLFS && log_must_busy zfs destroy -r $VOLFS
+	datasetexists $ZVOL && log_must_busy zfs destroy -r $ZVOL
+	log_must zfs inherit volmode $TESTPOOL
+	udev_wait
+	sysctl_inhibit_dev 0
+	udev_cleanup
+}
+
+#
+# Set zvol_inhibit_dev tunable to $value
+#
+function sysctl_inhibit_dev # value
+{
+	typeset value="$1"
+
+	if is_linux; then
+		log_note "Setting zvol_inhibit_dev tunable to $value"
+		log_must eval "echo $value > "\
+		    "/sys/module/zfs/parameters/zvol_inhibit_dev"
+	fi
+}
+
+#
+# Set volmode tunable to $value
+#
+function sysctl_volmode # value
+{
+	typeset value="$1"
+
+	log_note "Setting volmode tunable to $value"
+	if is_linux; then
+		echo "$value" > '/sys/module/zfs/parameters/zvol_volmode'
+	else
+		sysctl 'vfs.zfs.vol.mode' "$value"
+	fi
+	if [[ $? -ne 0 ]]; then
+		log_fail "Unable to set volmode tunable to $value"
+	fi
+}
+
+log_assert "Verify that ZFS volume property 'volmode' works as intended"
+log_onexit cleanup
+
+VOLFS="$TESTPOOL/volfs"
+ZVOL="$TESTPOOL/vol"
+ZDEV="${ZVOL_DEVDIR}/$ZVOL"
+SUBZVOL="$VOLFS/subvol"
+SUBZDEV="${ZVOL_DEVDIR}/$SUBZVOL"
+
+log_must zfs create -o mountpoint=none $VOLFS
+log_must zfs create -V $VOLSIZE -s $SUBZVOL
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+
+# 1. Verify "volmode" property does not accept invalid values
+typeset badvals=("off" "on" "1" "nope" "-")
+for badval in ${badvals[@]}
+do
+	log_mustnot zfs set volmode="$badval" $ZVOL
+done
+
+# 2. Verify "volmode=none" hides ZVOL device nodes
+log_must zfs set volmode=none $ZVOL
+blockdev_missing $ZDEV
+log_must_busy zfs destroy $ZVOL
+
+# 3. Verify "volmode=full" exposes a fully functional device
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=full $ZVOL
+blockdev_exists $ZDEV
+log_must verify_partition $ZDEV
+udev_wait
+# 3.1 Verify "volmode=geom" is an alias for "volmode=full"
+log_must zfs set volmode=geom $ZVOL
+blockdev_exists $ZDEV
+if [[ "$(get_prop 'volmode' $ZVOL)" != "full" ]]; then
+	log_fail " Volmode value 'geom' is not an alias for 'full'"
+fi
+udev_wait
+log_must_busy zfs destroy $ZVOL
+
+# 4. Verify "volmode=dev" hides partition info on the device
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=dev $ZVOL
+blockdev_exists $ZDEV
+log_mustnot verify_partition $ZDEV
+udev_wait
+log_must_busy zfs destroy $ZVOL
+
+# 5. Verify "volmode=default" behaves accordingly to "volmode" module parameter
+# 5.1 Verify sysctl "volmode=full"
+sysctl_volmode 1
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=default $ZVOL
+blockdev_exists $ZDEV
+log_must verify_partition $ZDEV
+udev_wait
+log_must_busy zfs destroy $ZVOL
+# 5.2 Verify sysctl "volmode=dev"
+sysctl_volmode 2
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=default $ZVOL
+blockdev_exists $ZDEV
+log_mustnot verify_partition $ZDEV
+udev_wait
+log_must_busy zfs destroy $ZVOL
+# 5.2 Verify sysctl "volmode=none"
+sysctl_volmode 3
+log_must zfs create -V $VOLSIZE -s $ZVOL
+udev_wait
+log_must zfs set volmode=default $ZVOL
+blockdev_missing $ZDEV
+
+# 6. Verify "volmode" property is inherited correctly
+log_must zfs inherit volmode $ZVOL
+# 6.1 Check volmode=full case
+log_must zfs set volmode=full $TESTPOOL
+verify_inherited 'volmode' 'full' $ZVOL $TESTPOOL
+blockdev_exists $ZDEV
+# 6.2 Check volmode=none case
+log_must zfs set volmode=none $TESTPOOL
+verify_inherited 'volmode' 'none' $ZVOL $TESTPOOL
+blockdev_missing $ZDEV
+# 6.3 Check volmode=dev case
+log_must zfs set volmode=dev $TESTPOOL
+verify_inherited 'volmode' 'dev' $ZVOL $TESTPOOL
+blockdev_exists $ZDEV
+# 6.4 Check volmode=default case
+sysctl_volmode 1
+log_must zfs set volmode=default $TESTPOOL
+verify_inherited 'volmode' 'default' $ZVOL $TESTPOOL
+blockdev_exists $ZDEV
+# 6.5 Check inheritance on multiple levels
+log_must zfs inherit volmode $SUBZVOL
+udev_wait
+log_must zfs set volmode=none $VOLFS
+udev_wait
+log_must zfs set volmode=full $TESTPOOL
+verify_inherited 'volmode' 'none' $SUBZVOL $VOLFS
+blockdev_missing $SUBZDEV
+blockdev_exists $ZDEV
+log_must_busy zfs destroy $ZVOL
+log_must_busy zfs destroy $SUBZVOL
+
+# 7. Verify "volmode" behaves accordingly to zvol_inhibit_dev (Linux only)
+if is_linux; then
+	sysctl_inhibit_dev 1
+	# 7.1 Verify device nodes not are not created with "volmode=full"
+	sysctl_volmode 1
+	log_must zfs create -V $VOLSIZE -s $ZVOL
+	blockdev_missing $ZDEV
+	log_must zfs set volmode=full $ZVOL
+	blockdev_missing $ZDEV
+	log_must_busy zfs destroy $ZVOL
+	# 7.1 Verify device nodes not are not created with "volmode=dev"
+	sysctl_volmode 2
+	log_must zfs create -V $VOLSIZE -s $ZVOL
+	blockdev_missing $ZDEV
+	log_must zfs set volmode=dev $ZVOL
+	blockdev_missing $ZDEV
+	log_must_busy zfs destroy $ZVOL
+	# 7.1 Verify device nodes not are not created with "volmode=none"
+	sysctl_volmode 3
+	log_must zfs create -V $VOLSIZE -s $ZVOL
+	blockdev_missing $ZDEV
+	log_must zfs set volmode=none $ZVOL
+	blockdev_missing $ZDEV
+fi
+
+log_pass "Verify that ZFS volume property 'volmode' works as intended"