Illumos #2619 and #2747

2619 asynchronous destruction of ZFS file systems
2747 SPA versioning with zfs feature flags
Reviewed by: Matt Ahrens <[email protected]>
Reviewed by: George Wilson <[email protected]>
Reviewed by: Richard Lowe <[email protected]>
Reviewed by: Dan Kruchinin <[email protected]>
Approved by: Eric Schrock <[email protected]>

References:
  illumos/illumos-gate@53089ab7c84db6fb76c16ca50076c147cda11757
  illumos/illumos-gate@ad135b5d644628e791c3188a6ecbd9c257961ef8
  illumos changeset: 13700:2889e2596bd6
  https://www.illumos.org/issues/2619
  https://www.illumos.org/issues/2747

NOTE: The grub specific changes were not ported.  This change
must be made to the Linux grub packages.

Ported-by: Brian Behlendorf <[email protected]>

7 files changed, 803 insertions, 41 deletions

diff --git a/cmd/Makefile.am b/cmd/Makefile.am
index 478da2616..afdba3440 100644
--- a/cmd/Makefile.am
+++ b/cmd/Makefile.am
@@ -1,2 +1,2 @@
-SUBDIRS  = zfs zpool zdb zinject zstreamdump ztest zpios mount_zfs
+SUBDIRS  = zfs zpool zdb zhack zinject zstreamdump ztest zpios mount_zfs
 SUBDIRS += zpool_layout zvol_id zpool_id vdev_id
diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c
index ce6318ea1..de4ac510e 100644
--- a/cmd/zdb/zdb.c
+++ b/cmd/zdb/zdb.c
@@ -18,8 +18,10 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #include <stdio.h>
@@ -54,6 +56,7 @@
 #include <sys/zfs_fuid.h>
 #include <sys/arc.h>
 #include <sys/ddt.h>
+#include <sys/zfeature.h>
 #undef ZFS_MAXNAMELEN
 #include <libzfs.h>
 
@@ -62,7 +65,8 @@
 #define	ZDB_CHECKSUM_NAME(idx) ((idx) < ZIO_CHECKSUM_FUNCTIONS ? \
     zio_checksum_table[(idx)].ci_name : "UNKNOWN")
 #define	ZDB_OT_NAME(idx) ((idx) < DMU_OT_NUMTYPES ? \
-    dmu_ot[(idx)].ot_name : "UNKNOWN")
+    dmu_ot[(idx)].ot_name : DMU_OT_IS_VALID(idx) ? \
+    dmu_ot_byteswap[DMU_OT_BYTESWAP(idx)].ob_name : "UNKNOWN")
 #define	ZDB_OT_TYPE(idx) ((idx) < DMU_OT_NUMTYPES ? (idx) : DMU_OT_NUMTYPES)
 
 #ifndef lint
@@ -1099,7 +1103,7 @@ dump_dsl_dataset(objset_t *os, uint64_t object, void *data, size_t size)
 
 	ASSERT(size == sizeof (*ds));
 	crtime = ds->ds_creation_time;
-	zdb_nicenum(ds->ds_used_bytes, used);
+	zdb_nicenum(ds->ds_referenced_bytes, used);
 	zdb_nicenum(ds->ds_compressed_bytes, compressed);
 	zdb_nicenum(ds->ds_uncompressed_bytes, uncompressed);
 	zdb_nicenum(ds->ds_unique_bytes, unique);
@@ -1143,6 +1147,44 @@ dump_dsl_dataset(objset_t *os, uint64_t object, void *data, size_t size)
 
 /* ARGSUSED */
 static int
+dump_bptree_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
+{
+	char blkbuf[BP_SPRINTF_LEN];
+
+	if (bp->blk_birth != 0) {
+		sprintf_blkptr(blkbuf, bp);
+		(void) printf("\t%s\n", blkbuf);
+	}
+	return (0);
+}
+
+static void
+dump_bptree(objset_t *os, uint64_t obj, char *name)
+{
+	char bytes[32];
+	bptree_phys_t *bt;
+	dmu_buf_t *db;
+
+	if (dump_opt['d'] < 3)
+		return;
+
+	VERIFY3U(0, ==, dmu_bonus_hold(os, obj, FTAG, &db));
+	bt = db->db_data;
+	zdb_nicenum(bt->bt_bytes, bytes);
+	(void) printf("\n    %s: %llu datasets, %s\n",
+	    name, (unsigned long long)(bt->bt_end - bt->bt_begin), bytes);
+	dmu_buf_rele(db, FTAG);
+
+	if (dump_opt['d'] < 5)
+		return;
+
+	(void) printf("\n");
+
+	(void) bptree_iterate(os, obj, B_FALSE, dump_bptree_cb, NULL, NULL);
+}
+
+/* ARGSUSED */
+static int
 dump_bpobj_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
 {
 	char blkbuf[BP_SPRINTF_LEN];
@@ -1888,11 +1930,13 @@ typedef struct zdb_blkstats {
  */
 #define	ZDB_OT_DEFERRED	(DMU_OT_NUMTYPES + 0)
 #define	ZDB_OT_DITTO	(DMU_OT_NUMTYPES + 1)
-#define	ZDB_OT_TOTAL	(DMU_OT_NUMTYPES + 2)
+#define	ZDB_OT_OTHER	(DMU_OT_NUMTYPES + 2)
+#define	ZDB_OT_TOTAL	(DMU_OT_NUMTYPES + 3)
 
 static char *zdb_ot_extname[] = {
 	"deferred free",
 	"dedup ditto",
+	"other",
 	"Total",
 };
 
@@ -1974,9 +2018,10 @@ zdb_blkptr_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, arc_buf_t *pbuf,
 
 	type = BP_GET_TYPE(bp);
 
-	zdb_count_block(zcb, zilog, bp, type);
+	zdb_count_block(zcb, zilog, bp,
+	    (type & DMU_OT_NEWTYPE) ? ZDB_OT_OTHER : type);
 
-	is_metadata = (BP_GET_LEVEL(bp) != 0 || dmu_ot[type].ot_metadata);
+	is_metadata = (BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type));
 
 	if (dump_opt['c'] > 1 || (dump_opt['c'] && is_metadata)) {
 		int ioerr;
@@ -2207,6 +2252,12 @@ dump_block_stats(spa_t *spa)
 	    count_block_cb, &zcb, NULL);
 	(void) bpobj_iterate_nofree(&spa->spa_dsl_pool->dp_free_bpobj,
 	    count_block_cb, &zcb, NULL);
+	if (spa_feature_is_active(spa,
+	    &spa_feature_table[SPA_FEATURE_ASYNC_DESTROY])) {
+		VERIFY3U(0, ==, bptree_iterate(spa->spa_meta_objset,
+		    spa->spa_dsl_pool->dp_bptree_obj, B_FALSE, count_block_cb,
+		    &zcb, NULL));
+	}
 
 	if (dump_opt['c'] > 1)
 		flags |= TRAVERSE_PREFETCH_DATA;
@@ -2383,7 +2434,7 @@ zdb_ddt_add_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
 	}
 
 	if (BP_IS_HOLE(bp) || BP_GET_CHECKSUM(bp) == ZIO_CHECKSUM_OFF ||
-	    BP_GET_LEVEL(bp) > 0 || dmu_ot[BP_GET_TYPE(bp)].ot_metadata)
+	    BP_GET_LEVEL(bp) > 0 || DMU_OT_IS_METADATA(BP_GET_TYPE(bp)))
 		return (0);
 
 	ddt_key_fill(&zdde_search.zdde_key, bp);
@@ -2491,7 +2542,14 @@ dump_zpool(spa_t *spa)
 			dump_bpobj(&spa->spa_deferred_bpobj, "Deferred frees");
 			if (spa_version(spa) >= SPA_VERSION_DEADLISTS) {
 				dump_bpobj(&spa->spa_dsl_pool->dp_free_bpobj,
-				    "Pool frees");
+				    "Pool snapshot frees");
+			}
+
+			if (spa_feature_is_active(spa,
+			    &spa_feature_table[SPA_FEATURE_ASYNC_DESTROY])) {
+				dump_bptree(spa->spa_meta_objset,
+				    spa->spa_dsl_pool->dp_bptree_obj,
+				    "Pool dataset frees");
 			}
 			dump_dtl(spa->spa_root_vdev, 0);
 		}
diff --git a/cmd/zhack/.gitignore b/cmd/zhack/.gitignore
new file mode 100644
index 000000000..763a18898
--- /dev/null
+++ b/cmd/zhack/.gitignore
@@ -0,0 +1 @@
+/zhack
diff --git a/cmd/zhack/Makefile.am b/cmd/zhack/Makefile.am
new file mode 100644
index 000000000..47da2453e
--- /dev/null
+++ b/cmd/zhack/Makefile.am
@@ -0,0 +1,18 @@
+include $(top_srcdir)/config/Rules.am
+
+DEFAULT_INCLUDES += \
+	-I$(top_srcdir)/include \
+	-I$(top_srcdir)/lib/libspl/include
+
+sbin_PROGRAMS = zhack
+
+zhack_SOURCES = \
+	$(top_srcdir)/cmd/zhack/zhack.c
+
+zhack_LDADD = \
+	$(top_builddir)/lib/libnvpair/libnvpair.la \
+	$(top_builddir)/lib/libuutil/libuutil.la \
+	$(top_builddir)/lib/libzpool/libzpool.la \
+	$(top_builddir)/lib/libzfs/libzfs.la
+
+zhack_LDFLAGS = -pthread -lm $(ZLIB) -lrt -ldl $(LIBUUID) $(LIBBLKID)
diff --git a/cmd/zhack/zhack.c b/cmd/zhack/zhack.c
new file mode 100644
index 000000000..b2cf815ca
--- /dev/null
+++ b/cmd/zhack/zhack.c
@@ -0,0 +1,533 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2012 by Delphix. All rights reserved.
+ */
+
+/*
+ * zhack is a debugging tool that can write changes to ZFS pool using libzpool
+ * for testing purposes. Altering pools with zhack is unsupported and may
+ * result in corrupted pools.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <sys/zfs_context.h>
+#include <sys/spa.h>
+#include <sys/spa_impl.h>
+#include <sys/dmu.h>
+#include <sys/zap.h>
+#include <sys/zfs_znode.h>
+#include <sys/dsl_synctask.h>
+#include <sys/vdev.h>
+#include <sys/fs/zfs.h>
+#include <sys/dmu_objset.h>
+#include <sys/dsl_pool.h>
+#include <sys/zio_checksum.h>
+#include <sys/zio_compress.h>
+#include <sys/zfeature.h>
+#undef ZFS_MAXNAMELEN
+#include <libzfs.h>
+
+extern boolean_t zfeature_checks_disable;
+
+const char cmdname[] = "zhack";
+libzfs_handle_t *g_zfs;
+static importargs_t g_importargs;
+static char *g_pool;
+static boolean_t g_readonly;
+
+static void
+usage(void)
+{
+	(void) fprintf(stderr,
+	    "Usage: %s [-c cachefile] [-d dir] <subcommand> <args> ...\n"
+	    "where <subcommand> <args> is one of the following:\n"
+	    "\n", cmdname);
+
+	(void) fprintf(stderr,
+	    "    feature stat <pool>\n"
+	    "        print information about enabled features\n"
+	    "    feature enable [-d desc] <pool> <feature>\n"
+	    "        add a new enabled feature to the pool\n"
+	    "        -d <desc> sets the feature's description\n"
+	    "    feature ref [-md] <pool> <feature>\n"
+	    "        change the refcount on the given feature\n"
+	    "        -d decrease instead of increase the refcount\n"
+	    "        -m add the feature to the label if increasing refcount\n"
+	    "\n"
+	    "    <feature> : should be a feature guid\n");
+	exit(1);
+}
+
+
+static void
+fatal(const char *fmt, ...)
+{
+	va_list ap;
+
+	va_start(ap, fmt);
+	(void) fprintf(stderr, "%s: ", cmdname);
+	(void) vfprintf(stderr, fmt, ap);
+	va_end(ap);
+	(void) fprintf(stderr, "\n");
+
+	exit(1);
+}
+
+/* ARGSUSED */
+static int
+space_delta_cb(dmu_object_type_t bonustype, void *data,
+    uint64_t *userp, uint64_t *groupp)
+{
+	/*
+	 * Is it a valid type of object to track?
+	 */
+	if (bonustype != DMU_OT_ZNODE && bonustype != DMU_OT_SA)
+		return (ENOENT);
+	(void) fprintf(stderr, "modifying object that needs user accounting");
+	abort();
+	/* NOTREACHED */
+}
+
+/*
+ * Target is the dataset whose pool we want to open.
+ */
+static void
+import_pool(const char *target, boolean_t readonly)
+{
+	nvlist_t *config;
+	nvlist_t *pools;
+	int error;
+	char *sepp;
+	spa_t *spa;
+	nvpair_t *elem;
+	nvlist_t *props;
+	const char *name;
+
+	kernel_init(readonly ? FREAD : (FREAD | FWRITE));
+	g_zfs = libzfs_init();
+	ASSERT(g_zfs != NULL);
+
+	dmu_objset_register_type(DMU_OST_ZFS, space_delta_cb);
+
+	g_readonly = readonly;
+
+	/*
+	 * If we only want readonly access, it's OK if we find
+	 * a potentially-active (ie, imported into the kernel) pool from the
+	 * default cachefile.
+	 */
+	if (readonly && spa_open(target, &spa, FTAG) == 0) {
+		spa_close(spa, FTAG);
+		return;
+	}
+
+	g_importargs.unique = B_TRUE;
+	g_importargs.can_be_active = readonly;
+	g_pool = strdup(target);
+	if ((sepp = strpbrk(g_pool, "/@")) != NULL)
+		*sepp = '\0';
+	g_importargs.poolname = g_pool;
+	pools = zpool_search_import(g_zfs, &g_importargs);
+
+	if (pools == NULL || nvlist_next_nvpair(pools, NULL) == NULL) {
+		if (!g_importargs.can_be_active) {
+			g_importargs.can_be_active = B_TRUE;
+			if (zpool_search_import(g_zfs, &g_importargs) != NULL ||
+			    spa_open(target, &spa, FTAG) == 0) {
+				fatal("cannot import '%s': pool is active; run "
+				    "\"zpool export %s\" first\n",
+				    g_pool, g_pool);
+			}
+		}
+
+		fatal("cannot import '%s': no such pool available\n", g_pool);
+	}
+
+	elem = nvlist_next_nvpair(pools, NULL);
+	name = nvpair_name(elem);
+	VERIFY(nvpair_value_nvlist(elem, &config) == 0);
+
+	props = NULL;
+	if (readonly) {
+		VERIFY(nvlist_alloc(&props, NV_UNIQUE_NAME, 0) == 0);
+		VERIFY(nvlist_add_uint64(props,
+		    zpool_prop_to_name(ZPOOL_PROP_READONLY), 1) == 0);
+	}
+
+	zfeature_checks_disable = B_TRUE;
+	error = spa_import(name, config, props, ZFS_IMPORT_NORMAL);
+	zfeature_checks_disable = B_FALSE;
+	if (error == EEXIST)
+		error = 0;
+
+	if (error)
+		fatal("can't import '%s': %s", name, strerror(error));
+}
+
+static void
+zhack_spa_open(const char *target, boolean_t readonly, void *tag, spa_t **spa)
+{
+	int err;
+
+	import_pool(target, readonly);
+
+	zfeature_checks_disable = B_TRUE;
+	err = spa_open(target, spa, tag);
+	zfeature_checks_disable = B_FALSE;
+
+	if (err != 0)
+		fatal("cannot open '%s': %s", target, strerror(err));
+	if (spa_version(*spa) < SPA_VERSION_FEATURES) {
+		fatal("'%s' has version %d, features not enabled", target,
+		    (int)spa_version(*spa));
+	}
+}
+
+static void
+dump_obj(objset_t *os, uint64_t obj, const char *name)
+{
+	zap_cursor_t zc;
+	zap_attribute_t za;
+
+	(void) printf("%s_obj:\n", name);
+
+	for (zap_cursor_init(&zc, os, obj);
+	    zap_cursor_retrieve(&zc, &za) == 0;
+	    zap_cursor_advance(&zc)) {
+		if (za.za_integer_length == 8) {
+			ASSERT(za.za_num_integers == 1);
+			(void) printf("\t%s = %llu\n",
+			    za.za_name, (u_longlong_t)za.za_first_integer);
+		} else {
+			ASSERT(za.za_integer_length == 1);
+			char val[1024];
+			VERIFY(zap_lookup(os, obj, za.za_name,
+			    1, sizeof (val), val) == 0);
+			(void) printf("\t%s = %s\n", za.za_name, val);
+		}
+	}
+	zap_cursor_fini(&zc);
+}
+
+static void
+dump_mos(spa_t *spa)
+{
+	nvlist_t *nv = spa->spa_label_features;
+	nvpair_t *pair;
+
+	(void) printf("label config:\n");
+	for (pair = nvlist_next_nvpair(nv, NULL);
+	    pair != NULL;
+	    pair = nvlist_next_nvpair(nv, pair)) {
+		(void) printf("\t%s\n", nvpair_name(pair));
+	}
+}
+
+static void
+zhack_do_feature_stat(int argc, char **argv)
+{
+	spa_t *spa;
+	objset_t *os;
+	char *target;
+
+	argc--;
+	argv++;
+
+	if (argc < 1) {
+		(void) fprintf(stderr, "error: missing pool name\n");
+		usage();
+	}
+	target = argv[0];
+
+	zhack_spa_open(target, B_TRUE, FTAG, &spa);
+	os = spa->spa_meta_objset;
+
+	dump_obj(os, spa->spa_feat_for_read_obj, "for_read");
+	dump_obj(os, spa->spa_feat_for_write_obj, "for_write");
+	dump_obj(os, spa->spa_feat_desc_obj, "descriptions");
+	dump_mos(spa);
+
+	spa_close(spa, FTAG);
+}
+
+static void
+feature_enable_sync(void *arg1, void *arg2, dmu_tx_t *tx)
+{
+	spa_t *spa = arg1;
+	zfeature_info_t *feature = arg2;
+
+	spa_feature_enable(spa, feature, tx);
+}
+
+static void
+zhack_do_feature_enable(int argc, char **argv)
+{
+	char c;
+	char *desc, *target;
+	spa_t *spa;
+	objset_t *mos;
+	zfeature_info_t feature;
+	zfeature_info_t *nodeps[] = { NULL };
+
+	/*
+	 * Features are not added to the pool's label until their refcounts
+	 * are incremented, so fi_mos can just be left as false for now.
+	 */
+	desc = NULL;
+	feature.fi_uname = "zhack";
+	feature.fi_mos = B_FALSE;
+	feature.fi_can_readonly = B_FALSE;
+	feature.fi_depends = nodeps;
+
+	optind = 1;
+	while ((c = getopt(argc, argv, "rmd:")) != -1) {
+		switch (c) {
+		case 'r':
+			feature.fi_can_readonly = B_TRUE;
+			break;
+		case 'd':
+			desc = strdup(optarg);
+			break;
+		default:
+			usage();
+			break;
+		}
+	}
+
+	if (desc == NULL)
+		desc = strdup("zhack injected");
+	feature.fi_desc = desc;
+
+	argc -= optind;
+	argv += optind;
+
+	if (argc < 2) {
+		(void) fprintf(stderr, "error: missing feature or pool name\n");
+		usage();
+	}
+	target = argv[0];
+	feature.fi_guid = argv[1];
+
+	if (!zfeature_is_valid_guid(feature.fi_guid))
+		fatal("invalid feature guid: %s", feature.fi_guid);
+
+	zhack_spa_open(target, B_FALSE, FTAG, &spa);
+	mos = spa->spa_meta_objset;
+
+	if (0 == zfeature_lookup_guid(feature.fi_guid, NULL))
+		fatal("'%s' is a real feature, will not enable");
+	if (0 == zap_contains(mos, spa->spa_feat_desc_obj, feature.fi_guid))
+		fatal("feature already enabled: %s", feature.fi_guid);
+
+	VERIFY3U(0, ==, dsl_sync_task_do(spa->spa_dsl_pool, NULL,
+	    feature_enable_sync, spa, &feature, 5));
+
+	spa_close(spa, FTAG);
+
+	free(desc);
+}
+
+static void
+feature_incr_sync(void *arg1, void *arg2, dmu_tx_t *tx)
+{
+	spa_t *spa = arg1;
+	zfeature_info_t *feature = arg2;
+
+	spa_feature_incr(spa, feature, tx);
+}
+
+static void
+feature_decr_sync(void *arg1, void *arg2, dmu_tx_t *tx)
+{
+	spa_t *spa = arg1;
+	zfeature_info_t *feature = arg2;
+
+	spa_feature_decr(spa, feature, tx);
+}
+
+static void
+zhack_do_feature_ref(int argc, char **argv)
+{
+	char c;
+	char *target;
+	boolean_t decr = B_FALSE;
+	spa_t *spa;
+	objset_t *mos;
+	zfeature_info_t feature;
+	zfeature_info_t *nodeps[] = { NULL };
+
+	/*
+	 * fi_desc does not matter here because it was written to disk
+	 * when the feature was enabled, but we need to properly set the
+	 * feature for read or write based on the information we read off
+	 * disk later.
+	 */
+	feature.fi_uname = "zhack";
+	feature.fi_mos = B_FALSE;
+	feature.fi_desc = NULL;
+	feature.fi_depends = nodeps;
+
+	optind = 1;
+	while ((c = getopt(argc, argv, "md")) != -1) {
+		switch (c) {
+		case 'm':
+			feature.fi_mos = B_TRUE;
+			break;
+		case 'd':
+			decr = B_TRUE;
+			break;
+		default:
+			usage();
+			break;
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (argc < 2) {
+		(void) fprintf(stderr, "error: missing feature or pool name\n");
+		usage();
+	}
+	target = argv[0];
+	feature.fi_guid = argv[1];
+
+	if (!zfeature_is_valid_guid(feature.fi_guid))
+		fatal("invalid feature guid: %s", feature.fi_guid);
+
+	zhack_spa_open(target, B_FALSE, FTAG, &spa);
+	mos = spa->spa_meta_objset;
+
+	if (0 == zfeature_lookup_guid(feature.fi_guid, NULL))
+		fatal("'%s' is a real feature, will not change refcount");
+
+	if (0 == zap_contains(mos, spa->spa_feat_for_read_obj,
+	    feature.fi_guid)) {
+		feature.fi_can_readonly = B_FALSE;
+	} else if (0 == zap_contains(mos, spa->spa_feat_for_write_obj,
+	    feature.fi_guid)) {
+		feature.fi_can_readonly = B_TRUE;
+	} else {
+		fatal("feature is not enabled: %s", feature.fi_guid);
+	}
+
+	if (decr && !spa_feature_is_active(spa, &feature))
+		fatal("feature refcount already 0: %s", feature.fi_guid);
+
+	VERIFY3U(0, ==, dsl_sync_task_do(spa->spa_dsl_pool, NULL,
+	    decr ? feature_decr_sync : feature_incr_sync, spa, &feature, 5));
+
+	spa_close(spa, FTAG);
+}
+
+static int
+zhack_do_feature(int argc, char **argv)
+{
+	char *subcommand;
+
+	argc--;
+	argv++;
+	if (argc == 0) {
+		(void) fprintf(stderr,
+		    "error: no feature operation specified\n");
+		usage();
+	}
+
+	subcommand = argv[0];
+	if (strcmp(subcommand, "stat") == 0) {
+		zhack_do_feature_stat(argc, argv);
+	} else if (strcmp(subcommand, "enable") == 0) {
+		zhack_do_feature_enable(argc, argv);
+	} else if (strcmp(subcommand, "ref") == 0) {
+		zhack_do_feature_ref(argc, argv);
+	} else {
+		(void) fprintf(stderr, "error: unknown subcommand: %s\n",
+		    subcommand);
+		usage();
+	}
+
+	return (0);
+}
+
+#define	MAX_NUM_PATHS 1024
+
+int
+main(int argc, char **argv)
+{
+	extern void zfs_prop_init(void);
+
+	char *path[MAX_NUM_PATHS];
+	const char *subcommand;
+	int rv = 0;
+	char c;
+
+	g_importargs.path = path;
+
+	dprintf_setup(&argc, argv);
+	zfs_prop_init();
+
+	while ((c = getopt(argc, argv, "c:d:")) != -1) {
+		switch (c) {
+		case 'c':
+			g_importargs.cachefile = optarg;
+			break;
+		case 'd':
+			assert(g_importargs.paths < MAX_NUM_PATHS);
+			g_importargs.path[g_importargs.paths++] = optarg;
+			break;
+		default:
+			usage();
+			break;
+		}
+	}
+
+	argc -= optind;
+	argv += optind;
+	optind = 1;
+
+	if (argc == 0) {
+		(void) fprintf(stderr, "error: no command specified\n");
+		usage();
+	}
+
+	subcommand = argv[0];
+
+	if (strcmp(subcommand, "feature") == 0) {
+		rv = zhack_do_feature(argc, argv);
+	} else {
+		(void) fprintf(stderr, "error: unknown subcommand: %s\n",
+		    subcommand);
+		usage();
+	}
+
+	if (!g_readonly && spa_export(g_pool, NULL, B_TRUE, B_TRUE) != 0) {
+		fatal("pool export failed; "
+		    "changes may not be committed to disk\n");
+	}
+
+	libzfs_fini(g_zfs);
+	kernel_fini();
+
+	return (rv);
+}
diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c
index cb0535a98..d0c0a923f 100644
--- a/cmd/zpool/zpool_main.c
+++ b/cmd/zpool/zpool_main.c
@@ -54,6 +54,7 @@
 
 #include "zpool_util.h"
 #include "zfs_comutil.h"
+#include "zfeature_common.h"
 
 #include "statcommon.h"
 
@@ -208,7 +209,7 @@ get_usage(zpool_help_t idx) {
 	case HELP_CLEAR:
 		return (gettext("\tclear [-nF] <pool> [device]\n"));
 	case HELP_CREATE:
-		return (gettext("\tcreate [-fn] [-o property=value] ... \n"
+		return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
 		    "\t    [-O file-system-property=value] ... \n"
 		    "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
 	case HELP_DESTROY:
@@ -341,6 +342,12 @@ usage(boolean_t requested)
 		/* Iterate over all properties */
 		(void) zprop_iter(print_prop_cb, fp, B_FALSE, B_TRUE,
 		    ZFS_TYPE_POOL);
+
+		(void) fprintf(fp, "\t%-15s   ", "feature@...");
+		(void) fprintf(fp, "YES   disabled | enabled | active\n");
+
+		(void) fprintf(fp, gettext("\nThe feature@ properties must be "
+		    "appended with a feature name.\nSee zpool-features(5).\n"));
 	}
 
 	/*
@@ -407,12 +414,16 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
 	proplist = *props;
 
 	if (poolprop) {
-		if ((prop = zpool_name_to_prop(propname)) == ZPROP_INVAL) {
+		if ((prop = zpool_name_to_prop(propname)) == ZPROP_INVAL &&
+		    !zpool_prop_feature(propname)) {
 			(void) fprintf(stderr, gettext("property '%s' is "
 			    "not a valid pool property\n"), propname);
 			return (2);
 		}
-		normnm = zpool_prop_to_name(prop);
+		if (zpool_prop_feature(propname))
+			normnm = propname;
+		else
+			normnm = zpool_prop_to_name(prop);
 	} else {
 		if ((fprop = zfs_name_to_prop(propname)) != ZPROP_INVAL) {
 			normnm = zfs_prop_to_name(fprop);
@@ -601,7 +612,7 @@ zpool_do_remove(int argc, char **argv)
 }
 
 /*
- * zpool create [-fn] [-o property=value] ...
+ * zpool create [-fnd] [-o property=value] ...
  *		[-O file-system-property=value] ...
  *		[-R root] [-m mountpoint] <pool> <dev> ...
  *
@@ -610,8 +621,10 @@ zpool_do_remove(int argc, char **argv)
  *		were to be created.
  *      -R	Create a pool under an alternate root
  *      -m	Set default mountpoint for the root dataset.  By default it's
- *      	'/<pool>'
+ *		'/<pool>'
  *	-o	Set property=value.
+ *	-d	Don't automatically enable all supported pool features
+ *		(individual features can be enabled with -o).
  *	-O	Set fsproperty=value in the pool's root file system
  *
  * Creates the named pool according to the given vdev specification.  The
@@ -624,6 +637,7 @@ zpool_do_create(int argc, char **argv)
 {
 	boolean_t force = B_FALSE;
 	boolean_t dryrun = B_FALSE;
+	boolean_t enable_all_pool_feat = B_TRUE;
 	int c;
 	nvlist_t *nvroot = NULL;
 	char *poolname;
@@ -635,7 +649,7 @@ zpool_do_create(int argc, char **argv)
 	char *propval;
 
 	/* check options */
-	while ((c = getopt(argc, argv, ":fnR:m:o:O:")) != -1) {
+	while ((c = getopt(argc, argv, ":fndR:m:o:O:")) != -1) {
 		switch (c) {
 		case 'f':
 			force = B_TRUE;
@@ -643,6 +657,9 @@ zpool_do_create(int argc, char **argv)
 		case 'n':
 			dryrun = B_TRUE;
 			break;
+		case 'd':
+			enable_all_pool_feat = B_FALSE;
+			break;
 		case 'R':
 			altroot = optarg;
 			if (add_prop_list(zpool_prop_to_name(
@@ -670,6 +687,21 @@ zpool_do_create(int argc, char **argv)
 
 			if (add_prop_list(optarg, propval, &props, B_TRUE))
 				goto errout;
+
+			/*
+			 * If the user is creating a pool that doesn't support
+			 * feature flags, don't enable any features.
+			 */
+			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
+				char *end;
+				u_longlong_t ver;
+
+				ver = strtoull(propval, &end, 10);
+				if (*end == '\0' &&
+				    ver < SPA_VERSION_FEATURES) {
+					enable_all_pool_feat = B_FALSE;
+				}
+			}
 			break;
 		case 'O':
 			if ((propval = strchr(optarg, '=')) == NULL) {
@@ -735,7 +767,6 @@ zpool_do_create(int argc, char **argv)
 		goto errout;
 	}
 
-
 	if (altroot != NULL && altroot[0] != '/') {
 		(void) fprintf(stderr, gettext("invalid alternate root '%s': "
 		    "must be an absolute path\n"), altroot);
@@ -817,6 +848,27 @@ zpool_do_create(int argc, char **argv)
 		/*
 		 * Hand off to libzfs.
 		 */
+		if (enable_all_pool_feat) {
+			int i;
+			for (i = 0; i < SPA_FEATURES; i++) {
+				char propname[MAXPATHLEN];
+				zfeature_info_t *feat = &spa_feature_table[i];
+
+				(void) snprintf(propname, sizeof (propname),
+				    "feature@%s", feat->fi_uname);
+
+				/*
+				 * Skip feature if user specified it manually
+				 * on the command line.
+				 */
+				if (nvlist_exists(props, propname))
+					continue;
+
+				if (add_prop_list(propname, ZFS_FEATURE_ENABLED,
+				    &props, B_TRUE) != 0)
+					goto errout;
+			}
+		}
 		if (zpool_create(g_zfs, poolname,
 		    nvroot, props, fsprops) == 0) {
 			zfs_handle_t *pool = zfs_open(g_zfs, poolname,
@@ -1148,6 +1200,10 @@ print_status_config(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
 			(void) printf(gettext("newer version"));
 			break;
 
+		case VDEV_AUX_UNSUP_FEAT:
+			(void) printf(gettext("unsupported feature(s)"));
+			break;
+
 		case VDEV_AUX_SPARED:
 			verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
 			    &cb.cb_guid) == 0);
@@ -1265,6 +1321,10 @@ print_import_config(const char *name, nvlist_t *nv, int namewidth, int depth)
 			(void) printf(gettext("newer version"));
 			break;
 
+		case VDEV_AUX_UNSUP_FEAT:
+			(void) printf(gettext("unsupported feature(s)"));
+			break;
+
 		case VDEV_AUX_ERR_EXCEEDED:
 			(void) printf(gettext("too many errors"));
 			break;
@@ -1431,6 +1491,20 @@ show_import(nvlist_t *config)
 		    "incompatible version.\n"));
 		break;
 
+	case ZPOOL_STATUS_UNSUP_FEAT_READ:
+		(void) printf(gettext("status: The pool uses the following "
+		    "feature(s) not supported on this sytem:\n"));
+		zpool_print_unsup_feat(config);
+		break;
+
+	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
+		(void) printf(gettext("status: The pool can only be accessed "
+		    "in read-only mode on this system. It\n\tcannot be "
+		    "accessed in read-write mode because it uses the "
+		    "following\n\tfeature(s) not supported on this system:\n"));
+		zpool_print_unsup_feat(config);
+		break;
+
 	case ZPOOL_STATUS_HOSTID_MISMATCH:
 		(void) printf(gettext(" status: The pool was last accessed by "
 		    "another system.\n"));
@@ -1488,6 +1562,20 @@ show_import(nvlist_t *config)
 			    "newer\n\tsoftware, or recreate the pool from "
 			    "backup.\n"));
 			break;
+		case ZPOOL_STATUS_UNSUP_FEAT_READ:
+			(void) printf(gettext("action: The pool cannot be "
+			    "imported. Access the pool on a system that "
+			    "supports\n\tthe required feature(s), or recreate "
+			    "the pool from backup.\n"));
+			break;
+		case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
+			(void) printf(gettext("action: The pool cannot be "
+			    "imported in read-write mode. Import the pool "
+			    "with\n"
+			    "\t\"-o readonly=on\", access the pool on a system "
+			    "that supports the\n\trequired feature(s), or "
+			    "recreate the pool from backup.\n"));
+			break;
 		case ZPOOL_STATUS_MISSING_DEV_R:
 		case ZPOOL_STATUS_MISSING_DEV_NR:
 		case ZPOOL_STATUS_BAD_GUID_SUM:
@@ -1563,9 +1651,9 @@ do_import(nvlist_t *config, const char *newname, const char *mntopts,
 	    ZPOOL_CONFIG_POOL_STATE, &state) == 0);
 	verify(nvlist_lookup_uint64(config,
 	    ZPOOL_CONFIG_VERSION, &version) == 0);
-	if (version > SPA_VERSION) {
+	if (!SPA_VERSION_IS_SUPPORTED(version)) {
 		(void) fprintf(stderr, gettext("cannot import '%s': pool "
-		    "is formatted using a newer ZFS version\n"), name);
+		    "is formatted using an unsupported ZFS version\n"), name);
 		return (1);
 	} else if (state != POOL_STATE_EXPORTED &&
 	    !(flags & ZFS_IMPORT_ANY_HOST)) {
@@ -2556,15 +2644,13 @@ static void
 print_header(list_cbdata_t *cb)
 {
 	zprop_list_t *pl = cb->cb_proplist;
+	char headerbuf[ZPOOL_MAXPROPLEN];
 	const char *header;
 	boolean_t first = B_TRUE;
 	boolean_t right_justify;
 	size_t width = 0;
 
 	for (; pl != NULL; pl = pl->pl_next) {
-		if (pl->pl_prop == ZPROP_INVAL)
-			continue;
-
 		width = pl->pl_width;
 		if (first && cb->cb_verbose) {
 			/*
@@ -2579,8 +2665,18 @@ print_header(list_cbdata_t *cb)
 		else
 			first = B_FALSE;
 
-		header = zpool_prop_column_name(pl->pl_prop);
-		right_justify = zpool_prop_align_right(pl->pl_prop);
+		right_justify = B_FALSE;
+		if (pl->pl_prop != ZPROP_INVAL) {
+			header = zpool_prop_column_name(pl->pl_prop);
+			right_justify = zpool_prop_align_right(pl->pl_prop);
+		} else {
+			int i;
+
+			for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
+				headerbuf[i] = toupper(pl->pl_user_prop[i]);
+			headerbuf[i] = '\0';
+			header = headerbuf;
+		}
 
 		if (pl->pl_next == NULL && !right_justify)
 			(void) printf("%s", header);
@@ -2639,6 +2735,11 @@ print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
 				propstr = property;
 
 			right_justify = zpool_prop_align_right(pl->pl_prop);
+		} else if ((zpool_prop_feature(pl->pl_user_prop) ||
+		    zpool_prop_unsupported(pl->pl_user_prop)) &&
+		    zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
+		    sizeof (property)) == 0) {
+			propstr = property;
 		} else {
 			propstr = "-";
 		}
@@ -3958,6 +4059,31 @@ status_callback(zpool_handle_t *zhp, void *data)
 		    "backup.\n"));
 		break;
 
+	case ZPOOL_STATUS_UNSUP_FEAT_READ:
+		(void) printf(gettext("status: The pool cannot be accessed on "
+		    "this system because it uses the\n\tfollowing feature(s) "
+		    "not supported on this system:\n"));
+		zpool_print_unsup_feat(config);
+		(void) printf("\n");
+		(void) printf(gettext("action: Access the pool from a system "
+		    "that supports the required feature(s),\n\tor restore the "
+		    "pool from backup.\n"));
+		break;
+
+	case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
+		(void) printf(gettext("status: The pool can only be accessed "
+		    "in read-only mode on this system. It\n\tcannot be "
+		    "accessed in read-write mode because it uses the "
+		    "following\n\tfeature(s) not supported on this system:\n"));
+		zpool_print_unsup_feat(config);
+		(void) printf("\n");
+		(void) printf(gettext("action: The pool cannot be accessed in "
+		    "read-write mode. Import the pool with\n"
+		    "\t\"-o readonly=on\", access the pool from a system that "
+		    "supports the\n\trequired feature(s), or restore the "
+		    "pool from backup.\n"));
+		break;
+
 	case ZPOOL_STATUS_FAULTED_DEV_R:
 		(void) printf(gettext("status: One or more devices are "
 		    "faulted in response to persistent errors.\n\tSufficient "
@@ -4182,7 +4308,8 @@ upgrade_cb(zpool_handle_t *zhp, void *arg)
 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
 	    &version) == 0);
 
-	if (!cbp->cb_newer && version < SPA_VERSION) {
+	if (!cbp->cb_newer && SPA_VERSION_IS_SUPPORTED(version) &&
+	    version != SPA_VERSION) {
 		if (!cbp->cb_all) {
 			if (cbp->cb_first) {
 				(void) printf(gettext("The following pools are "
@@ -4205,13 +4332,14 @@ upgrade_cb(zpool_handle_t *zhp, void *arg)
 				    "'%s'\n\n"), zpool_get_name(zhp));
 			}
 		}
-	} else if (cbp->cb_newer && version > SPA_VERSION) {
+	} else if (cbp->cb_newer && !SPA_VERSION_IS_SUPPORTED(version)) {
 		assert(!cbp->cb_all);
 
 		if (cbp->cb_first) {
 			(void) printf(gettext("The following pools are "
-			    "formatted using a newer software version and\n"
-			    "cannot be accessed on the current system.\n\n"));
+			    "formatted using an unsupported software version "
+			    "and\ncannot be accessed on the current "
+			    "system.\n\n"));
 			(void) printf(gettext("VER  POOL\n"));
 			(void) printf(gettext("---  ------------\n"));
 			cbp->cb_first = B_FALSE;
@@ -4295,8 +4423,8 @@ zpool_do_upgrade(int argc, char **argv)
 			break;
 		case 'V':
 			cb.cb_version = strtoll(optarg, &end, 10);
-			if (*end != '\0' || cb.cb_version > SPA_VERSION ||
-			    cb.cb_version < SPA_VERSION_1) {
+			if (*end != '\0' ||
+			    !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
 				(void) fprintf(stderr,
 				    gettext("invalid version '%s'\n"), optarg);
 				usage(B_FALSE);
@@ -4341,8 +4469,8 @@ zpool_do_upgrade(int argc, char **argv)
 		}
 	}
 
-	(void) printf(gettext("This system is currently running "
-	    "ZFS pool version %llu.\n\n"), SPA_VERSION);
+	(void) printf(gettext("This system supports ZFS pool feature "
+	    "flags.\n\n"));
 	cb.cb_first = B_TRUE;
 	if (showversions) {
 		(void) printf(gettext("The following versions are "
@@ -4923,13 +5051,26 @@ get_callback(zpool_handle_t *zhp, void *data)
 		    pl == cbp->cb_proplist)
 			continue;
 
-		if (zpool_get_prop(zhp, pl->pl_prop,
-		    value, sizeof (value), &srctype) != 0)
-			continue;
+		if (pl->pl_prop == ZPROP_INVAL &&
+		    (zpool_prop_feature(pl->pl_user_prop) ||
+		    zpool_prop_unsupported(pl->pl_user_prop))) {
+			srctype = ZPROP_SRC_LOCAL;
+
+			if (zpool_prop_get_feature(zhp, pl->pl_user_prop,
+			    value, sizeof (value)) == 0) {
+				zprop_print_one_property(zpool_get_name(zhp),
+				    cbp, pl->pl_user_prop, value, srctype,
+				    NULL, NULL);
+			}
+		} else {
+			if (zpool_get_prop(zhp, pl->pl_prop, value,
+			    sizeof (value), &srctype) != 0)
+				continue;
 
-		zprop_print_one_property(zpool_get_name(zhp), cbp,
-		    zpool_prop_to_name(pl->pl_prop), value, srctype, NULL,
-		    NULL);
+			zprop_print_one_property(zpool_get_name(zhp), cbp,
+			    zpool_prop_to_name(pl->pl_prop), value, srctype,
+			    NULL, NULL);
+		}
 	}
 	return (0);
 }
@@ -4941,8 +5082,11 @@ zpool_do_get(int argc, char **argv)
 	zprop_list_t fake_name = { 0 };
 	int ret;
 
-	if (argc < 3)
+	if (argc < 2) {
+		(void) fprintf(stderr, gettext("missing property "
+		    "argument\n"));
 		usage(B_FALSE);
+	}
 
 	cb.cb_first = B_TRUE;
 	cb.cb_sources = ZPROP_SRC_ALL;
@@ -4952,7 +5096,7 @@ zpool_do_get(int argc, char **argv)
 	cb.cb_columns[3] = GET_COL_SOURCE;
 	cb.cb_type = ZFS_TYPE_POOL;
 
-	if (zprop_get_list(g_zfs, argv[1],  &cb.cb_proplist,
+	if (zprop_get_list(g_zfs, argv[1], &cb.cb_proplist,
 	    ZFS_TYPE_POOL) != 0)
 		usage(B_FALSE);
 
diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c
index 7e941b5cb..cc2db31e4 100644
--- a/cmd/ztest/ztest.c
+++ b/cmd/ztest/ztest.c
@@ -109,6 +109,7 @@
 #include <sys/dsl_scan.h>
 #include <sys/zio_checksum.h>
 #include <sys/refcount.h>
+#include <sys/zfeature.h>
 #include <stdio.h>
 #include <stdio_ext.h>
 #include <stdlib.h>
@@ -5771,10 +5772,9 @@ make_random_props(void)
 {
 	nvlist_t *props;
 
-	if (ztest_random(2) == 0)
-		return (NULL);
-
 	VERIFY(nvlist_alloc(&props, NV_UNIQUE_NAME, 0) == 0);
+	if (ztest_random(2) == 0)
+		return (props);
 	VERIFY(nvlist_add_uint64(props, "autoreplace", 1) == 0);
 
 	return (props);
@@ -5789,6 +5789,7 @@ ztest_init(ztest_shared_t *zs)
 {
 	spa_t *spa;
 	nvlist_t *nvroot, *props;
+	int i;
 
 	mutex_init(&ztest_vdev_lock, NULL, MUTEX_DEFAULT, NULL);
 	rw_init(&ztest_name_lock, NULL, RW_DEFAULT, NULL);
@@ -5805,6 +5806,13 @@ ztest_init(ztest_shared_t *zs)
 	nvroot = make_vdev_root(NULL, NULL, ztest_opts.zo_vdev_size, 0,
 	    0, ztest_opts.zo_raidz, zs->zs_mirrors, 1);
 	props = make_random_props();
+	for (i = 0; i < SPA_FEATURES; i++) {
+		char *buf;
+		VERIFY3S(-1, !=, asprintf(&buf, "feature@%s",
+		    spa_feature_table[i].fi_uname));
+		VERIFY3U(0, ==, nvlist_add_uint64(props, buf, 0));
+		free(buf);
+	}
 	VERIFY3U(0, ==, spa_create(ztest_opts.zo_pool, nvroot, props,
 	    NULL, NULL));
 	nvlist_free(nvroot);