Add "compatibility" property for zpool feature sets

Property to allow sets of features to be specified; for compatibility
with specific versions / releases / external systems. Influences
the behavior of 'zpool upgrade' and 'zpool create'. Initial man
page changes and test cases included.

Brief synopsis:

zpool create -o compatibility=off|legacy|file[,file...] pool vdev...

compatibility = off : disable compatibility mode (enable all features)
compatibility = legacy : request that no features be enabled
compatibility = file[,file...] : read features from specified files.
Only features present in *all* files will be enabled on the
resulting pool. Filenames may be absolute, or relative to
/etc/zfs/compatibility.d or /usr/share/zfs/compatibility.d (/etc
checked first).

Only affects zpool create, zpool upgrade and zpool status.

ABI changes in libzfs:

* New function "zpool_load_compat" to load and parse compat sets.
* Add "zpool_compat_status_t" typedef for compatibility parse status.
* Add ZPOOL_PROP_COMPATIBILITY to the pool properties enum
* Add ZPOOL_STATUS_COMPATIBILITY_ERR to the pool status enum

An initial set of base compatibility sets are included in
cmd/zpool/compatibility.d, and the Makefile for cmd/zpool is
modified to install these in $pkgdatadir/compatibility.d and to
create symbolic links to a reasonable set of aliases.

Reviewed-by: ericloewe
Reviewed-by: Matthew Ahrens <[email protected]>
Reviewed-by: Richard Laager <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Colm Buckley <[email protected]>
Closes #11468 

19 files changed, 504 insertions, 26 deletions

diff --git a/cmd/zpool/Makefile.am b/cmd/zpool/Makefile.am
index fb03e99a3..d47051e4f 100644
--- a/cmd/zpool/Makefile.am
+++ b/cmd/zpool/Makefile.am
@@ -39,7 +39,7 @@ include $(top_srcdir)/config/CppCheck.am
 zpoolconfdir = $(sysconfdir)/zfs/zpool.d
 zpoolexecdir = $(zfsexecdir)/zpool.d
 
-EXTRA_DIST = zpool.d/README
+EXTRA_DIST = zpool.d/README compatibility.d
 
 dist_zpoolexec_SCRIPTS = \
 	zpool.d/dm-deps \
@@ -129,6 +129,48 @@ zpoolconfdefaults = \
 	test_progress \
 	test_ended
 
+zpoolcompatdir = $(pkgdatadir)/compatibility.d
+
+dist_zpoolcompat_DATA = \
+	compatibility.d/compat-2018 \
+	compatibility.d/compat-2019 \
+	compatibility.d/compat-2020 \
+	compatibility.d/compat-2021 \
+	compatibility.d/freebsd-11.0 \
+	compatibility.d/freebsd-11.2 \
+	compatibility.d/freebsd-11.3 \
+	compatibility.d/freenas-9.10.2 \
+	compatibility.d/grub2 \
+	compatibility.d/openzfsonosx-1.7.0 \
+	compatibility.d/openzfsonosx-1.8.1 \
+	compatibility.d/openzfsonosx-1.9.3 \
+	compatibility.d/openzfs-2.0-freebsd \
+	compatibility.d/openzfs-2.0-linux \
+	compatibility.d/zol-0.6.5 \
+	compatibility.d/zol-0.7 \
+	compatibility.d/zol-0.8
+
+# canonical <- alias symbolic link pairs
+# eg: "2018" is a link to "compat-2018"
+zpoolcompatlinks = \
+	"compat-2018		2018" \
+	"compat-2019		2019" \
+	"compat-2020		2020" \
+	"compat-2021		2021" \
+	"freebsd-11.0		freebsd-11.1" \
+	"freebsd-11.0		freenas-11.0" \
+	"freebsd-11.2		freenas-11.2" \
+	"freebsd-11.3		freebsd-11.4" \
+	"freebsd-11.3		freebsd-12.0" \
+	"freebsd-11.3		freebsd-12.1" \
+	"freebsd-11.3		freebsd-12.2" \
+	"freebsd-11.3		freenas-11.3" \
+	"freenas-11.0		freenas-11.1" \
+	"openzfsonosx-1.9.3	openzfsonosx-1.9.4" \
+	"openzfs-2.0-freebsd	truenas-12.0" \
+	"zol-0.7		ubuntu-18.04" \
+	"zol-0.8		ubuntu-20.04"
+
 install-data-hook:
 	$(MKDIR_P) "$(DESTDIR)$(zpoolconfdir)"
 	for f in $(zpoolconfdefaults); do \
@@ -136,3 +178,6 @@ install-data-hook:
 	       -L "$(DESTDIR)$(zpoolconfdir)/$${f}" || \
 	    ln -s "$(zpoolexecdir)/$${f}" "$(DESTDIR)$(zpoolconfdir)"; \
 	done
+	for l in $(zpoolcompatlinks); do \
+		(cd "$(DESTDIR)$(zpoolcompatdir)"; ln -s $${l} ); \
+	done
diff --git a/cmd/zpool/compatibility.d/compat-2018 b/cmd/zpool/compatibility.d/compat-2018
new file mode 100644
index 000000000..7be44e1ee
--- /dev/null
+++ b/cmd/zpool/compatibility.d/compat-2018
@@ -0,0 +1,12 @@
+# Features supported by all Tier 1 platforms as of 2018
+async_destroy
+bookmarks
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+spacemap_histogram
diff --git a/cmd/zpool/compatibility.d/compat-2019 b/cmd/zpool/compatibility.d/compat-2019
new file mode 100644
index 000000000..c105cc70c
--- /dev/null
+++ b/cmd/zpool/compatibility.d/compat-2019
@@ -0,0 +1,15 @@
+# Features supported by all Tier 1 platforms as of 2019
+async_destroy
+bookmarks
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+multi_vdev_crash_dump
+sha512
+skein
+spacemap_histogram
diff --git a/cmd/zpool/compatibility.d/compat-2020 b/cmd/zpool/compatibility.d/compat-2020
new file mode 100644
index 000000000..8d46a571e
--- /dev/null
+++ b/cmd/zpool/compatibility.d/compat-2020
@@ -0,0 +1,15 @@
+# Features supported by all Tier 1 platforms as of 2020
+async_destroy
+bookmarks
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+multi_vdev_crash_dump
+sha512
+skein
+spacemap_histogram
diff --git a/cmd/zpool/compatibility.d/compat-2021 b/cmd/zpool/compatibility.d/compat-2021
new file mode 100644
index 000000000..f45c82d65
--- /dev/null
+++ b/cmd/zpool/compatibility.d/compat-2021
@@ -0,0 +1,19 @@
+# Features supported by all Tier 1 platforms as of 2021
+async_destroy
+bookmarks
+device_removal
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+zpool_checkpoint
diff --git a/cmd/zpool/compatibility.d/freebsd-11.0 b/cmd/zpool/compatibility.d/freebsd-11.0
new file mode 100644
index 000000000..8718559ff
--- /dev/null
+++ b/cmd/zpool/compatibility.d/freebsd-11.0
@@ -0,0 +1,15 @@
+# Features supported by FreeBSD 11.0
+async_destroy
+bookmarks
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+multi_vdev_crash_dump
+sha512
+skein
+spacemap_histogram
diff --git a/cmd/zpool/compatibility.d/freebsd-11.2 b/cmd/zpool/compatibility.d/freebsd-11.2
new file mode 100644
index 000000000..14d2d573b
--- /dev/null
+++ b/cmd/zpool/compatibility.d/freebsd-11.2
@@ -0,0 +1,18 @@
+# Features supported by FreeBSD 11.2
+async_destroy
+bookmarks
+device_removal
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+sha512
+skein
+spacemap_histogram
+zpool_checkpoint
diff --git a/cmd/zpool/compatibility.d/freebsd-11.3 b/cmd/zpool/compatibility.d/freebsd-11.3
new file mode 100644
index 000000000..802cc3630
--- /dev/null
+++ b/cmd/zpool/compatibility.d/freebsd-11.3
@@ -0,0 +1,19 @@
+# Features supported by FreeBSD 11.3
+async_destroy
+bookmarks
+device_removal
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+zpool_checkpoint
diff --git a/cmd/zpool/compatibility.d/freenas-9.10.2 b/cmd/zpool/compatibility.d/freenas-9.10.2
new file mode 100644
index 000000000..10789c96c
--- /dev/null
+++ b/cmd/zpool/compatibility.d/freenas-9.10.2
@@ -0,0 +1,13 @@
+# Features supported by FreeNAS 9.10.2
+async_destroy
+bookmarks
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+multi_vdev_crash_dump
+spacemap_histogram
diff --git a/cmd/zpool/compatibility.d/grub2 b/cmd/zpool/compatibility.d/grub2
new file mode 100644
index 000000000..4e8f21362
--- /dev/null
+++ b/cmd/zpool/compatibility.d/grub2
@@ -0,0 +1,12 @@
+# Features which are supported by GRUB2
+async_destroy
+bookmarks
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+spacemap_histogram
diff --git a/cmd/zpool/compatibility.d/openzfs-2.0-freebsd b/cmd/zpool/compatibility.d/openzfs-2.0-freebsd
new file mode 100644
index 000000000..e7ee2f247
--- /dev/null
+++ b/cmd/zpool/compatibility.d/openzfs-2.0-freebsd
@@ -0,0 +1,33 @@
+# Features supported by OpenZFS 2.0 on FreeBSD
+allocation_classes
+async_destroy
+bookmark_v2
+bookmark_written
+bookmarks
+device_rebuild
+device_removal
+embedded_data
+empty_bpobj
+enabled_txg
+encryption
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+large_dnode
+livelist
+log_spacemap
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+project_quota
+redacted_datasets
+redaction_bookmarks
+resilver_defer
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+userobj_accounting
+zpool_checkpoint
+zstd_compress
diff --git a/cmd/zpool/compatibility.d/openzfs-2.0-linux b/cmd/zpool/compatibility.d/openzfs-2.0-linux
new file mode 100644
index 000000000..ac0f5c863
--- /dev/null
+++ b/cmd/zpool/compatibility.d/openzfs-2.0-linux
@@ -0,0 +1,34 @@
+# Features supported by OpenZFS 2.0 on Linux
+allocation_classes
+async_destroy
+bookmark_v2
+bookmark_written
+bookmarks
+device_rebuild
+device_removal
+edonr
+embedded_data
+empty_bpobj
+enabled_txg
+encryption
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+large_dnode
+livelist
+log_spacemap
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+project_quota
+redacted_datasets
+redaction_bookmarks
+resilver_defer
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+userobj_accounting
+zpool_checkpoint
+zstd_compress
diff --git a/cmd/zpool/compatibility.d/openzfsonosx-1.7.0 b/cmd/zpool/compatibility.d/openzfsonosx-1.7.0
new file mode 100644
index 000000000..4ae87c964
--- /dev/null
+++ b/cmd/zpool/compatibility.d/openzfsonosx-1.7.0
@@ -0,0 +1,16 @@
+# Features supported by OpenZFSonOSX 1.7.0
+async_destroy
+bookmarks
+edonr
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+multi_vdev_crash_dump
+sha512
+skein
+spacemap_histogram
diff --git a/cmd/zpool/compatibility.d/openzfsonosx-1.8.1 b/cmd/zpool/compatibility.d/openzfsonosx-1.8.1
new file mode 100644
index 000000000..162ff32a7
--- /dev/null
+++ b/cmd/zpool/compatibility.d/openzfsonosx-1.8.1
@@ -0,0 +1,21 @@
+# Features supported by OpenZFSonOSX 1.8.1
+async_destroy
+bookmarks
+device_removal
+edonr
+embedded_data
+empty_bpobj
+enabled_txg
+encryption
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+zpool_checkpoint
diff --git a/cmd/zpool/compatibility.d/openzfsonosx-1.9.3 b/cmd/zpool/compatibility.d/openzfsonosx-1.9.3
new file mode 100644
index 000000000..b0b28ec04
--- /dev/null
+++ b/cmd/zpool/compatibility.d/openzfsonosx-1.9.3
@@ -0,0 +1,27 @@
+# Features supported by OpenZFSonOSX 1.9.3
+allocation_classes
+async_destroy
+bookmark_v2
+bookmarks
+device_removal
+edonr
+embedded_data
+empty_bpobj
+enabled_txg
+encryption
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+large_dnode
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+project_quota
+resilver_defer
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+userobj_accounting
+zpool_checkpoint
diff --git a/cmd/zpool/compatibility.d/zol-0.6.5 b/cmd/zpool/compatibility.d/zol-0.6.5
new file mode 100644
index 000000000..cb9a94d88
--- /dev/null
+++ b/cmd/zpool/compatibility.d/zol-0.6.5
@@ -0,0 +1,12 @@
+# Features supported by ZFSonLinux v0.6.5
+async_destroy
+bookmarks
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+lz4_compress
+spacemap_histogram
diff --git a/cmd/zpool/compatibility.d/zol-0.7 b/cmd/zpool/compatibility.d/zol-0.7
new file mode 100644
index 000000000..22a02936d
--- /dev/null
+++ b/cmd/zpool/compatibility.d/zol-0.7
@@ -0,0 +1,18 @@
+# Features supported by ZFSonLinux v0.7
+async_destroy
+bookmarks
+edonr
+embedded_data
+empty_bpobj
+enabled_txg
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+large_dnode
+lz4_compress
+multi_vdev_crash_dump
+sha512
+skein
+spacemap_histogram
+userobj_accounting
diff --git a/cmd/zpool/compatibility.d/zol-0.8 b/cmd/zpool/compatibility.d/zol-0.8
new file mode 100644
index 000000000..762848ef7
--- /dev/null
+++ b/cmd/zpool/compatibility.d/zol-0.8
@@ -0,0 +1,27 @@
+# Features supported by ZFSonLinux v0.8
+allocation_classes
+async_destroy
+bookmark_v2
+bookmarks
+device_removal
+edonr
+embedded_data
+empty_bpobj
+enabled_txg
+encryption
+extensible_dataset
+filesystem_limits
+hole_birth
+large_blocks
+large_dnode
+lz4_compress
+multi_vdev_crash_dump
+obsolete_counts
+project_quota
+resilver_defer
+sha512
+skein
+spacemap_histogram
+spacemap_v2
+userobj_accounting
+zpool_checkpoint
diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c
index eef12d97f..29252e6a2 100644
--- a/cmd/zpool/zpool_main.c
+++ b/cmd/zpool/zpool_main.c
@@ -31,6 +31,7 @@
  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  * Copyright (c) 2017, Intel Corporation.
  * Copyright (c) 2019, loli10K <[email protected]>
+ * Copyright (c) 2021, Colm Buckley <[email protected]>
  */
 
 #include <assert.h>
@@ -124,6 +125,9 @@ static int zpool_do_version(int, char **);
 
 static int zpool_do_wait(int, char **);
 
+static zpool_compat_status_t zpool_do_load_compat(
+    const char *, boolean_t *);
+
 /*
  * These libumem hooks provide a reasonable set of defaults for the allocator's
  * debugging facilities.
@@ -782,6 +786,8 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
 
 	if (poolprop) {
 		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
+		const char *fname =
+		    zpool_prop_to_name(ZPOOL_PROP_COMPATIBILITY);
 
 		if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
 		    !zpool_prop_feature(propname)) {
@@ -804,6 +810,19 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
 			return (2);
 		}
 
+		/*
+		 * compatibility property and version should not be specified
+		 * at the same time.
+		 */
+		if ((prop == ZPOOL_PROP_COMPATIBILITY &&
+		    nvlist_exists(proplist, vname)) ||
+		    (prop == ZPOOL_PROP_VERSION &&
+		    nvlist_exists(proplist, fname))) {
+			(void) fprintf(stderr, gettext("'compatibility' and "
+			    "'version' properties cannot be specified "
+			    "together\n"));
+			return (2);
+		}
 
 		if (zpool_prop_feature(propname))
 			normnm = propname;
@@ -1374,13 +1393,15 @@ zpool_do_create(int argc, char **argv)
 {
 	boolean_t force = B_FALSE;
 	boolean_t dryrun = B_FALSE;
-	boolean_t enable_all_pool_feat = B_TRUE;
+	boolean_t enable_pool_features = B_TRUE;
+
 	int c;
 	nvlist_t *nvroot = NULL;
 	char *poolname;
 	char *tname = NULL;
 	int ret = 1;
 	char *altroot = NULL;
+	char *compat = NULL;
 	char *mountpoint = NULL;
 	nvlist_t *fsprops = NULL;
 	nvlist_t *props = NULL;
@@ -1396,7 +1417,7 @@ zpool_do_create(int argc, char **argv)
 			dryrun = B_TRUE;
 			break;
 		case 'd':
-			enable_all_pool_feat = B_FALSE;
+			enable_pool_features = B_FALSE;
 			break;
 		case 'R':
 			altroot = optarg;
@@ -1434,11 +1455,14 @@ zpool_do_create(int argc, char **argv)
 				ver = strtoull(propval, &end, 10);
 				if (*end == '\0' &&
 				    ver < SPA_VERSION_FEATURES) {
-					enable_all_pool_feat = B_FALSE;
+					enable_pool_features = B_FALSE;
 				}
 			}
 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
 				altroot = propval;
+			if (zpool_name_to_prop(optarg) ==
+			    ZPOOL_PROP_COMPATIBILITY)
+				compat = propval;
 			break;
 		case 'O':
 			if ((propval = strchr(optarg, '=')) == NULL) {
@@ -1632,10 +1656,26 @@ zpool_do_create(int argc, char **argv)
 		ret = 0;
 	} else {
 		/*
-		 * Hand off to libzfs.
+		 * Load in feature set.
+		 * Note: if compatibility property not given, we'll have
+		 * NULL, which means 'all features'.
 		 */
-		spa_feature_t i;
-		for (i = 0; i < SPA_FEATURES; i++) {
+		boolean_t requested_features[SPA_FEATURES];
+		if (zpool_do_load_compat(compat, requested_features) !=
+		    ZPOOL_COMPATIBILITY_OK)
+			goto errout;
+
+		/*
+		 * props contains list of features to enable.
+		 * For each feature:
+		 *  - remove it if feature@name=disabled
+		 *  - leave it there if feature@name=enabled
+		 *  - add it if:
+		 *    - enable_pool_features (ie: no '-d' or '-o version')
+		 *    - it's supported by the kernel module
+		 *    - it's in the requested feature set
+		 */
+		for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
 			char propname[MAXPATHLEN];
 			char *propval;
 			zfeature_info_t *feat = &spa_feature_table[i];
@@ -1643,18 +1683,14 @@ zpool_do_create(int argc, char **argv)
 			(void) snprintf(propname, sizeof (propname),
 			    "feature@%s", feat->fi_uname);
 
-			/*
-			 * Only features contained in props will be enabled:
-			 * remove from the nvlist every ZFS_FEATURE_DISABLED
-			 * value and add every missing ZFS_FEATURE_ENABLED if
-			 * enable_all_pool_feat is set.
-			 */
 			if (!nvlist_lookup_string(props, propname, &propval)) {
 				if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
 					(void) nvlist_remove_all(props,
 					    propname);
-			} else if (enable_all_pool_feat &&
-			    feat->fi_zfs_mod_supported) {
+			} else if (
+			    enable_pool_features &&
+			    feat->fi_zfs_mod_supported &&
+			    requested_features[i]) {
 				ret = add_prop_list(propname,
 				    ZFS_FEATURE_ENABLED, &props, B_TRUE);
 				if (ret != 0)
@@ -2674,8 +2710,15 @@ show_import(nvlist_t *config)
 
 	case ZPOOL_STATUS_FEAT_DISABLED:
 		printf_color(ANSI_BOLD, gettext("status: "));
-		printf_color(ANSI_YELLOW, gettext("Some supported features are "
-		    "not enabled on the pool.\n"));
+		printf_color(ANSI_YELLOW, gettext("Some supported and "
+		    "requested features are not enabled on the pool.\n"));
+		break;
+
+	case ZPOOL_STATUS_COMPATIBILITY_ERR:
+		printf_color(ANSI_BOLD, gettext("status: "));
+		printf_color(ANSI_YELLOW, gettext("Error reading or parsing "
+		    "the file(s) indicated by the 'compatibility'\n"
+		    "property.\n"));
 		break;
 
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
@@ -2767,6 +2810,12 @@ show_import(nvlist_t *config)
 			    "imported using its name or numeric identifier, "
 			    "though\n\tsome features will not be available "
 			    "without an explicit 'zpool upgrade'.\n"));
+		} else if (reason == ZPOOL_STATUS_COMPATIBILITY_ERR) {
+			(void) printf(gettext(" action: The pool can be "
+			    "imported using its name or numeric\n\tidentifier, "
+			    "though the file(s) indicated by its "
+			    "'compatibility'\n\tproperty cannot be parsed at "
+			    "this time.\n"));
 		} else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
 			(void) printf(gettext(" action: The pool can be "
 			    "imported using its name or numeric "
@@ -7942,7 +7991,8 @@ status_callback(zpool_handle_t *zhp, void *data)
 	if (cbp->cb_explain &&
 	    (reason == ZPOOL_STATUS_OK ||
 	    reason == ZPOOL_STATUS_VERSION_OLDER ||
-	    reason == ZPOOL_STATUS_FEAT_DISABLED)) {
+	    reason == ZPOOL_STATUS_FEAT_DISABLED ||
+	    reason == ZPOOL_STATUS_COMPATIBILITY_ERR)) {
 		if (!cbp->cb_allpools) {
 			(void) printf(gettext("pool '%s' is healthy\n"),
 			    zpool_get_name(zhp));
@@ -8117,9 +8167,10 @@ status_callback(zpool_handle_t *zhp, void *data)
 
 	case ZPOOL_STATUS_FEAT_DISABLED:
 		printf_color(ANSI_BOLD, gettext("status: "));
-		printf_color(ANSI_YELLOW, gettext("Some supported features are "
-		    "not enabled on the pool. The pool can\n\tstill be used, "
-		    "but some features are unavailable.\n"));
+		printf_color(ANSI_YELLOW, gettext("Some supported and "
+		    "requested features are not enabled on the pool.\n\t"
+		    "The pool can still be used, but some features are "
+		    "unavailable.\n"));
 		printf_color(ANSI_BOLD, gettext("action: "));
 		printf_color(ANSI_YELLOW, gettext("Enable all features using "
 		    "'zpool upgrade'. Once this is done,\n\tthe pool may no "
@@ -8127,6 +8178,19 @@ status_callback(zpool_handle_t *zhp, void *data)
 		    "the features. See zpool-features(5) for details.\n"));
 		break;
 
+	case ZPOOL_STATUS_COMPATIBILITY_ERR:
+		printf_color(ANSI_BOLD, gettext("status: "));
+		printf_color(ANSI_YELLOW, gettext("This pool has a "
+		    "compatibility list specified, but it could not be\n\t"
+		    "read/parsed at this time. The pool can still be used, "
+		    "but this\n\tshould be investigated.\n"));
+		printf_color(ANSI_BOLD, gettext("action: "));
+		printf_color(ANSI_YELLOW, gettext("Check the value of the "
+		    "'compatibility' property against the\n\t"
+		    "appropriate file in " ZPOOL_SYSCONF_COMPAT_D " or "
+		    ZPOOL_DATA_COMPAT_D ".\n"));
+		break;
+
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
 		printf_color(ANSI_BOLD, gettext("status: "));
 		printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
@@ -8625,6 +8689,16 @@ upgrade_enable_all(zpool_handle_t *zhp, int *countp)
 	boolean_t firstff = B_TRUE;
 	nvlist_t *enabled = zpool_get_features(zhp);
 
+	char compat[ZFS_MAXPROPLEN];
+	if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
+	    ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
+		compat[0] = '\0';
+
+	boolean_t requested_features[SPA_FEATURES];
+	if (zpool_do_load_compat(compat, requested_features) !=
+	    ZPOOL_COMPATIBILITY_OK)
+		return (-1);
+
 	count = 0;
 	for (i = 0; i < SPA_FEATURES; i++) {
 		const char *fname = spa_feature_table[i].fi_uname;
@@ -8633,7 +8707,7 @@ upgrade_enable_all(zpool_handle_t *zhp, int *countp)
 		if (!spa_feature_table[i].fi_zfs_mod_supported)
 			continue;
 
-		if (!nvlist_exists(enabled, fguid)) {
+		if (!nvlist_exists(enabled, fguid) && requested_features[i]) {
 			char *propname;
 			verify(-1 != asprintf(&propname, "feature@%s", fname));
 			ret = zpool_set_prop(zhp, propname,
@@ -8855,7 +8929,7 @@ upgrade_one(zpool_handle_t *zhp, void *data)
 			printnl = B_TRUE;
 		} else if (cur_version == SPA_VERSION) {
 			(void) printf(gettext("Pool '%s' already has all "
-			    "supported features enabled.\n"),
+			    "supported and requested features enabled.\n"),
 			    zpool_get_name(zhp));
 		}
 	}
@@ -9016,8 +9090,8 @@ zpool_do_upgrade(int argc, char **argv)
 				(void) printf(gettext("All pools are already "
 				    "formatted using feature flags.\n\n"));
 				(void) printf(gettext("Every feature flags "
-				    "pool already has all supported features "
-				    "enabled.\n"));
+				    "pool already has all supported and "
+				    "requested features enabled.\n"));
 			} else {
 				(void) printf(gettext("All pools are already "
 				    "formatted with version %llu or higher.\n"),
@@ -9043,7 +9117,7 @@ zpool_do_upgrade(int argc, char **argv)
 
 		if (cb.cb_first) {
 			(void) printf(gettext("Every feature flags pool has "
-			    "all supported features enabled.\n"));
+			    "all supported and requested features enabled.\n"));
 		} else {
 			(void) printf(gettext("\n"));
 		}
@@ -10347,6 +10421,39 @@ zpool_do_version(int argc, char **argv)
 	return (0);
 }
 
+/*
+ * Do zpool_load_compat() and print error message on failure
+ */
+static zpool_compat_status_t
+zpool_do_load_compat(const char *compat, boolean_t *list)
+{
+	char badword[ZFS_MAXPROPLEN];
+	char badfile[MAXPATHLEN];
+	zpool_compat_status_t ret;
+
+	switch (ret = zpool_load_compat(compat, list, badword, badfile)) {
+	case ZPOOL_COMPATIBILITY_OK:
+		break;
+	case ZPOOL_COMPATIBILITY_READERR:
+		(void) fprintf(stderr, gettext("error reading compatibility "
+		    "file '%s'\n"), badfile);
+		break;
+	case ZPOOL_COMPATIBILITY_BADFILE:
+		(void) fprintf(stderr, gettext("compatibility file '%s' "
+		    "too large or not newline-terminated\n"), badfile);
+		break;
+	case ZPOOL_COMPATIBILITY_BADWORD:
+		(void) fprintf(stderr, gettext("unknown feature '%s' in "
+		    "compatibility file '%s'\n"), badword, badfile);
+		break;
+	case ZPOOL_COMPATIBILITY_NOFILES:
+		(void) fprintf(stderr, gettext("no compatibility files "
+		    "specified\n"));
+		break;
+	}
+	return (ret);
+}
+
 int
 main(int argc, char **argv)
 {