Update to onnv_147

This is the last official OpenSolaris tag before the public
development tree was closed.

9 files changed, 1314 insertions, 398 deletions

diff --git a/lib/libzfs/include/libzfs.h b/lib/libzfs/include/libzfs.h
index 6f7fed62c..ea34cc9ef 100644
--- a/lib/libzfs/include/libzfs.h
+++ b/lib/libzfs/include/libzfs.h
@@ -103,7 +103,6 @@ enum {
 	EZFS_BADPERM,		/* invalid permission */
 	EZFS_BADPERMSET,	/* invalid permission set name */
 	EZFS_NODELEGATION,	/* delegated administration is disabled */
-	EZFS_PERMRDONLY,	/* pemissions are readonly */
 	EZFS_UNSHARESMBFAILED,	/* failed to unshare over smb */
 	EZFS_SHARESMBFAILED,	/* failed to share over smb */
 	EZFS_BADCACHE,		/* bad cache file */
@@ -120,6 +119,9 @@ enum {
 	EZFS_POSTSPLIT_ONLINE,	/* onlining a disk after splitting it */
 	EZFS_SCRUBBING,		/* currently scrubbing */
 	EZFS_NO_SCRUB,		/* no active scrub */
+	EZFS_DIFF,		/* general failure of zfs diff */
+	EZFS_DIFFDATA,		/* bad zfs diff data */
+	EZFS_POOLREADONLY,	/* pool is in read-only mode */
 	EZFS_UNKNOWN
 };
 
@@ -326,7 +328,7 @@ extern int zpool_export_force(zpool_handle_t *);
 extern int zpool_import(libzfs_handle_t *, nvlist_t *, const char *,
     char *altroot);
 extern int zpool_import_props(libzfs_handle_t *, nvlist_t *, const char *,
-    nvlist_t *, boolean_t);
+    nvlist_t *, int);
 
 /*
  * Search for pools to import
@@ -492,6 +494,17 @@ extern int zfs_iter_filesystems(zfs_handle_t *, zfs_iter_f, void *);
 extern int zfs_iter_snapshots(zfs_handle_t *, zfs_iter_f, void *);
 extern int zfs_iter_snapshots_sorted(zfs_handle_t *, zfs_iter_f, void *);
 
+typedef struct get_all_cb {
+	zfs_handle_t	**cb_handles;
+	size_t		cb_alloc;
+	size_t		cb_used;
+	boolean_t	cb_verbose;
+	int		(*cb_getone)(zfs_handle_t *, void *);
+} get_all_cb_t;
+
+void libzfs_add_handle(get_all_cb_t *, zfs_handle_t *);
+int libzfs_dataset_cmp(const void *, const void *);
+
 /*
  * Functions to create and destroy datasets.
  */
@@ -533,12 +546,8 @@ extern int zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 
 extern int zfs_promote(zfs_handle_t *);
 extern int zfs_hold(zfs_handle_t *, const char *, const char *, boolean_t,
-    boolean_t, boolean_t);
-extern int zfs_hold_range(zfs_handle_t *, const char *, const char *,
-    const char *, boolean_t, boolean_t, snapfilter_cb_t, void *);
+    boolean_t, boolean_t, int, uint64_t, uint64_t);
 extern int zfs_release(zfs_handle_t *, const char *, const char *, boolean_t);
-extern int zfs_release_range(zfs_handle_t *, const char *, const char *,
-    const char *, boolean_t);
 extern uint64_t zvol_volsize_to_reservation(uint64_t, nvlist_t *);
 
 typedef int (*zfs_userspace_cb_t)(void *arg, const char *domain,
@@ -579,6 +588,15 @@ typedef struct recvflags {
 extern int zfs_receive(libzfs_handle_t *, const char *, recvflags_t,
     int, avl_tree_t *);
 
+typedef enum diff_flags {
+	ZFS_DIFF_PARSEABLE = 0x1,
+	ZFS_DIFF_TIMESTAMP = 0x2,
+	ZFS_DIFF_CLASSIFY = 0x4
+} diff_flags_t;
+
+extern int zfs_show_diffs(zfs_handle_t *, int, const char *, const char *,
+    int);
+
 /*
  * Miscellaneous functions.
  */
diff --git a/lib/libzfs/include/libzfs_impl.h b/lib/libzfs/include/libzfs_impl.h
index 89c48c1c0..c9b09a205 100644
--- a/lib/libzfs/include/libzfs_impl.h
+++ b/lib/libzfs/include/libzfs_impl.h
@@ -20,8 +20,7 @@
  */
 
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef	_LIBFS_IMPL_H
@@ -69,6 +68,7 @@ struct libzfs_handle {
 	char libzfs_desc[1024];
 	char *libzfs_log_str;
 	int libzfs_printerr;
+	int libzfs_storeerr; /* stuff error messages into buffer */
 	void *libzfs_sharehdl; /* libshare handle */
 	uint_t libzfs_shareflags;
 	boolean_t libzfs_mnttab_enable;
@@ -136,6 +136,7 @@ int zfs_error_fmt(libzfs_handle_t *, int, const char *, ...);
 void zfs_error_aux(libzfs_handle_t *, const char *, ...);
 void *zfs_alloc(libzfs_handle_t *, size_t);
 void *zfs_realloc(libzfs_handle_t *, void *, size_t, size_t);
+char *zfs_asprintf(libzfs_handle_t *, const char *, ...);
 char *zfs_strdup(libzfs_handle_t *, const char *);
 int no_memory(libzfs_handle_t *);
 
@@ -188,6 +189,9 @@ int zpool_open_silent(libzfs_handle_t *, const char *, zpool_handle_t **);
 
 boolean_t zpool_name_valid(libzfs_handle_t *, boolean_t, const char *);
 
+int zfs_validate_name(libzfs_handle_t *hdl, const char *path, int type,
+    boolean_t modifying);
+
 void namespace_clear(libzfs_handle_t *);
 
 /*
diff --git a/lib/libzfs/libzfs_dataset.c b/lib/libzfs/libzfs_dataset.c
index a3f5a7d0f..b7c1360db 100644
--- a/lib/libzfs/libzfs_dataset.c
+++ b/lib/libzfs/libzfs_dataset.c
@@ -20,8 +20,7 @@
  */
 
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include <ctype.h>
@@ -126,7 +125,7 @@ path_to_str(const char *path, int types)
  * provide a more meaningful error message.  We call zfs_error_aux() to
  * explain exactly why the name was not valid.
  */
-static int
+int
 zfs_validate_name(libzfs_handle_t *hdl, const char *path, int type,
     boolean_t modifying)
 {
@@ -1212,39 +1211,46 @@ badlabel:
 		(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
 		goto error;
 	}
+	return (ret);
+
+error:
+	nvlist_free(ret);
+	return (NULL);
+}
+
+int
+zfs_add_synthetic_resv(zfs_handle_t *zhp, nvlist_t *nvl)
+{
+	uint64_t old_volsize;
+	uint64_t new_volsize;
+	uint64_t old_reservation;
+	uint64_t new_reservation;
+	zfs_prop_t resv_prop;
 
 	/*
 	 * If this is an existing volume, and someone is setting the volsize,
 	 * make sure that it matches the reservation, or add it if necessary.
 	 */
-	if (zhp != NULL && type == ZFS_TYPE_VOLUME &&
-	    nvlist_lookup_uint64(ret, zfs_prop_to_name(ZFS_PROP_VOLSIZE),
-	    &intval) == 0) {
-		uint64_t old_volsize = zfs_prop_get_int(zhp,
-		    ZFS_PROP_VOLSIZE);
-		uint64_t old_reservation;
-		uint64_t new_reservation;
-		zfs_prop_t resv_prop;
-
-		if (zfs_which_resv_prop(zhp, &resv_prop) < 0)
-			goto error;
-		old_reservation = zfs_prop_get_int(zhp, resv_prop);
-
-		if (old_volsize == old_reservation &&
-		    nvlist_lookup_uint64(ret, zfs_prop_to_name(resv_prop),
-		    &new_reservation) != 0) {
-			if (nvlist_add_uint64(ret,
-			    zfs_prop_to_name(resv_prop), intval) != 0) {
-				(void) no_memory(hdl);
-				goto error;
-			}
-		}
+	old_volsize = zfs_prop_get_int(zhp, ZFS_PROP_VOLSIZE);
+	if (zfs_which_resv_prop(zhp, &resv_prop) < 0)
+		return (-1);
+	old_reservation = zfs_prop_get_int(zhp, resv_prop);
+	if ((zvol_volsize_to_reservation(old_volsize, zhp->zfs_props) !=
+	    old_reservation) || nvlist_lookup_uint64(nvl,
+	    zfs_prop_to_name(resv_prop), &new_reservation) != ENOENT) {
+		return (0);
 	}
-	return (ret);
-
-error:
-	nvlist_free(ret);
-	return (NULL);
+	if (nvlist_lookup_uint64(nvl, zfs_prop_to_name(ZFS_PROP_VOLSIZE),
+	    &new_volsize) != 0)
+		return (-1);
+	new_reservation = zvol_volsize_to_reservation(new_volsize,
+	    zhp->zfs_props);
+	if (nvlist_add_uint64(nvl, zfs_prop_to_name(resv_prop),
+	    new_reservation) != 0) {
+		(void) no_memory(zhp->zfs_hdl);
+		return (-1);
+	}
+	return (1);
 }
 
 void
@@ -1346,6 +1352,7 @@ zfs_prop_set(zfs_handle_t *zhp, const char *propname, const char *propval)
 	zfs_prop_t prop;
 	boolean_t do_prefix;
 	uint64_t idx;
+	int added_resv;
 
 	(void) snprintf(errbuf, sizeof (errbuf),
 	    dgettext(TEXT_DOMAIN, "cannot set property for '%s'"),
@@ -1366,6 +1373,11 @@ zfs_prop_set(zfs_handle_t *zhp, const char *propname, const char *propval)
 
 	prop = zfs_name_to_prop(propname);
 
+	if (prop == ZFS_PROP_VOLSIZE) {
+		if ((added_resv = zfs_add_synthetic_resv(zhp, nvl)) == -1)
+			goto error;
+	}
+
 	if ((cl = changelist_gather(zhp, prop, 0, 0)) == NULL)
 		goto error;
 
@@ -1400,6 +1412,22 @@ zfs_prop_set(zfs_handle_t *zhp, const char *propname, const char *propval)
 
 	if (ret != 0) {
 		zfs_setprop_error(hdl, prop, errno, errbuf);
+		if (added_resv && errno == ENOSPC) {
+			/* clean up the volsize property we tried to set */
+			uint64_t old_volsize = zfs_prop_get_int(zhp,
+			    ZFS_PROP_VOLSIZE);
+			nvlist_free(nvl);
+			zcmd_free_nvlists(&zc);
+			if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
+				goto error;
+			if (nvlist_add_uint64(nvl,
+			    zfs_prop_to_name(ZFS_PROP_VOLSIZE),
+			    old_volsize) != 0)
+				goto error;
+			if (zcmd_write_src_nvlist(hdl, &zc, nvl) != 0)
+				goto error;
+			(void) zfs_ioctl(hdl, ZFS_IOC_SET_PROP, &zc);
+		}
 	} else {
 		if (do_prefix)
 			ret = changelist_postfix(cl);
@@ -1474,7 +1502,7 @@ zfs_prop_inherit(zfs_handle_t *zhp, const char *propname, boolean_t received)
 		return (zfs_error(hdl, EZFS_PROPTYPE, errbuf));
 
 	/*
-	 * Normalize the name, to get rid of shorthand abbrevations.
+	 * Normalize the name, to get rid of shorthand abbreviations.
 	 */
 	propname = zfs_prop_to_name(prop);
 	(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
@@ -2173,14 +2201,11 @@ static int
 idmap_id_to_numeric_domain_rid(uid_t id, boolean_t isuser,
     char **domainp, idmap_rid_t *ridp)
 {
-	idmap_handle_t *idmap_hdl = NULL;
 	idmap_get_handle_t *get_hdl = NULL;
 	idmap_stat status;
 	int err = EINVAL;
 
-	if (idmap_init(&idmap_hdl) != IDMAP_SUCCESS)
-		goto out;
-	if (idmap_get_create(idmap_hdl, &get_hdl) != IDMAP_SUCCESS)
+	if (idmap_get_create(&get_hdl) != IDMAP_SUCCESS)
 		goto out;
 
 	if (isuser) {
@@ -2199,8 +2224,6 @@ idmap_id_to_numeric_domain_rid(uid_t id, boolean_t isuser,
 out:
 	if (get_hdl)
 		idmap_get_destroy(get_hdl);
-	if (idmap_hdl)
-		(void) idmap_fini(idmap_hdl);
 	return (err);
 }
 
@@ -3898,11 +3921,14 @@ zfs_userspace(zfs_handle_t *zhp, zfs_userquota_prop_t type,
 
 int
 zfs_hold(zfs_handle_t *zhp, const char *snapname, const char *tag,
-    boolean_t recursive, boolean_t temphold, boolean_t enoent_ok)
+    boolean_t recursive, boolean_t temphold, boolean_t enoent_ok,
+    int cleanup_fd, uint64_t dsobj, uint64_t createtxg)
 {
 	zfs_cmd_t zc = { 0 };
 	libzfs_handle_t *hdl = zhp->zfs_hdl;
 
+	ASSERT(!recursive || dsobj == 0);
+
 	(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
 	(void) strlcpy(zc.zc_value, snapname, sizeof (zc.zc_value));
 	if (strlcpy(zc.zc_string, tag, sizeof (zc.zc_string))
@@ -3910,6 +3936,9 @@ zfs_hold(zfs_handle_t *zhp, const char *snapname, const char *tag,
 		return (zfs_error(hdl, EZFS_TAGTOOLONG, tag));
 	zc.zc_cookie = recursive;
 	zc.zc_temphold = temphold;
+	zc.zc_cleanup_fd = cleanup_fd;
+	zc.zc_sendobj = dsobj;
+	zc.zc_createtxg = createtxg;
 
 	if (zfs_ioctl(hdl, ZFS_IOC_HOLD, &zc) != 0) {
 		char errbuf[ZFS_MAXNAMELEN+32];
@@ -3939,7 +3968,7 @@ zfs_hold(zfs_handle_t *zhp, const char *snapname, const char *tag,
 			return (zfs_error(hdl, EZFS_REFTAG_HOLD, errbuf));
 		case ENOENT:
 			if (enoent_ok)
-				return (0);
+				return (ENOENT);
 			/* FALLTHROUGH */
 		default:
 			return (zfs_standard_error_fmt(hdl, errno, errbuf));
@@ -3949,102 +3978,6 @@ zfs_hold(zfs_handle_t *zhp, const char *snapname, const char *tag,
 	return (0);
 }
 
-struct hold_range_arg {
-	zfs_handle_t	*origin;
-	const char	*fromsnap;
-	const char	*tosnap;
-	char		lastsnapheld[ZFS_MAXNAMELEN];
-	const char	*tag;
-	boolean_t	temphold;
-	boolean_t	seento;
-	boolean_t	seenfrom;
-	boolean_t	holding;
-	boolean_t	recursive;
-	snapfilter_cb_t	*filter_cb;
-	void		*filter_cb_arg;
-};
-
-static int
-zfs_hold_range_one(zfs_handle_t *zhp, void *arg)
-{
-	struct hold_range_arg *hra = arg;
-	const char *thissnap;
-	int error;
-
-	thissnap = strchr(zfs_get_name(zhp), '@') + 1;
-
-	if (hra->fromsnap && !hra->seenfrom &&
-	    strcmp(hra->fromsnap, thissnap) == 0)
-		hra->seenfrom = B_TRUE;
-
-	/* snap is older or newer than the desired range, ignore it */
-	if (hra->seento || !hra->seenfrom) {
-		zfs_close(zhp);
-		return (0);
-	}
-
-	if (!hra->seento && strcmp(hra->tosnap, thissnap) == 0)
-		hra->seento = B_TRUE;
-
-	if (hra->filter_cb != NULL &&
-	    hra->filter_cb(zhp, hra->filter_cb_arg) == B_FALSE) {
-		zfs_close(zhp);
-		return (0);
-	}
-
-	if (hra->holding) {
-		/* We could be racing with destroy, so ignore ENOENT. */
-		error = zfs_hold(hra->origin, thissnap, hra->tag,
-		    hra->recursive, hra->temphold, B_TRUE);
-		if (error == 0) {
-			(void) strlcpy(hra->lastsnapheld, zfs_get_name(zhp),
-			    sizeof (hra->lastsnapheld));
-		}
-	} else {
-		error = zfs_release(hra->origin, thissnap, hra->tag,
-		    hra->recursive);
-	}
-
-	zfs_close(zhp);
-	return (error);
-}
-
-/*
- * Add a user hold on the set of snapshots starting with fromsnap up to
- * and including tosnap. If we're unable to to acquire a particular hold,
- * undo any holds up to that point.
- */
-int
-zfs_hold_range(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
-    const char *tag, boolean_t recursive, boolean_t temphold,
-    snapfilter_cb_t filter_cb, void *cbarg)
-{
-	struct hold_range_arg arg = { 0 };
-	int error;
-
-	arg.origin = zhp;
-	arg.fromsnap = fromsnap;
-	arg.tosnap = tosnap;
-	arg.tag = tag;
-	arg.temphold = temphold;
-	arg.holding = B_TRUE;
-	arg.recursive = recursive;
-	arg.seenfrom = (fromsnap == NULL);
-	arg.filter_cb = filter_cb;
-	arg.filter_cb_arg = cbarg;
-
-	error = zfs_iter_snapshots_sorted(zhp, zfs_hold_range_one, &arg);
-
-	/*
-	 * Make sure we either hold the entire range or none.
-	 */
-	if (error && arg.lastsnapheld[0] != '\0') {
-		(void) zfs_release_range(zhp, fromsnap,
-		    (const char *)arg.lastsnapheld, tag, recursive);
-	}
-	return (error);
-}
-
 int
 zfs_release(zfs_handle_t *zhp, const char *snapname, const char *tag,
     boolean_t recursive)
@@ -4086,26 +4019,6 @@ zfs_release(zfs_handle_t *zhp, const char *snapname, const char *tag,
 	return (0);
 }
 
-/*
- * Release a user hold from the set of snapshots starting with fromsnap
- * up to and including tosnap.
- */
-int
-zfs_release_range(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
-    const char *tag, boolean_t recursive)
-{
-	struct hold_range_arg arg = { 0 };
-
-	arg.origin = zhp;
-	arg.fromsnap = fromsnap;
-	arg.tosnap = tosnap;
-	arg.tag = tag;
-	arg.recursive = recursive;
-	arg.seenfrom = (fromsnap == NULL);
-
-	return (zfs_iter_snapshots_sorted(zhp, zfs_hold_range_one, &arg));
-}
-
 uint64_t
 zvol_volsize_to_reservation(uint64_t volsize, nvlist_t *props)
 {
diff --git a/lib/libzfs/libzfs_diff.c b/lib/libzfs/libzfs_diff.c
new file mode 100644
index 000000000..888224f3b
--- /dev/null
+++ b/lib/libzfs/libzfs_diff.c
@@ -0,0 +1,826 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+/*
+ * zfs diff support
+ */
+#include <ctype.h>
+#include <errno.h>
+#include <libintl.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <attr.h>
+#include <stddef.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stropts.h>
+#include <pthread.h>
+#include <sys/zfs_ioctl.h>
+#include <libzfs.h>
+#include "libzfs_impl.h"
+
+#define	ZDIFF_SNAPDIR		"/.zfs/snapshot/"
+#define	ZDIFF_SHARESDIR 	"/.zfs/shares/"
+#define	ZDIFF_PREFIX		"zfs-diff-%d"
+
+#define	ZDIFF_ADDED	'+'
+#define	ZDIFF_MODIFIED	'M'
+#define	ZDIFF_REMOVED	'-'
+#define	ZDIFF_RENAMED	'R'
+
+static boolean_t
+do_name_cmp(const char *fpath, const char *tpath)
+{
+	char *fname, *tname;
+	fname = strrchr(fpath, '/') + 1;
+	tname = strrchr(tpath, '/') + 1;
+	return (strcmp(fname, tname) == 0);
+}
+
+typedef struct differ_info {
+	zfs_handle_t *zhp;
+	char *fromsnap;
+	char *frommnt;
+	char *tosnap;
+	char *tomnt;
+	char *ds;
+	char *dsmnt;
+	char *tmpsnap;
+	char errbuf[1024];
+	boolean_t isclone;
+	boolean_t scripted;
+	boolean_t classify;
+	boolean_t timestamped;
+	uint64_t shares;
+	int zerr;
+	int cleanupfd;
+	int outputfd;
+	int datafd;
+} differ_info_t;
+
+/*
+ * Given a {dsname, object id}, get the object path
+ */
+static int
+get_stats_for_obj(differ_info_t *di, const char *dsname, uint64_t obj,
+    char *pn, int maxlen, zfs_stat_t *sb)
+{
+	zfs_cmd_t zc = { 0 };
+	int error;
+
+	(void) strlcpy(zc.zc_name, dsname, sizeof (zc.zc_name));
+	zc.zc_obj = obj;
+
+	errno = 0;
+	error = ioctl(di->zhp->zfs_hdl->libzfs_fd, ZFS_IOC_OBJ_TO_STATS, &zc);
+	di->zerr = errno;
+
+	/* we can get stats even if we failed to get a path */
+	(void) memcpy(sb, &zc.zc_stat, sizeof (zfs_stat_t));
+	if (error == 0) {
+		ASSERT(di->zerr == 0);
+		(void) strlcpy(pn, zc.zc_value, maxlen);
+		return (0);
+	}
+
+	if (di->zerr == EPERM) {
+		(void) snprintf(di->errbuf, sizeof (di->errbuf),
+		    dgettext(TEXT_DOMAIN,
+		    "The sys_config privilege or diff delegated permission "
+		    "is needed\nto discover path names"));
+		return (-1);
+	} else {
+		(void) snprintf(di->errbuf, sizeof (di->errbuf),
+		    dgettext(TEXT_DOMAIN,
+		    "Unable to determine path or stats for "
+		    "object %lld in %s"), obj, dsname);
+		return (-1);
+	}
+}
+
+/*
+ * stream_bytes
+ *
+ * Prints a file name out a character at a time.  If the character is
+ * not in the range of what we consider "printable" ASCII, display it
+ * as an escaped 3-digit octal value.  ASCII values less than a space
+ * are all control characters and we declare the upper end as the
+ * DELete character.  This also is the last 7-bit ASCII character.
+ * We choose to treat all 8-bit ASCII as not printable for this
+ * application.
+ */
+static void
+stream_bytes(FILE *fp, const char *string)
+{
+	while (*string) {
+		if (*string > ' ' && *string != '\\' && *string < '\177')
+			(void) fprintf(fp, "%c", *string++);
+		else
+			(void) fprintf(fp, "\\%03o", *string++);
+	}
+}
+
+static void
+print_what(FILE *fp, mode_t what)
+{
+	char symbol;
+
+	switch (what & S_IFMT) {
+	case S_IFBLK:
+		symbol = 'B';
+		break;
+	case S_IFCHR:
+		symbol = 'C';
+		break;
+	case S_IFDIR:
+		symbol = '/';
+		break;
+	case S_IFDOOR:
+		symbol = '>';
+		break;
+	case S_IFIFO:
+		symbol = '|';
+		break;
+	case S_IFLNK:
+		symbol = '@';
+		break;
+	case S_IFPORT:
+		symbol = 'P';
+		break;
+	case S_IFSOCK:
+		symbol = '=';
+		break;
+	case S_IFREG:
+		symbol = 'F';
+		break;
+	default:
+		symbol = '?';
+		break;
+	}
+	(void) fprintf(fp, "%c", symbol);
+}
+
+static void
+print_cmn(FILE *fp, differ_info_t *di, const char *file)
+{
+	stream_bytes(fp, di->dsmnt);
+	stream_bytes(fp, file);
+}
+
+static void
+print_rename(FILE *fp, differ_info_t *di, const char *old, const char *new,
+    zfs_stat_t *isb)
+{
+	if (di->timestamped)
+		(void) fprintf(fp, "%10lld.%09lld\t",
+		    (longlong_t)isb->zs_ctime[0],
+		    (longlong_t)isb->zs_ctime[1]);
+	(void) fprintf(fp, "%c\t", ZDIFF_RENAMED);
+	if (di->classify) {
+		print_what(fp, isb->zs_mode);
+		(void) fprintf(fp, "\t");
+	}
+	print_cmn(fp, di, old);
+	if (di->scripted)
+		(void) fprintf(fp, "\t");
+	else
+		(void) fprintf(fp, " -> ");
+	print_cmn(fp, di, new);
+	(void) fprintf(fp, "\n");
+}
+
+static void
+print_link_change(FILE *fp, differ_info_t *di, int delta, const char *file,
+    zfs_stat_t *isb)
+{
+	if (di->timestamped)
+		(void) fprintf(fp, "%10lld.%09lld\t",
+		    (longlong_t)isb->zs_ctime[0],
+		    (longlong_t)isb->zs_ctime[1]);
+	(void) fprintf(fp, "%c\t", ZDIFF_MODIFIED);
+	if (di->classify) {
+		print_what(fp, isb->zs_mode);
+		(void) fprintf(fp, "\t");
+	}
+	print_cmn(fp, di, file);
+	(void) fprintf(fp, "\t(%+d)", delta);
+	(void) fprintf(fp, "\n");
+}
+
+static void
+print_file(FILE *fp, differ_info_t *di, char type, const char *file,
+    zfs_stat_t *isb)
+{
+	if (di->timestamped)
+		(void) fprintf(fp, "%10lld.%09lld\t",
+		    (longlong_t)isb->zs_ctime[0],
+		    (longlong_t)isb->zs_ctime[1]);
+	(void) fprintf(fp, "%c\t", type);
+	if (di->classify) {
+		print_what(fp, isb->zs_mode);
+		(void) fprintf(fp, "\t");
+	}
+	print_cmn(fp, di, file);
+	(void) fprintf(fp, "\n");
+}
+
+static int
+write_inuse_diffs_one(FILE *fp, differ_info_t *di, uint64_t dobj)
+{
+	struct zfs_stat fsb, tsb;
+	boolean_t same_name;
+	mode_t fmode, tmode;
+	char fobjname[MAXPATHLEN], tobjname[MAXPATHLEN];
+	int fobjerr, tobjerr;
+	int change;
+
+	if (dobj == di->shares)
+		return (0);
+
+	/*
+	 * Check the from and to snapshots for info on the object. If
+	 * we get ENOENT, then the object just didn't exist in that
+	 * snapshot.  If we get ENOTSUP, then we tried to get
+	 * info on a non-ZPL object, which we don't care about anyway.
+	 */
+	fobjerr = get_stats_for_obj(di, di->fromsnap, dobj, fobjname,
+	    MAXPATHLEN, &fsb);
+	if (fobjerr && di->zerr != ENOENT && di->zerr != ENOTSUP)
+		return (-1);
+
+	tobjerr = get_stats_for_obj(di, di->tosnap, dobj, tobjname,
+	    MAXPATHLEN, &tsb);
+	if (tobjerr && di->zerr != ENOENT && di->zerr != ENOTSUP)
+		return (-1);
+
+	/*
+	 * Unallocated object sharing the same meta dnode block
+	 */
+	if (fobjerr && tobjerr) {
+		ASSERT(di->zerr == ENOENT || di->zerr == ENOTSUP);
+		di->zerr = 0;
+		return (0);
+	}
+
+	di->zerr = 0; /* negate get_stats_for_obj() from side that failed */
+	fmode = fsb.zs_mode & S_IFMT;
+	tmode = tsb.zs_mode & S_IFMT;
+	if (fmode == S_IFDIR || tmode == S_IFDIR || fsb.zs_links == 0 ||
+	    tsb.zs_links == 0)
+		change = 0;
+	else
+		change = tsb.zs_links - fsb.zs_links;
+
+	if (fobjerr) {
+		if (change) {
+			print_link_change(fp, di, change, tobjname, &tsb);
+			return (0);
+		}
+		print_file(fp, di, ZDIFF_ADDED, tobjname, &tsb);
+		return (0);
+	} else if (tobjerr) {
+		if (change) {
+			print_link_change(fp, di, change, fobjname, &fsb);
+			return (0);
+		}
+		print_file(fp, di, ZDIFF_REMOVED, fobjname, &fsb);
+		return (0);
+	}
+
+	if (fmode != tmode && fsb.zs_gen == tsb.zs_gen)
+		tsb.zs_gen++;	/* Force a generational difference */
+	same_name = do_name_cmp(fobjname, tobjname);
+
+	/* Simple modification or no change */
+	if (fsb.zs_gen == tsb.zs_gen) {
+		/* No apparent changes.  Could we assert !this?  */
+		if (fsb.zs_ctime[0] == tsb.zs_ctime[0] &&
+		    fsb.zs_ctime[1] == tsb.zs_ctime[1])
+			return (0);
+		if (change) {
+			print_link_change(fp, di, change,
+			    change > 0 ? fobjname : tobjname, &tsb);
+		} else if (same_name) {
+			print_file(fp, di, ZDIFF_MODIFIED, fobjname, &tsb);
+		} else {
+			print_rename(fp, di, fobjname, tobjname, &tsb);
+		}
+		return (0);
+	} else {
+		/* file re-created or object re-used */
+		print_file(fp, di, ZDIFF_REMOVED, fobjname, &fsb);
+		print_file(fp, di, ZDIFF_ADDED, tobjname, &tsb);
+		return (0);
+	}
+}
+
+static int
+write_inuse_diffs(FILE *fp, differ_info_t *di, dmu_diff_record_t *dr)
+{
+	uint64_t o;
+	int err;
+
+	for (o = dr->ddr_first; o <= dr->ddr_last; o++) {
+		if (err = write_inuse_diffs_one(fp, di, o))
+			return (err);
+	}
+	return (0);
+}
+
+static int
+describe_free(FILE *fp, differ_info_t *di, uint64_t object, char *namebuf,
+    int maxlen)
+{
+	struct zfs_stat sb;
+
+	if (get_stats_for_obj(di, di->fromsnap, object, namebuf,
+	    maxlen, &sb) != 0) {
+		/* Let it slide, if in the delete queue on from side */
+		if (di->zerr == ENOENT && sb.zs_links == 0) {
+			di->zerr = 0;
+			return (0);
+		}
+		return (-1);
+	}
+
+	print_file(fp, di, ZDIFF_REMOVED, namebuf, &sb);
+	return (0);
+}
+
+static int
+write_free_diffs(FILE *fp, differ_info_t *di, dmu_diff_record_t *dr)
+{
+	zfs_cmd_t zc = { 0 };
+	libzfs_handle_t *lhdl = di->zhp->zfs_hdl;
+	char fobjname[MAXPATHLEN];
+
+	(void) strlcpy(zc.zc_name, di->fromsnap, sizeof (zc.zc_name));
+	zc.zc_obj = dr->ddr_first - 1;
+
+	ASSERT(di->zerr == 0);
+
+	while (zc.zc_obj < dr->ddr_last) {
+		int err;
+
+		err = ioctl(lhdl->libzfs_fd, ZFS_IOC_NEXT_OBJ, &zc);
+		if (err == 0) {
+			if (zc.zc_obj == di->shares) {
+				zc.zc_obj++;
+				continue;
+			}
+			if (zc.zc_obj > dr->ddr_last) {
+				break;
+			}
+			err = describe_free(fp, di, zc.zc_obj, fobjname,
+			    MAXPATHLEN);
+			if (err)
+				break;
+		} else if (errno == ESRCH) {
+			break;
+		} else {
+			(void) snprintf(di->errbuf, sizeof (di->errbuf),
+			    dgettext(TEXT_DOMAIN,
+			    "next allocated object (> %lld) find failure"),
+			    zc.zc_obj);
+			di->zerr = errno;
+			break;
+		}
+	}
+	if (di->zerr)
+		return (-1);
+	return (0);
+}
+
+static void *
+differ(void *arg)
+{
+	differ_info_t *di = arg;
+	dmu_diff_record_t dr;
+	FILE *ofp;
+	int err = 0;
+
+	if ((ofp = fdopen(di->outputfd, "w")) == NULL) {
+		di->zerr = errno;
+		(void) strerror_r(errno, di->errbuf, sizeof (di->errbuf));
+		(void) close(di->datafd);
+		return ((void *)-1);
+	}
+
+	for (;;) {
+		char *cp = (char *)&dr;
+		int len = sizeof (dr);
+		int rv;
+
+		do {
+			rv = read(di->datafd, cp, len);
+			cp += rv;
+			len -= rv;
+		} while (len > 0 && rv > 0);
+
+		if (rv < 0 || (rv == 0 && len != sizeof (dr))) {
+			di->zerr = EPIPE;
+			break;
+		} else if (rv == 0) {
+			/* end of file at a natural breaking point */
+			break;
+		}
+
+		switch (dr.ddr_type) {
+		case DDR_FREE:
+			err = write_free_diffs(ofp, di, &dr);
+			break;
+		case DDR_INUSE:
+			err = write_inuse_diffs(ofp, di, &dr);
+			break;
+		default:
+			di->zerr = EPIPE;
+			break;
+		}
+
+		if (err || di->zerr)
+			break;
+	}
+
+	(void) fclose(ofp);
+	(void) close(di->datafd);
+	if (err)
+		return ((void *)-1);
+	if (di->zerr) {
+		ASSERT(di->zerr == EINVAL);
+		(void) snprintf(di->errbuf, sizeof (di->errbuf),
+		    dgettext(TEXT_DOMAIN,
+		    "Internal error: bad data from diff IOCTL"));
+		return ((void *)-1);
+	}
+	return ((void *)0);
+}
+
+static int
+find_shares_object(differ_info_t *di)
+{
+	char fullpath[MAXPATHLEN];
+	struct stat64 sb = { 0 };
+
+	(void) strlcpy(fullpath, di->dsmnt, MAXPATHLEN);
+	(void) strlcat(fullpath, ZDIFF_SHARESDIR, MAXPATHLEN);
+
+	if (stat64(fullpath, &sb) != 0) {
+		(void) snprintf(di->errbuf, sizeof (di->errbuf),
+		    dgettext(TEXT_DOMAIN, "Cannot stat %s"), fullpath);
+		return (zfs_error(di->zhp->zfs_hdl, EZFS_DIFF, di->errbuf));
+	}
+
+	di->shares = (uint64_t)sb.st_ino;
+	return (0);
+}
+
+static int
+make_temp_snapshot(differ_info_t *di)
+{
+	libzfs_handle_t *hdl = di->zhp->zfs_hdl;
+	zfs_cmd_t zc = { 0 };
+
+	(void) snprintf(zc.zc_value, sizeof (zc.zc_value),
+	    ZDIFF_PREFIX, getpid());
+	(void) strlcpy(zc.zc_name, di->ds, sizeof (zc.zc_name));
+	zc.zc_cleanup_fd = di->cleanupfd;
+
+	if (ioctl(hdl->libzfs_fd, ZFS_IOC_TMP_SNAPSHOT, &zc) != 0) {
+		int err = errno;
+		if (err == EPERM) {
+			(void) snprintf(di->errbuf, sizeof (di->errbuf),
+			    dgettext(TEXT_DOMAIN, "The diff delegated "
+			    "permission is needed in order\nto create a "
+			    "just-in-time snapshot for diffing\n"));
+			return (zfs_error(hdl, EZFS_DIFF, di->errbuf));
+		} else {
+			(void) snprintf(di->errbuf, sizeof (di->errbuf),
+			    dgettext(TEXT_DOMAIN, "Cannot create just-in-time "
+			    "snapshot of '%s'"), zc.zc_name);
+			return (zfs_standard_error(hdl, err, di->errbuf));
+		}
+	}
+
+	di->tmpsnap = zfs_strdup(hdl, zc.zc_value);
+	di->tosnap = zfs_asprintf(hdl, "%s@%s", di->ds, di->tmpsnap);
+	return (0);
+}
+
+static void
+teardown_differ_info(differ_info_t *di)
+{
+	free(di->ds);
+	free(di->dsmnt);
+	free(di->fromsnap);
+	free(di->frommnt);
+	free(di->tosnap);
+	free(di->tmpsnap);
+	free(di->tomnt);
+	(void) close(di->cleanupfd);
+}
+
+static int
+get_snapshot_names(differ_info_t *di, const char *fromsnap,
+    const char *tosnap)
+{
+	libzfs_handle_t *hdl = di->zhp->zfs_hdl;
+	char *atptrf = NULL;
+	char *atptrt = NULL;
+	int fdslen, fsnlen;
+	int tdslen, tsnlen;
+
+	/*
+	 * Can accept
+	 *    dataset@snap1
+	 *    dataset@snap1 dataset@snap2
+	 *    dataset@snap1 @snap2
+	 *    dataset@snap1 dataset
+	 *    @snap1 dataset@snap2
+	 */
+	if (tosnap == NULL) {
+		/* only a from snapshot given, must be valid */
+		(void) snprintf(di->errbuf, sizeof (di->errbuf),
+		    dgettext(TEXT_DOMAIN,
+		    "Badly formed snapshot name %s"), fromsnap);
+
+		if (!zfs_validate_name(hdl, fromsnap, ZFS_TYPE_SNAPSHOT,
+		    B_FALSE)) {
+			return (zfs_error(hdl, EZFS_INVALIDNAME,
+			    di->errbuf));
+		}
+
+		atptrf = strchr(fromsnap, '@');
+		ASSERT(atptrf != NULL);
+		fdslen = atptrf - fromsnap;
+
+		di->fromsnap = zfs_strdup(hdl, fromsnap);
+		di->ds = zfs_strdup(hdl, fromsnap);
+		di->ds[fdslen] = '\0';
+
+		/* the to snap will be a just-in-time snap of the head */
+		return (make_temp_snapshot(di));
+	}
+
+	(void) snprintf(di->errbuf, sizeof (di->errbuf),
+	    dgettext(TEXT_DOMAIN,
+	    "Unable to determine which snapshots to compare"));
+
+	atptrf = strchr(fromsnap, '@');
+	atptrt = strchr(tosnap, '@');
+	fdslen = atptrf ? atptrf - fromsnap : strlen(fromsnap);
+	tdslen = atptrt ? atptrt - tosnap : strlen(tosnap);
+	fsnlen = strlen(fromsnap) - fdslen;	/* includes @ sign */
+	tsnlen = strlen(tosnap) - tdslen;	/* includes @ sign */
+
+	if (fsnlen <= 1 || tsnlen == 1 || (fdslen == 0 && tdslen == 0) ||
+	    (fsnlen == 0 && tsnlen == 0)) {
+		return (zfs_error(hdl, EZFS_INVALIDNAME, di->errbuf));
+	} else if ((fdslen > 0 && tdslen > 0) &&
+	    ((tdslen != fdslen || strncmp(fromsnap, tosnap, fdslen) != 0))) {
+		/*
+		 * not the same dataset name, might be okay if
+		 * tosnap is a clone of a fromsnap descendant.
+		 */
+		char origin[ZFS_MAXNAMELEN];
+		zprop_source_t src;
+		zfs_handle_t *zhp;
+
+		di->ds = zfs_alloc(di->zhp->zfs_hdl, tdslen + 1);
+		(void) strncpy(di->ds, tosnap, tdslen);
+		di->ds[tdslen] = '\0';
+
+		zhp = zfs_open(hdl, di->ds, ZFS_TYPE_FILESYSTEM);
+		while (zhp != NULL) {
+			(void) zfs_prop_get(zhp, ZFS_PROP_ORIGIN,
+			    origin, sizeof (origin), &src, NULL, 0, B_FALSE);
+
+			if (strncmp(origin, fromsnap, fsnlen) == 0)
+				break;
+
+			(void) zfs_close(zhp);
+			zhp = zfs_open(hdl, origin, ZFS_TYPE_FILESYSTEM);
+		}
+
+		if (zhp == NULL) {
+			(void) snprintf(di->errbuf, sizeof (di->errbuf),
+			    dgettext(TEXT_DOMAIN,
+			    "Not an earlier snapshot from the same fs"));
+			return (zfs_error(hdl, EZFS_INVALIDNAME, di->errbuf));
+		} else {
+			(void) zfs_close(zhp);
+		}
+
+		di->isclone = B_TRUE;
+		di->fromsnap = zfs_strdup(hdl, fromsnap);
+		if (tsnlen) {
+			di->tosnap = zfs_strdup(hdl, tosnap);
+		} else {
+			return (make_temp_snapshot(di));
+		}
+	} else {
+		int dslen = fdslen ? fdslen : tdslen;
+
+		di->ds = zfs_alloc(hdl, dslen + 1);
+		(void) strncpy(di->ds, fdslen ? fromsnap : tosnap, dslen);
+		di->ds[dslen] = '\0';
+
+		di->fromsnap = zfs_asprintf(hdl, "%s%s", di->ds, atptrf);
+		if (tsnlen) {
+			di->tosnap = zfs_asprintf(hdl, "%s%s", di->ds, atptrt);
+		} else {
+			return (make_temp_snapshot(di));
+		}
+	}
+	return (0);
+}
+
+static int
+get_mountpoint(differ_info_t *di, char *dsnm, char **mntpt)
+{
+	boolean_t mounted;
+
+	mounted = is_mounted(di->zhp->zfs_hdl, dsnm, mntpt);
+	if (mounted == B_FALSE) {
+		(void) snprintf(di->errbuf, sizeof (di->errbuf),
+		    dgettext(TEXT_DOMAIN,
+		    "Cannot diff an unmounted snapshot"));
+		return (zfs_error(di->zhp->zfs_hdl, EZFS_BADTYPE, di->errbuf));
+	}
+
+	/* Avoid a double slash at the beginning of root-mounted datasets */
+	if (**mntpt == '/' && *(*mntpt + 1) == '\0')
+		**mntpt = '\0';
+	return (0);
+}
+
+static int
+get_mountpoints(differ_info_t *di)
+{
+	char *strptr;
+	char *frommntpt;
+
+	/*
+	 * first get the mountpoint for the parent dataset
+	 */
+	if (get_mountpoint(di, di->ds, &di->dsmnt) != 0)
+		return (-1);
+
+	strptr = strchr(di->tosnap, '@');
+	ASSERT3P(strptr, !=, NULL);
+	di->tomnt = zfs_asprintf(di->zhp->zfs_hdl, "%s%s%s", di->dsmnt,
+	    ZDIFF_SNAPDIR, ++strptr);
+
+	strptr = strchr(di->fromsnap, '@');
+	ASSERT3P(strptr, !=, NULL);
+
+	frommntpt = di->dsmnt;
+	if (di->isclone) {
+		char *mntpt;
+		int err;
+
+		*strptr = '\0';
+		err = get_mountpoint(di, di->fromsnap, &mntpt);
+		*strptr = '@';
+		if (err != 0)
+			return (-1);
+		frommntpt = mntpt;
+	}
+
+	di->frommnt = zfs_asprintf(di->zhp->zfs_hdl, "%s%s%s", frommntpt,
+	    ZDIFF_SNAPDIR, ++strptr);
+
+	if (di->isclone)
+		free(frommntpt);
+
+	return (0);
+}
+
+static int
+setup_differ_info(zfs_handle_t *zhp, const char *fromsnap,
+    const char *tosnap, differ_info_t *di)
+{
+	di->zhp = zhp;
+
+	di->cleanupfd = open(ZFS_DEV, O_RDWR|O_EXCL);
+	VERIFY(di->cleanupfd >= 0);
+
+	if (get_snapshot_names(di, fromsnap, tosnap) != 0)
+		return (-1);
+
+	if (get_mountpoints(di) != 0)
+		return (-1);
+
+	if (find_shares_object(di) != 0)
+		return (-1);
+
+	return (0);
+}
+
+int
+zfs_show_diffs(zfs_handle_t *zhp, int outfd, const char *fromsnap,
+    const char *tosnap, int flags)
+{
+	zfs_cmd_t zc = { 0 };
+	char errbuf[1024];
+	differ_info_t di = { 0 };
+	pthread_t tid;
+	int pipefd[2];
+	int iocerr;
+
+	(void) snprintf(errbuf, sizeof (errbuf),
+	    dgettext(TEXT_DOMAIN, "zfs diff failed"));
+
+	if (setup_differ_info(zhp, fromsnap, tosnap, &di)) {
+		teardown_differ_info(&di);
+		return (-1);
+	}
+
+	if (pipe(pipefd)) {
+		zfs_error_aux(zhp->zfs_hdl, strerror(errno));
+		teardown_differ_info(&di);
+		return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED, errbuf));
+	}
+
+	di.scripted = (flags & ZFS_DIFF_PARSEABLE);
+	di.classify = (flags & ZFS_DIFF_CLASSIFY);
+	di.timestamped = (flags & ZFS_DIFF_TIMESTAMP);
+
+	di.outputfd = outfd;
+	di.datafd = pipefd[0];
+
+	if (pthread_create(&tid, NULL, differ, &di)) {
+		zfs_error_aux(zhp->zfs_hdl, strerror(errno));
+		(void) close(pipefd[0]);
+		(void) close(pipefd[1]);
+		teardown_differ_info(&di);
+		return (zfs_error(zhp->zfs_hdl,
+		    EZFS_THREADCREATEFAILED, errbuf));
+	}
+
+	/* do the ioctl() */
+	(void) strlcpy(zc.zc_value, di.fromsnap, strlen(di.fromsnap) + 1);
+	(void) strlcpy(zc.zc_name, di.tosnap, strlen(di.tosnap) + 1);
+	zc.zc_cookie = pipefd[1];
+
+	iocerr = ioctl(zhp->zfs_hdl->libzfs_fd, ZFS_IOC_DIFF, &zc);
+	if (iocerr != 0) {
+		(void) snprintf(errbuf, sizeof (errbuf),
+		    dgettext(TEXT_DOMAIN, "Unable to obtain diffs"));
+		if (errno == EPERM) {
+			zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
+			    "\n   The sys_mount privilege or diff delegated "
+			    "permission is needed\n   to execute the "
+			    "diff ioctl"));
+		} else if (errno == EXDEV) {
+			zfs_error_aux(zhp->zfs_hdl, dgettext(TEXT_DOMAIN,
+			    "\n   Not an earlier snapshot from the same fs"));
+		} else if (errno != EPIPE || di.zerr == 0) {
+			zfs_error_aux(zhp->zfs_hdl, strerror(errno));
+		}
+		(void) close(pipefd[1]);
+		(void) pthread_cancel(tid);
+		(void) pthread_join(tid, NULL);
+		teardown_differ_info(&di);
+		if (di.zerr != 0 && di.zerr != EPIPE) {
+			zfs_error_aux(zhp->zfs_hdl, strerror(di.zerr));
+			return (zfs_error(zhp->zfs_hdl, EZFS_DIFF, di.errbuf));
+		} else {
+			return (zfs_error(zhp->zfs_hdl, EZFS_DIFFDATA, errbuf));
+		}
+	}
+
+	(void) close(pipefd[1]);
+	(void) pthread_join(tid, NULL);
+
+	if (di.zerr != 0) {
+		zfs_error_aux(zhp->zfs_hdl, strerror(di.zerr));
+		return (zfs_error(zhp->zfs_hdl, EZFS_DIFF, di.errbuf));
+	}
+	teardown_differ_info(&di);
+	return (0);
+}
diff --git a/lib/libzfs/libzfs_import.c b/lib/libzfs/libzfs_import.c
index fd3044b1d..e1370350f 100644
--- a/lib/libzfs/libzfs_import.c
+++ b/lib/libzfs/libzfs_import.c
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*
@@ -1559,6 +1558,17 @@ zpool_in_use(libzfs_handle_t *hdl, int fd, pool_state_t *state, char **namestr,
 
 	switch (stateval) {
 	case POOL_STATE_EXPORTED:
+		/*
+		 * A pool with an exported state may in fact be imported
+		 * read-only, so check the in-core state to see if it's
+		 * active and imported read-only.  If it is, set
+		 * its state to active.
+		 */
+		if (pool_active(hdl, name, guid, &isactive) == 0 && isactive &&
+		    (zhp = zpool_open_canfail(hdl, name)) != NULL &&
+		    zpool_get_prop_int(zhp, ZPOOL_PROP_READONLY, NULL))
+			stateval = POOL_STATE_ACTIVE;
+
 		ret = B_TRUE;
 		break;
 
diff --git a/lib/libzfs/libzfs_mount.c b/lib/libzfs/libzfs_mount.c
index 0675ec229..922220620 100644
--- a/lib/libzfs/libzfs_mount.c
+++ b/lib/libzfs/libzfs_mount.c
@@ -270,6 +270,12 @@ zfs_mount(zfs_handle_t *zhp, const char *options, int flags)
 	else
 		(void) strlcpy(mntopts, options, sizeof (mntopts));
 
+	/*
+	 * If the pool is imported read-only then all mounts must be read-only
+	 */
+	if (zpool_get_prop_int(zhp->zpool_hdl, ZPOOL_PROP_READONLY, NULL))
+		flags |= MS_RDONLY;
+
 	if (!zfs_is_mountable(zhp, mountpoint, sizeof (mountpoint), NULL))
 		return (0);
 
@@ -437,18 +443,14 @@ zfs_is_shared(zfs_handle_t *zhp)
 int
 zfs_share(zfs_handle_t *zhp)
 {
-	if (ZFS_IS_VOLUME(zhp))
-		return (0);
-
+	assert(!ZFS_IS_VOLUME(zhp));
 	return (zfs_share_proto(zhp, share_all_proto));
 }
 
 int
 zfs_unshare(zfs_handle_t *zhp)
 {
-	if (ZFS_IS_VOLUME(zhp))
-		return (0);
-
+	assert(!ZFS_IS_VOLUME(zhp));
 	return (zfs_unshareall(zhp));
 }
 
@@ -979,18 +981,29 @@ remove_mountpoint(zfs_handle_t *zhp)
 	}
 }
 
-typedef struct mount_cbdata {
-	zfs_handle_t	**cb_datasets;
-	int 		cb_used;
-	int		cb_alloc;
-} mount_cbdata_t;
+void
+libzfs_add_handle(get_all_cb_t *cbp, zfs_handle_t *zhp)
+{
+	if (cbp->cb_alloc == cbp->cb_used) {
+		size_t newsz;
+		void *ptr;
+
+		newsz = cbp->cb_alloc ? cbp->cb_alloc * 2 : 64;
+		ptr = zfs_realloc(zhp->zfs_hdl,
+		    cbp->cb_handles, cbp->cb_alloc * sizeof (void *),
+		    newsz * sizeof (void *));
+		cbp->cb_handles = ptr;
+		cbp->cb_alloc = newsz;
+	}
+	cbp->cb_handles[cbp->cb_used++] = zhp;
+}
 
 static int
 mount_cb(zfs_handle_t *zhp, void *data)
 {
-	mount_cbdata_t *cbp = data;
+	get_all_cb_t *cbp = data;
 
-	if (!(zfs_get_type(zhp) & (ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME))) {
+	if (!(zfs_get_type(zhp) & ZFS_TYPE_FILESYSTEM)) {
 		zfs_close(zhp);
 		return (0);
 	}
@@ -1000,25 +1013,16 @@ mount_cb(zfs_handle_t *zhp, void *data)
 		return (0);
 	}
 
-	if (cbp->cb_alloc == cbp->cb_used) {
-		void *ptr;
-
-		if ((ptr = zfs_realloc(zhp->zfs_hdl,
-		    cbp->cb_datasets, cbp->cb_alloc * sizeof (void *),
-		    cbp->cb_alloc * 2 * sizeof (void *))) == NULL)
-			return (-1);
-		cbp->cb_datasets = ptr;
-
-		cbp->cb_alloc *= 2;
+	libzfs_add_handle(cbp, zhp);
+	if (zfs_iter_filesystems(zhp, mount_cb, cbp) != 0) {
+		zfs_close(zhp);
+		return (-1);
 	}
-
-	cbp->cb_datasets[cbp->cb_used++] = zhp;
-
-	return (zfs_iter_filesystems(zhp, mount_cb, cbp));
+	return (0);
 }
 
-static int
-dataset_cmp(const void *a, const void *b)
+int
+libzfs_dataset_cmp(const void *a, const void *b)
 {
 	zfs_handle_t **za = (zfs_handle_t **)a;
 	zfs_handle_t **zb = (zfs_handle_t **)b;
@@ -1056,7 +1060,7 @@ dataset_cmp(const void *a, const void *b)
 int
 zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags)
 {
-	mount_cbdata_t cb = { 0 };
+	get_all_cb_t cb = { 0 };
 	libzfs_handle_t *hdl = zhp->zpool_hdl;
 	zfs_handle_t *zfsp;
 	int i, ret = -1;
@@ -1065,23 +1069,17 @@ zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags)
 	/*
 	 * Gather all non-snap datasets within the pool.
 	 */
-	if ((cb.cb_datasets = zfs_alloc(hdl, 4 * sizeof (void *))) == NULL)
-		return (-1);
-	cb.cb_alloc = 4;
-
 	if ((zfsp = zfs_open(hdl, zhp->zpool_name, ZFS_TYPE_DATASET)) == NULL)
 		goto out;
 
-	cb.cb_datasets[0] = zfsp;
-	cb.cb_used = 1;
-
+	libzfs_add_handle(&cb, zfsp);
 	if (zfs_iter_filesystems(zfsp, mount_cb, &cb) != 0)
 		goto out;
-
 	/*
 	 * Sort the datasets by mountpoint.
 	 */
-	qsort(cb.cb_datasets, cb.cb_used, sizeof (void *), dataset_cmp);
+	qsort(cb.cb_handles, cb.cb_used, sizeof (void *),
+	    libzfs_dataset_cmp);
 
 	/*
 	 * And mount all the datasets, keeping track of which ones
@@ -1093,7 +1091,7 @@ zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags)
 
 	ret = 0;
 	for (i = 0; i < cb.cb_used; i++) {
-		if (zfs_mount(cb.cb_datasets[i], mntopts, flags) != 0)
+		if (zfs_mount(cb.cb_handles[i], mntopts, flags) != 0)
 			ret = -1;
 		else
 			good[i] = 1;
@@ -1106,7 +1104,7 @@ zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags)
 	 * zfs_alloc is supposed to exit if memory isn't available.
 	 */
 	for (i = 0; i < cb.cb_used; i++) {
-		if (good[i] && zfs_share(cb.cb_datasets[i]) != 0)
+		if (good[i] && zfs_share(cb.cb_handles[i]) != 0)
 			ret = -1;
 	}
 
@@ -1114,8 +1112,8 @@ zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags)
 
 out:
 	for (i = 0; i < cb.cb_used; i++)
-		zfs_close(cb.cb_datasets[i]);
-	free(cb.cb_datasets);
+		zfs_close(cb.cb_handles[i]);
+	free(cb.cb_handles);
 
 	return (ret);
 }
diff --git a/lib/libzfs/libzfs_pool.c b/lib/libzfs/libzfs_pool.c
index 7836e5873..7df7e910d 100644
--- a/lib/libzfs/libzfs_pool.c
+++ b/lib/libzfs/libzfs_pool.c
@@ -44,16 +44,15 @@
 
 static int read_efi_label(nvlist_t *config, diskaddr_t *sb);
 
-#if defined(__i386) || defined(__amd64)
-#define	BOOTCMD	"installgrub(1M)"
-#else
-#define	BOOTCMD	"installboot(1M)"
-#endif
-
 #define	DISK_ROOT	"/dev/dsk"
 #define	RDISK_ROOT	"/dev/rdsk"
 #define	BACKUP_SLICE	"s2"
 
+typedef struct prop_flags {
+	int create:1;	/* Validate property on creation */
+	int import:1;	/* Validate property on import */
+} prop_flags_t;
+
 /*
  * ====================================================================
  *   zpool property functions
@@ -376,7 +375,7 @@ pool_is_bootable(zpool_handle_t *zhp)
  */
 static nvlist_t *
 zpool_valid_proplist(libzfs_handle_t *hdl, const char *poolname,
-    nvlist_t *props, uint64_t version, boolean_t create_or_import, char *errbuf)
+    nvlist_t *props, uint64_t version, prop_flags_t flags, char *errbuf)
 {
 	nvpair_t *elem;
 	nvlist_t *retprops;
@@ -433,7 +432,7 @@ zpool_valid_proplist(libzfs_handle_t *hdl, const char *poolname,
 			break;
 
 		case ZPOOL_PROP_BOOTFS:
-			if (create_or_import) {
+			if (flags.create || flags.import) {
 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 				    "property '%s' cannot be set at creation "
 				    "or import time"), propname);
@@ -486,7 +485,7 @@ zpool_valid_proplist(libzfs_handle_t *hdl, const char *poolname,
 			break;
 
 		case ZPOOL_PROP_ALTROOT:
-			if (!create_or_import) {
+			if (!flags.create && !flags.import) {
 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 				    "property '%s' can only be set during pool "
 				    "creation or import"), propname);
@@ -541,6 +540,16 @@ zpool_valid_proplist(libzfs_handle_t *hdl, const char *poolname,
 
 			*slash = '/';
 			break;
+
+		case ZPOOL_PROP_READONLY:
+			if (!flags.import) {
+				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+				    "property '%s' can only be set at "
+				    "import time"), propname);
+				(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
+				goto error;
+			}
+			break;
 		}
 	}
 
@@ -562,6 +571,7 @@ zpool_set_prop(zpool_handle_t *zhp, const char *propname, const char *propval)
 	nvlist_t *nvl = NULL;
 	nvlist_t *realprops;
 	uint64_t version;
+	prop_flags_t flags = { 0 };
 
 	(void) snprintf(errbuf, sizeof (errbuf),
 	    dgettext(TEXT_DOMAIN, "cannot set property for '%s'"),
@@ -577,7 +587,7 @@ zpool_set_prop(zpool_handle_t *zhp, const char *propname, const char *propval)
 
 	version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
 	if ((realprops = zpool_valid_proplist(zhp->zpool_hdl,
-	    zhp->zpool_name, nvl, version, B_FALSE, errbuf)) == NULL) {
+	    zhp->zpool_name, nvl, version, flags, errbuf)) == NULL) {
 		nvlist_free(nvl);
 		return (-1);
 	}
@@ -884,8 +894,10 @@ zpool_create(libzfs_handle_t *hdl, const char *pool, nvlist_t *nvroot,
 		return (-1);
 
 	if (props) {
+		prop_flags_t flags = { .create = B_TRUE, .import = B_FALSE };
+
 		if ((zc_props = zpool_valid_proplist(hdl, pool, props,
-		    SPA_VERSION_1, B_TRUE, msg)) == NULL) {
+		    SPA_VERSION_1, flags, msg)) == NULL) {
 			goto create_failed;
 		}
 	}
@@ -1003,13 +1015,12 @@ zpool_destroy(zpool_handle_t *zhp)
 	char msg[1024];
 
 	if (zhp->zpool_state == POOL_STATE_ACTIVE &&
-	    (zfp = zfs_open(zhp->zpool_hdl, zhp->zpool_name,
-	    ZFS_TYPE_FILESYSTEM)) == NULL)
+	    (zfp = zfs_open(hdl, zhp->zpool_name, ZFS_TYPE_FILESYSTEM)) == NULL)
 		return (-1);
 
 	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
 
-	if (zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_POOL_DESTROY, &zc) != 0) {
+	if (zfs_ioctl(hdl, ZFS_IOC_POOL_DESTROY, &zc) != 0) {
 		(void) snprintf(msg, sizeof (msg), dgettext(TEXT_DOMAIN,
 		    "cannot destroy '%s'"), zhp->zpool_name);
 
@@ -1092,7 +1103,7 @@ zpool_add(zpool_handle_t *zhp, nvlist_t *nvroot)
 		return (-1);
 	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
 
-	if (zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_VDEV_ADD, &zc) != 0) {
+	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_ADD, &zc) != 0) {
 		switch (errno) {
 		case EBUSY:
 			/*
@@ -1208,19 +1219,23 @@ zpool_export_force(zpool_handle_t *zhp)
 
 static void
 zpool_rewind_exclaim(libzfs_handle_t *hdl, const char *name, boolean_t dryrun,
-    nvlist_t *rbi)
+    nvlist_t *config)
 {
+	nvlist_t *nv = NULL;
 	uint64_t rewindto;
 	int64_t loss = -1;
 	struct tm t;
 	char timestr[128];
 
-	if (!hdl->libzfs_printerr || rbi == NULL)
+	if (!hdl->libzfs_printerr || config == NULL)
 		return;
 
-	if (nvlist_lookup_uint64(rbi, ZPOOL_CONFIG_LOAD_TIME, &rewindto) != 0)
+	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, &nv) != 0)
 		return;
-	(void) nvlist_lookup_int64(rbi, ZPOOL_CONFIG_REWIND_TIME, &loss);
+
+	if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_LOAD_TIME, &rewindto) != 0)
+		return;
+	(void) nvlist_lookup_int64(nv, ZPOOL_CONFIG_REWIND_TIME, &loss);
 
 	if (localtime_r((time_t *)&rewindto, &t) != NULL &&
 	    strftime(timestr, 128, 0, &t) != 0) {
@@ -1255,6 +1270,7 @@ void
 zpool_explain_recover(libzfs_handle_t *hdl, const char *name, int reason,
     nvlist_t *config)
 {
+	nvlist_t *nv = NULL;
 	int64_t loss = -1;
 	uint64_t edata = UINT64_MAX;
 	uint64_t rewindto;
@@ -1270,12 +1286,12 @@ zpool_explain_recover(libzfs_handle_t *hdl, const char *name, int reason,
 		(void) printf(dgettext(TEXT_DOMAIN, "\t"));
 
 	/* All attempted rewinds failed if ZPOOL_CONFIG_LOAD_TIME missing */
-	if (nvlist_lookup_uint64(config,
-	    ZPOOL_CONFIG_LOAD_TIME, &rewindto) != 0)
+	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, &nv) != 0 ||
+	    nvlist_lookup_uint64(nv, ZPOOL_CONFIG_LOAD_TIME, &rewindto) != 0)
 		goto no_info;
 
-	(void) nvlist_lookup_int64(config, ZPOOL_CONFIG_REWIND_TIME, &loss);
-	(void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_LOAD_DATA_ERRORS,
+	(void) nvlist_lookup_int64(nv, ZPOOL_CONFIG_REWIND_TIME, &loss);
+	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_LOAD_DATA_ERRORS,
 	    &edata);
 
 	(void) printf(dgettext(TEXT_DOMAIN,
@@ -1359,12 +1375,40 @@ zpool_import(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 		}
 	}
 
-	ret = zpool_import_props(hdl, config, newname, props, B_FALSE);
+	ret = zpool_import_props(hdl, config, newname, props,
+	    ZFS_IMPORT_NORMAL);
 	if (props)
 		nvlist_free(props);
 	return (ret);
 }
 
+static void
+print_vdev_tree(libzfs_handle_t *hdl, const char *name, nvlist_t *nv,
+    int indent)
+{
+	nvlist_t **child;
+	uint_t c, children;
+	char *vname;
+	uint64_t is_log = 0;
+
+	(void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_IS_LOG,
+	    &is_log);
+
+	if (name != NULL)
+		(void) printf("\t%*s%s%s\n", indent, "", name,
+		    is_log ? " [log]" : "");
+
+	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
+	    &child, &children) != 0)
+		return;
+
+	for (c = 0; c < children; c++) {
+		vname = zpool_vdev_name(hdl, NULL, child[c], B_TRUE);
+		print_vdev_tree(hdl, vname, child[c], indent + 2);
+		free(vname);
+	}
+}
+
 /*
  * Import the given pool using the known configuration and a list of
  * properties to be set. The configuration should have come from
@@ -1373,15 +1417,17 @@ zpool_import(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
  */
 int
 zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
-    nvlist_t *props, boolean_t importfaulted)
+    nvlist_t *props, int flags)
 {
 	zfs_cmd_t zc = { 0 };
 	zpool_rewind_policy_t policy;
-	nvlist_t *nvi = NULL;
+	nvlist_t *nv = NULL;
+	nvlist_t *nvinfo = NULL;
+	nvlist_t *missing = NULL;
 	char *thename;
 	char *origname;
-	uint64_t returned_size;
 	int ret;
+	int error = 0;
 	char errbuf[1024];
 
 	verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
@@ -1402,12 +1448,13 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 
 	if (props) {
 		uint64_t version;
+		prop_flags_t flags = { .create = B_FALSE, .import = B_TRUE };
 
 		verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
 		    &version) == 0);
 
 		if ((props = zpool_valid_proplist(hdl, origname,
-		    props, version, B_TRUE, errbuf)) == NULL) {
+		    props, version, flags, errbuf)) == NULL) {
 			return (-1);
 		} else if (zcmd_write_src_nvlist(hdl, &zc, props) != 0) {
 			nvlist_free(props);
@@ -1424,27 +1471,36 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 		nvlist_free(props);
 		return (-1);
 	}
-	returned_size =  zc.zc_nvlist_conf_size + 512;
-	if (zcmd_alloc_dst_nvlist(hdl, &zc, returned_size) != 0) {
+	if (zcmd_alloc_dst_nvlist(hdl, &zc, zc.zc_nvlist_conf_size * 2) != 0) {
 		nvlist_free(props);
 		return (-1);
 	}
 
-	zc.zc_cookie = (uint64_t)importfaulted;
-	ret = 0;
-	if (zfs_ioctl(hdl, ZFS_IOC_POOL_IMPORT, &zc) != 0) {
+	zc.zc_cookie = flags;
+	while ((ret = zfs_ioctl(hdl, ZFS_IOC_POOL_IMPORT, &zc)) != 0 &&
+	    errno == ENOMEM) {
+		if (zcmd_expand_dst_nvlist(hdl, &zc) != 0) {
+			zcmd_free_nvlists(&zc);
+			return (-1);
+		}
+	}
+	if (ret != 0)
+		error = errno;
+
+	(void) zcmd_read_dst_nvlist(hdl, &zc, &nv);
+	zpool_get_rewind_policy(config, &policy);
+
+	if (error) {
 		char desc[1024];
 
-		(void) zcmd_read_dst_nvlist(hdl, &zc, &nvi);
-		zpool_get_rewind_policy(config, &policy);
 		/*
 		 * Dry-run failed, but we print out what success
 		 * looks like if we found a best txg
 		 */
-		if ((policy.zrp_request & ZPOOL_TRY_REWIND) && nvi) {
+		if (policy.zrp_request & ZPOOL_TRY_REWIND) {
 			zpool_rewind_exclaim(hdl, newname ? origname : thename,
-			    B_TRUE, nvi);
-			nvlist_free(nvi);
+			    B_TRUE, nv);
+			nvlist_free(nv);
 			return (-1);
 		}
 
@@ -1457,7 +1513,7 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 			    dgettext(TEXT_DOMAIN, "cannot import '%s' as '%s'"),
 			    origname, thename);
 
-		switch (errno) {
+		switch (error) {
 		case ENOTSUP:
 			/*
 			 * Unsupported version.
@@ -1475,15 +1531,32 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 			(void) zfs_error(hdl, EZFS_BADDEV, desc);
 			break;
 
+		case ENXIO:
+			if (nv && nvlist_lookup_nvlist(nv,
+			    ZPOOL_CONFIG_LOAD_INFO, &nvinfo) == 0 &&
+			    nvlist_lookup_nvlist(nvinfo,
+			    ZPOOL_CONFIG_MISSING_DEVICES, &missing) == 0) {
+				(void) printf(dgettext(TEXT_DOMAIN,
+				    "The devices below are missing, use "
+				    "'-m' to import the pool anyway:\n"));
+				print_vdev_tree(hdl, NULL, missing, 2);
+				(void) printf("\n");
+			}
+			(void) zpool_standard_error(hdl, error, desc);
+			break;
+
+		case EEXIST:
+			(void) zpool_standard_error(hdl, error, desc);
+			break;
+
 		default:
-			(void) zcmd_read_dst_nvlist(hdl, &zc, &nvi);
-			(void) zpool_standard_error(hdl, errno, desc);
+			(void) zpool_standard_error(hdl, error, desc);
 			zpool_explain_recover(hdl,
-			    newname ? origname : thename, -errno, nvi);
-			nvlist_free(nvi);
+			    newname ? origname : thename, -error, nv);
 			break;
 		}
 
+		nvlist_free(nv);
 		ret = -1;
 	} else {
 		zpool_handle_t *zhp;
@@ -1495,15 +1568,12 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 			ret = -1;
 		else if (zhp != NULL)
 			zpool_close(zhp);
-		(void) zcmd_read_dst_nvlist(hdl, &zc, &nvi);
-		zpool_get_rewind_policy(config, &policy);
 		if (policy.zrp_request &
 		    (ZPOOL_DO_REWIND | ZPOOL_TRY_REWIND)) {
 			zpool_rewind_exclaim(hdl, newname ? origname : thename,
-			    ((policy.zrp_request & ZPOOL_TRY_REWIND) != 0),
-			    nvi);
+			    ((policy.zrp_request & ZPOOL_TRY_REWIND) != 0), nv);
 		}
-		nvlist_free(nvi);
+		nvlist_free(nv);
 		return (0);
 	}
 
@@ -1526,7 +1596,7 @@ zpool_scan(zpool_handle_t *zhp, pool_scan_func_t func)
 	(void) strlcpy(zc.zc_name, zhp->zpool_name, sizeof (zc.zc_name));
 	zc.zc_cookie = func;
 
-	if (zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_POOL_SCAN, &zc) == 0 ||
+	if (zfs_ioctl(hdl, ZFS_IOC_POOL_SCAN, &zc) == 0 ||
 	    (errno == ENOENT && func != POOL_SCAN_NONE))
 		return (0);
 
@@ -1618,26 +1688,17 @@ vdev_to_nvlist_iter(nvlist_t *nv, nvlist_t *search, boolean_t *avail_spare,
 	srchkey = nvpair_name(pair);
 
 	switch (nvpair_type(pair)) {
-	case DATA_TYPE_UINT64: {
-		uint64_t srchval, theguid, present;
-
-		verify(nvpair_value_uint64(pair, &srchval) == 0);
+	case DATA_TYPE_UINT64:
 		if (strcmp(srchkey, ZPOOL_CONFIG_GUID) == 0) {
-			if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
-			    &present) == 0) {
-				/*
-				 * If the device has never been present since
-				 * import, the only reliable way to match the
-				 * vdev is by GUID.
-				 */
-				verify(nvlist_lookup_uint64(nv,
-				    ZPOOL_CONFIG_GUID, &theguid) == 0);
-				if (theguid == srchval)
-					return (nv);
-			}
+			uint64_t srchval, theguid;
+
+			verify(nvpair_value_uint64(pair, &srchval) == 0);
+			verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
+			    &theguid) == 0);
+			if (theguid == srchval)
+				return (nv);
 		}
 		break;
-	}
 
 	case DATA_TYPE_STRING: {
 		char *srchval, *val;
@@ -1819,6 +1880,9 @@ zpool_find_vdev_by_physpath(zpool_handle_t *zhp, const char *ppath,
 	    &nvroot) == 0);
 
 	*avail_spare = B_FALSE;
+	*l2cache = B_FALSE;
+	if (log != NULL)
+		*log = B_FALSE;
 	ret = vdev_to_nvlist_iter(nvroot, search, avail_spare, l2cache, log);
 	nvlist_free(search);
 
@@ -2114,14 +2178,14 @@ zpool_vdev_online(zpool_handle_t *zhp, const char *path, int flags,
 
 		if (wholedisk) {
 			pathname += strlen(DISK_ROOT) + 1;
-			(void) zpool_relabel_disk(zhp->zpool_hdl, pathname);
+			(void) zpool_relabel_disk(hdl, pathname);
 		}
 	}
 
 	zc.zc_cookie = VDEV_STATE_ONLINE;
 	zc.zc_obj = flags;
 
-	if (zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_VDEV_SET_STATE, &zc) != 0) {
+	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) != 0) {
 		if (errno == EINVAL) {
 			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "was split "
 			    "from this pool into a new one.  Use '%s' "
@@ -2163,7 +2227,7 @@ zpool_vdev_offline(zpool_handle_t *zhp, const char *path, boolean_t istmp)
 	zc.zc_cookie = VDEV_STATE_OFFLINE;
 	zc.zc_obj = istmp ? ZFS_OFFLINE_TEMPORARY : 0;
 
-	if (zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
+	if (zfs_ioctl(hdl, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
 		return (0);
 
 	switch (errno) {
@@ -2203,7 +2267,7 @@ zpool_vdev_fault(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
 	zc.zc_cookie = VDEV_STATE_FAULTED;
 	zc.zc_obj = aux;
 
-	if (ioctl(zhp->zpool_hdl->libzfs_fd, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
+	if (ioctl(hdl->libzfs_fd, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
 		return (0);
 
 	switch (errno) {
@@ -2238,7 +2302,7 @@ zpool_vdev_degrade(zpool_handle_t *zhp, uint64_t guid, vdev_aux_t aux)
 	zc.zc_cookie = VDEV_STATE_DEGRADED;
 	zc.zc_obj = aux;
 
-	if (ioctl(zhp->zpool_hdl->libzfs_fd, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
+	if (ioctl(hdl->libzfs_fd, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
 		return (0);
 
 	return (zpool_standard_error(hdl, errno, msg));
@@ -2286,7 +2350,7 @@ zpool_vdev_attach(zpool_handle_t *zhp,
 	nvlist_t *tgt;
 	boolean_t avail_spare, l2cache, islog;
 	uint64_t val;
-	char *path, *newname;
+	char *newname;
 	nvlist_t **child;
 	uint_t children;
 	nvlist_t *config_root;
@@ -2352,42 +2416,18 @@ zpool_vdev_attach(zpool_handle_t *zhp,
 		return (zfs_error(hdl, EZFS_BADTARGET, msg));
 	}
 
-	/*
-	 * If we are attempting to replace a spare, it canot be applied to an
-	 * already spared device.
-	 */
-	if (replacing &&
-	    nvlist_lookup_string(child[0], ZPOOL_CONFIG_PATH, &path) == 0 &&
-	    zpool_find_vdev(zhp, newname, &avail_spare,
-	    &l2cache, NULL) != NULL && avail_spare &&
-	    is_replacing_spare(config_root, tgt, 0)) {
-		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-		    "device has already been replaced with a spare"));
-		free(newname);
-		return (zfs_error(hdl, EZFS_BADTARGET, msg));
-	}
-
 	free(newname);
 
 	if (zcmd_write_conf_nvlist(hdl, &zc, nvroot) != 0)
 		return (-1);
 
-	ret = zfs_ioctl(zhp->zpool_hdl, ZFS_IOC_VDEV_ATTACH, &zc);
+	ret = zfs_ioctl(hdl, ZFS_IOC_VDEV_ATTACH, &zc);
 
 	zcmd_free_nvlists(&zc);
 
 	if (ret == 0) {
 		if (rootpool) {
 			/*
-			 * XXX - This should be removed once we can
-			 * automatically install the bootblocks on the
-			 * newly attached disk.
-			 */
-			(void) fprintf(stderr, dgettext(TEXT_DOMAIN, "Please "
-			    "be sure to invoke %s to make '%s' bootable.\n"),
-			    BOOTCMD, new_disk);
-
-			/*
 			 * XXX need a better way to prevent user from
 			 * booting up a half-baked vdev.
 			 */
@@ -2404,9 +2444,16 @@ zpool_vdev_attach(zpool_handle_t *zhp,
 		 * Can't attach to or replace this type of vdev.
 		 */
 		if (replacing) {
+			uint64_t version = zpool_get_prop_int(zhp,
+			    ZPOOL_PROP_VERSION, NULL);
+
 			if (islog)
 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 				    "cannot replace a log with a spare"));
+			else if (version >= SPA_VERSION_MULTI_REPLACE)
+				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+				    "already in replacing/spare config; wait "
+				    "for completion or use 'zpool detach'"));
 			else
 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 				    "cannot replace a replacing device"));
@@ -2504,7 +2551,7 @@ zpool_vdev_detach(zpool_handle_t *zhp, const char *path)
 		 */
 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "only "
 		    "applicable to mirror and replacing vdevs"));
-		(void) zfs_error(zhp->zpool_hdl, EZFS_BADTARGET, msg);
+		(void) zfs_error(hdl, EZFS_BADTARGET, msg);
 		break;
 
 	case EBUSY:
@@ -2596,8 +2643,9 @@ zpool_vdev_split(zpool_handle_t *zhp, char *newname, nvlist_t **newroot,
 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, &vers) == 0);
 
 	if (props) {
+		prop_flags_t flags = { .create = B_FALSE, .import = B_TRUE };
 		if ((zc_props = zpool_valid_proplist(hdl, zhp->zpool_name,
-		    props, vers, B_TRUE, msg)) == NULL)
+		    props, vers, flags, msg)) == NULL)
 			return (-1);
 	}
 
@@ -2831,6 +2879,7 @@ zpool_clear(zpool_handle_t *zhp, const char *path, nvlist_t *rewindnvl)
 	boolean_t avail_spare, l2cache;
 	libzfs_handle_t *hdl = zhp->zpool_hdl;
 	nvlist_t *nvi = NULL;
+	int error;
 
 	if (path)
 		(void) snprintf(msg, sizeof (msg),
@@ -2861,14 +2910,21 @@ zpool_clear(zpool_handle_t *zhp, const char *path, nvlist_t *rewindnvl)
 	zpool_get_rewind_policy(rewindnvl, &policy);
 	zc.zc_cookie = policy.zrp_request;
 
-	if (zcmd_alloc_dst_nvlist(hdl, &zc, 8192) != 0)
+	if (zcmd_alloc_dst_nvlist(hdl, &zc, zhp->zpool_config_size * 2) != 0)
 		return (-1);
 
-	if (zcmd_write_src_nvlist(zhp->zpool_hdl, &zc, rewindnvl) != 0)
+	if (zcmd_write_src_nvlist(hdl, &zc, rewindnvl) != 0)
 		return (-1);
 
-	if (zfs_ioctl(hdl, ZFS_IOC_CLEAR, &zc) == 0 ||
-	    ((policy.zrp_request & ZPOOL_TRY_REWIND) &&
+	while ((error = zfs_ioctl(hdl, ZFS_IOC_CLEAR, &zc)) != 0 &&
+	    errno == ENOMEM) {
+		if (zcmd_expand_dst_nvlist(hdl, &zc) != 0) {
+			zcmd_free_nvlists(&zc);
+			return (-1);
+		}
+	}
+
+	if (!error || ((policy.zrp_request & ZPOOL_TRY_REWIND) &&
 	    errno != EPERM && errno != EACCES)) {
 		if (policy.zrp_request &
 		    (ZPOOL_DO_REWIND | ZPOOL_TRY_REWIND)) {
diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c
index 672e004ef..3093ab974 100644
--- a/lib/libzfs/libzfs_sendrecv.c
+++ b/lib/libzfs/libzfs_sendrecv.c
@@ -51,7 +51,7 @@
 extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *);
 
 static int zfs_receive_impl(libzfs_handle_t *, const char *, recvflags_t,
-    int, const char *, nvlist_t *, avl_tree_t *, char **);
+    int, const char *, nvlist_t *, avl_tree_t *, char **, int, uint64_t *);
 
 static const zio_cksum_t zero_cksum = { 0 };
 
@@ -782,14 +782,30 @@ static int
 zfs_sort_snaps(zfs_handle_t *zhp, void *data)
 {
 	avl_tree_t *avl = data;
-	zfs_node_t *node = zfs_alloc(zhp->zfs_hdl, sizeof (zfs_node_t));
+	zfs_node_t *node;
+	zfs_node_t search;
+
+	search.zn_handle = zhp;
+	node = avl_find(avl, &search, NULL);
+	if (node) {
+		/*
+		 * If this snapshot was renamed while we were creating the
+		 * AVL tree, it's possible that we already inserted it under
+		 * its old name. Remove the old handle before adding the new
+		 * one.
+		 */
+		zfs_close(node->zn_handle);
+		avl_remove(avl, node);
+		free(node);
+	}
 
+	node = zfs_alloc(zhp->zfs_hdl, sizeof (zfs_node_t));
 	node->zn_handle = zhp;
 	avl_add(avl, node);
+
 	return (0);
 }
 
-/* ARGSUSED */
 static int
 zfs_snapshot_compare(const void *larg, const void *rarg)
 {
@@ -844,6 +860,7 @@ typedef struct send_dump_data {
 	const char *fromsnap;
 	const char *tosnap;
 	char prevsnap[ZFS_MAXNAMELEN];
+	uint64_t prevsnap_obj;
 	boolean_t seenfrom, seento, replicate, doall, fromorigin;
 	boolean_t verbose;
 	int outfd;
@@ -853,6 +870,8 @@ typedef struct send_dump_data {
 	snapfilter_cb_t *filter_cb;
 	void *filter_cb_arg;
 	nvlist_t *debugnv;
+	char holdtag[ZFS_MAXNAMELEN];
+	int cleanup_fd;
 } send_dump_data_t;
 
 /*
@@ -860,23 +879,21 @@ typedef struct send_dump_data {
  * NULL) to the file descriptor specified by outfd.
  */
 static int
-dump_ioctl(zfs_handle_t *zhp, const char *fromsnap, boolean_t fromorigin,
-    int outfd, boolean_t enoent_ok, boolean_t *got_enoent, nvlist_t *debugnv)
+dump_ioctl(zfs_handle_t *zhp, const char *fromsnap, uint64_t fromsnap_obj,
+    boolean_t fromorigin, int outfd, nvlist_t *debugnv)
 {
 	zfs_cmd_t zc = { 0 };
 	libzfs_handle_t *hdl = zhp->zfs_hdl;
 	nvlist_t *thisdbg;
 
 	assert(zhp->zfs_type == ZFS_TYPE_SNAPSHOT);
-	assert(fromsnap == NULL || fromsnap[0] == '\0' || !fromorigin);
+	assert(fromsnap_obj == 0 || !fromorigin);
 
 	(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
-	if (fromsnap)
-		(void) strlcpy(zc.zc_value, fromsnap, sizeof (zc.zc_value));
 	zc.zc_cookie = outfd;
 	zc.zc_obj = fromorigin;
-
-	*got_enoent = B_FALSE;
+	zc.zc_sendobj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID);
+	zc.zc_fromobj = fromsnap_obj;
 
 	VERIFY(0 == nvlist_alloc(&thisdbg, NV_UNIQUE_NAME, 0));
 	if (fromsnap && fromsnap[0] != '\0') {
@@ -904,10 +921,6 @@ dump_ioctl(zfs_handle_t *zhp, const char *fromsnap, boolean_t fromorigin,
 			return (zfs_error(hdl, EZFS_CROSSTARGET, errbuf));
 
 		case ENOENT:
-			if (enoent_ok) {
-				*got_enoent = B_TRUE;
-				return (0);
-			}
 			if (zfs_dataset_exists(hdl, zc.zc_name,
 			    ZFS_TYPE_SNAPSHOT)) {
 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
@@ -943,12 +956,47 @@ dump_ioctl(zfs_handle_t *zhp, const char *fromsnap, boolean_t fromorigin,
 }
 
 static int
+hold_for_send(zfs_handle_t *zhp, send_dump_data_t *sdd)
+{
+	zfs_handle_t *pzhp;
+	int error = 0;
+	char *thissnap;
+
+	assert(zhp->zfs_type == ZFS_TYPE_SNAPSHOT);
+
+	/*
+	 * zfs_send() only opens a cleanup_fd for sends that need it,
+	 * e.g. replication and doall.
+	 */
+	if (sdd->cleanup_fd == -1)
+		return (0);
+
+	thissnap = strchr(zhp->zfs_name, '@') + 1;
+	*(thissnap - 1) = '\0';
+	pzhp = zfs_open(zhp->zfs_hdl, zhp->zfs_name, ZFS_TYPE_DATASET);
+	*(thissnap - 1) = '@';
+
+	/*
+	 * It's OK if the parent no longer exists.  The send code will
+	 * handle that error.
+	 */
+	if (pzhp) {
+		error = zfs_hold(pzhp, thissnap, sdd->holdtag,
+		    B_FALSE, B_TRUE, B_TRUE, sdd->cleanup_fd,
+		    zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID),
+		    zfs_prop_get_int(zhp, ZFS_PROP_CREATETXG));
+		zfs_close(pzhp);
+	}
+
+	return (error);
+}
+
+static int
 dump_snapshot(zfs_handle_t *zhp, void *arg)
 {
 	send_dump_data_t *sdd = arg;
-	const char *thissnap;
+	char *thissnap;
 	int err;
-	boolean_t got_enoent;
 	boolean_t isfromsnap, istosnap;
 	boolean_t exclude = B_FALSE;
 
@@ -957,10 +1005,17 @@ dump_snapshot(zfs_handle_t *zhp, void *arg)
 	    strcmp(sdd->fromsnap, thissnap) == 0);
 
 	if (!sdd->seenfrom && isfromsnap) {
-		sdd->seenfrom = B_TRUE;
-		(void) strcpy(sdd->prevsnap, thissnap);
+		err = hold_for_send(zhp, sdd);
+		if (err == 0) {
+			sdd->seenfrom = B_TRUE;
+			(void) strcpy(sdd->prevsnap, thissnap);
+			sdd->prevsnap_obj = zfs_prop_get_int(zhp,
+			    ZFS_PROP_OBJSETID);
+		} else if (err == ENOENT) {
+			err = 0;
+		}
 		zfs_close(zhp);
-		return (0);
+		return (err);
 	}
 
 	if (sdd->seento || !sdd->seenfrom) {
@@ -1001,7 +1056,7 @@ dump_snapshot(zfs_handle_t *zhp, void *arg)
 	    sdd->filter_cb(zhp, sdd->filter_cb_arg) == B_FALSE)) {
 		/*
 		 * This snapshot is filtered out.  Don't send it, and don't
-		 * set prevsnap, so it will be as if this snapshot didn't
+		 * set prevsnap_obj, so it will be as if this snapshot didn't
 		 * exist, and the next accepted snapshot will be sent as
 		 * an incremental from the last accepted one, or as the
 		 * first (and full) snapshot in the case of a replication,
@@ -1011,20 +1066,26 @@ dump_snapshot(zfs_handle_t *zhp, void *arg)
 		return (0);
 	}
 
+	err = hold_for_send(zhp, sdd);
+	if (err) {
+		if (err == ENOENT)
+			err = 0;
+		zfs_close(zhp);
+		return (err);
+	}
+
 	/* send it */
 	if (sdd->verbose) {
 		(void) fprintf(stderr, "sending from @%s to %s\n",
 		    sdd->prevsnap, zhp->zfs_name);
 	}
 
-	err = dump_ioctl(zhp, sdd->prevsnap,
+	err = dump_ioctl(zhp, sdd->prevsnap, sdd->prevsnap_obj,
 	    sdd->prevsnap[0] == '\0' && (sdd->fromorigin || sdd->replicate),
-	    sdd->outfd, B_TRUE, &got_enoent, sdd->debugnv);
+	    sdd->outfd, sdd->debugnv);
 
-	if (got_enoent)
-		err = 0;
-	else
-		(void) strcpy(sdd->prevsnap, thissnap);
+	(void) strcpy(sdd->prevsnap, thissnap);
+	sdd->prevsnap_obj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID);
 	zfs_close(zhp);
 	return (err);
 }
@@ -1064,6 +1125,7 @@ dump_filesystem(zfs_handle_t *zhp, void *arg)
 	}
 
 	sdd->seenfrom = sdd->seento = sdd->prevsnap[0] = 0;
+	sdd->prevsnap_obj = 0;
 	if (sdd->fromsnap == NULL || missingfrom)
 		sdd->seenfrom = B_TRUE;
 
@@ -1202,7 +1264,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 	int err;
 	nvlist_t *fss = NULL;
 	avl_tree_t *fsavl = NULL;
-	char holdtag[128];
 	static uint64_t holdseq;
 	int spa_version;
 	boolean_t holdsnaps = B_FALSE;
@@ -1211,14 +1272,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 	dedup_arg_t dda = { 0 };
 	int featureflags = 0;
 
-	if (zhp->zfs_type == ZFS_TYPE_FILESYSTEM) {
-		uint64_t version;
-		version = zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
-		if (version >= ZPL_VERSION_SA) {
-			featureflags |= DMU_BACKUP_FEATURE_SA_SPILL;
-		}
-	}
-
 	(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
 	    "cannot send '%s'"), zhp->zfs_name);
 
@@ -1228,8 +1281,17 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 		return (zfs_error(zhp->zfs_hdl, EZFS_NOENT, errbuf));
 	}
 
+	if (zhp->zfs_type == ZFS_TYPE_FILESYSTEM) {
+		uint64_t version;
+		version = zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
+		if (version >= ZPL_VERSION_SA) {
+			featureflags |= DMU_BACKUP_FEATURE_SA_SPILL;
+		}
+	}
+
 	if (zfs_spa_version(zhp, &spa_version) == 0 &&
-	    spa_version >= SPA_VERSION_USERREFS)
+	    spa_version >= SPA_VERSION_USERREFS &&
+	    (flags.doall || flags.replicate))
 		holdsnaps = B_TRUE;
 
 	if (flags.dedup) {
@@ -1258,17 +1320,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 		size_t buflen = 0;
 		zio_cksum_t zc = { 0 };
 
-		if (holdsnaps) {
-			(void) snprintf(holdtag, sizeof (holdtag),
-			    ".send-%d-%llu", getpid(), (u_longlong_t)holdseq);
-			++holdseq;
-			err = zfs_hold_range(zhp, fromsnap, tosnap,
-			    holdtag, flags.replicate, B_TRUE, filter_func,
-			    cb_arg);
-			if (err)
-				goto err_out;
-		}
-
 		if (flags.replicate || flags.props) {
 			nvlist_t *hdrnv;
 
@@ -1285,13 +1336,8 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 
 			err = gather_nvlist(zhp->zfs_hdl, zhp->zfs_name,
 			    fromsnap, tosnap, flags.replicate, &fss, &fsavl);
-			if (err) {
-				if (holdsnaps) {
-					(void) zfs_release_range(zhp, fromsnap,
-					    tosnap, holdtag, flags.replicate);
-				}
+			if (err)
 				goto err_out;
-			}
 			VERIFY(0 == nvlist_add_nvlist(hdrnv, "fss", fss));
 			err = nvlist_pack(hdrnv, &packbuf, &buflen,
 			    NV_ENCODE_XDR, 0);
@@ -1302,10 +1348,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 			if (err) {
 				fsavl_destroy(fsavl);
 				nvlist_free(fss);
-				if (holdsnaps) {
-					(void) zfs_release_range(zhp, fromsnap,
-					    tosnap, holdtag, flags.replicate);
-				}
 				goto stderr_out;
 			}
 		}
@@ -1331,10 +1373,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 		if (err == -1) {
 			fsavl_destroy(fsavl);
 			nvlist_free(fss);
-			if (holdsnaps) {
-				(void) zfs_release_range(zhp, fromsnap, tosnap,
-				    holdtag, flags.replicate);
-			}
 			err = errno;
 			goto stderr_out;
 		}
@@ -1349,10 +1387,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 				fsavl_destroy(fsavl);
 				nvlist_free(fss);
 				err = errno;
-				if (holdsnaps) {
-					(void) zfs_release_range(zhp, fromsnap,
-					    tosnap, holdtag, flags.replicate);
-				}
 				goto stderr_out;
 			}
 		}
@@ -1375,6 +1409,18 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 	sdd.filter_cb_arg = cb_arg;
 	if (debugnvp)
 		sdd.debugnv = *debugnvp;
+	if (holdsnaps) {
+		++holdseq;
+		(void) snprintf(sdd.holdtag, sizeof (sdd.holdtag),
+		    ".send-%d-%llu", getpid(), (u_longlong_t)holdseq);
+		sdd.cleanup_fd = open(ZFS_DEV, O_RDWR|O_EXCL);
+		if (sdd.cleanup_fd < 0) {
+			err = errno;
+			goto stderr_out;
+		}
+	} else {
+		sdd.cleanup_fd = -1;
+	}
 	err = dump_filesystems(zhp, &sdd);
 	fsavl_destroy(fsavl);
 	nvlist_free(fss);
@@ -1384,6 +1430,11 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 		(void) pthread_join(tid, NULL);
 	}
 
+	if (sdd.cleanup_fd != -1) {
+		VERIFY(0 == close(sdd.cleanup_fd));
+		sdd.cleanup_fd = -1;
+	}
+
 	if (flags.replicate || flags.doall || flags.props) {
 		/*
 		 * write final end record.  NB: want to do this even if
@@ -1392,10 +1443,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 		 */
 		dmu_replay_record_t drr = { 0 };
 		drr.drr_type = DRR_END;
-		if (holdsnaps) {
-			(void) zfs_release_range(zhp, fromsnap, tosnap,
-			    holdtag, flags.replicate);
-		}
 		if (write(outfd, &drr, sizeof (drr)) == -1) {
 			return (zfs_standard_error(zhp->zfs_hdl,
 			    errno, errbuf));
@@ -1407,6 +1454,8 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
 stderr_out:
 	err = zfs_standard_error(zhp->zfs_hdl, err, errbuf);
 err_out:
+	if (sdd.cleanup_fd != -1)
+		VERIFY(0 == close(sdd.cleanup_fd));
 	if (flags.dedup) {
 		(void) pthread_cancel(tid);
 		(void) pthread_join(tid, NULL);
@@ -1992,7 +2041,7 @@ again:
 static int
 zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname,
     recvflags_t flags, dmu_replay_record_t *drr, zio_cksum_t *zc,
-    char **top_zfs)
+    char **top_zfs, int cleanup_fd, uint64_t *action_handlep)
 {
 	nvlist_t *stream_nv = NULL;
 	avl_tree_t *stream_avl = NULL;
@@ -2158,7 +2207,8 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname,
 		 * recv_skip() and return 0).
 		 */
 		error = zfs_receive_impl(hdl, destname, flags, fd,
-		    sendfs, stream_nv, stream_avl, top_zfs);
+		    sendfs, stream_nv, stream_avl, top_zfs, cleanup_fd,
+		    action_handlep);
 		if (error == ENODATA) {
 			error = 0;
 			break;
@@ -2281,7 +2331,8 @@ static int
 zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
     recvflags_t flags, dmu_replay_record_t *drr,
     dmu_replay_record_t *drr_noswap, const char *sendfs,
-    nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs)
+    nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd,
+    uint64_t *action_handlep)
 {
 	zfs_cmd_t zc = { 0 };
 	time_t begin_time;
@@ -2609,6 +2660,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 
 	zc.zc_nvlist_dst = (uint64_t)(uintptr_t)prop_errbuf;
 	zc.zc_nvlist_dst_size = sizeof (prop_errbuf);
+	zc.zc_cleanup_fd = cleanup_fd;
+	zc.zc_action_handle = *action_handlep;
 
 	err = ioctl_err = zfs_ioctl(hdl, ZFS_IOC_RECV, &zc);
 	ioctl_errno = errno;
@@ -2796,6 +2849,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 	if (err || ioctl_err)
 		return (-1);
 
+	*action_handlep = zc.zc_action_handle;
+
 	if (flags.verbose) {
 		char buf1[64];
 		char buf2[64];
@@ -2816,7 +2871,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
 static int
 zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, recvflags_t flags,
     int infd, const char *sendfs, nvlist_t *stream_nv, avl_tree_t *stream_avl,
-    char **top_zfs)
+    char **top_zfs, int cleanup_fd, uint64_t *action_handlep)
 {
 	int err;
 	dmu_replay_record_t drr, drr_noswap;
@@ -2909,12 +2964,12 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, recvflags_t flags,
 		}
 		return (zfs_receive_one(hdl, infd, tosnap, flags,
 		    &drr, &drr_noswap, sendfs, stream_nv, stream_avl,
-		    top_zfs));
+		    top_zfs, cleanup_fd, action_handlep));
 	} else {
 		assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
 		    DMU_COMPOUNDSTREAM);
 		return (zfs_receive_package(hdl, infd, tosnap, flags,
-		    &drr, &zcksum, top_zfs));
+		    &drr, &zcksum, top_zfs, cleanup_fd, action_handlep));
 	}
 }
 
@@ -2930,9 +2985,16 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, recvflags_t flags,
 {
 	char *top_zfs = NULL;
 	int err;
+	int cleanup_fd;
+	uint64_t action_handle = 0;
+
+	cleanup_fd = open(ZFS_DEV, O_RDWR|O_EXCL);
+	VERIFY(cleanup_fd >= 0);
 
 	err = zfs_receive_impl(hdl, tosnap, flags, infd, NULL, NULL,
-	    stream_avl, &top_zfs);
+	    stream_avl, &top_zfs, cleanup_fd, &action_handle);
+
+	VERIFY(0 == close(cleanup_fd));
 
 	if (err == 0 && !flags.nomount && top_zfs) {
 		zfs_handle_t *zhp;
diff --git a/lib/libzfs/libzfs_util.c b/lib/libzfs/libzfs_util.c
index 2e73f76ea..01b7c8732 100644
--- a/lib/libzfs/libzfs_util.c
+++ b/lib/libzfs/libzfs_util.c
@@ -69,7 +69,7 @@ libzfs_error_description(libzfs_handle_t *hdl)
 	case EZFS_BADPROP:
 		return (dgettext(TEXT_DOMAIN, "invalid property value"));
 	case EZFS_PROPREADONLY:
-		return (dgettext(TEXT_DOMAIN, "read only property"));
+		return (dgettext(TEXT_DOMAIN, "read-only property"));
 	case EZFS_PROPTYPE:
 		return (dgettext(TEXT_DOMAIN, "property doesn't apply to "
 		    "datasets of this type"));
@@ -89,7 +89,7 @@ libzfs_error_description(libzfs_handle_t *hdl)
 	case EZFS_BADSTREAM:
 		return (dgettext(TEXT_DOMAIN, "invalid backup stream"));
 	case EZFS_DSREADONLY:
-		return (dgettext(TEXT_DOMAIN, "dataset is read only"));
+		return (dgettext(TEXT_DOMAIN, "dataset is read-only"));
 	case EZFS_VOLTOOBIG:
 		return (dgettext(TEXT_DOMAIN, "volume size exceeds limit for "
 		    "this system"));
@@ -181,9 +181,6 @@ libzfs_error_description(libzfs_handle_t *hdl)
 	case EZFS_NODELEGATION:
 		return (dgettext(TEXT_DOMAIN, "delegated administration is "
 		    "disabled on pool"));
-	case EZFS_PERMRDONLY:
-		return (dgettext(TEXT_DOMAIN, "snapshot permissions cannot be"
-		    " modified"));
 	case EZFS_BADCACHE:
 		return (dgettext(TEXT_DOMAIN, "invalid or missing cache file"));
 	case EZFS_ISL2CACHE:
@@ -219,6 +216,12 @@ libzfs_error_description(libzfs_handle_t *hdl)
 		    "use 'zpool scrub -s' to cancel current scrub"));
 	case EZFS_NO_SCRUB:
 		return (dgettext(TEXT_DOMAIN, "there is no active scrub"));
+	case EZFS_DIFF:
+		return (dgettext(TEXT_DOMAIN, "unable to generate diffs"));
+	case EZFS_DIFFDATA:
+		return (dgettext(TEXT_DOMAIN, "invalid diff data"));
+	case EZFS_POOLREADONLY:
+		return (dgettext(TEXT_DOMAIN, "pool is read-only"));
 	case EZFS_UNKNOWN:
 		return (dgettext(TEXT_DOMAIN, "unknown error"));
 	default:
@@ -367,9 +370,7 @@ zfs_standard_error_fmt(libzfs_handle_t *hdl, int error, const char *fmt, ...)
 		zfs_verror(hdl, EZFS_BUSY, fmt, ap);
 		break;
 	case EROFS:
-		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-		    "snapshot permissions cannot be modified"));
-		zfs_verror(hdl, EZFS_PERMRDONLY, fmt, ap);
+		zfs_verror(hdl, EZFS_POOLREADONLY, fmt, ap);
 		break;
 	case ENAMETOOLONG:
 		zfs_verror(hdl, EZFS_NAMETOOLONG, fmt, ap);
@@ -455,12 +456,17 @@ zpool_standard_error_fmt(libzfs_handle_t *hdl, int error, const char *fmt, ...)
 	case EDQUOT:
 		zfs_verror(hdl, EZFS_NOSPC, fmt, ap);
 		return (-1);
+
 	case EAGAIN:
 		zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
 		    "pool I/O is currently suspended"));
 		zfs_verror(hdl, EZFS_POOLUNAVAIL, fmt, ap);
 		break;
 
+	case EROFS:
+		zfs_verror(hdl, EZFS_POOLREADONLY, fmt, ap);
+		break;
+
 	default:
 		zfs_error_aux(hdl, strerror(error));
 		zfs_verror(hdl, EZFS_UNKNOWN, fmt, ap);
@@ -494,6 +500,29 @@ zfs_alloc(libzfs_handle_t *hdl, size_t size)
 }
 
 /*
+ * A safe form of asprintf() which will die if the allocation fails.
+ */
+/*PRINTFLIKE2*/
+char *
+zfs_asprintf(libzfs_handle_t *hdl, const char *fmt, ...)
+{
+	va_list ap;
+	char *ret;
+	int err;
+
+	va_start(ap, fmt);
+
+	err = vasprintf(&ret, fmt, ap);
+
+	va_end(ap);
+
+	if (err < 0)
+		(void) no_memory(hdl);
+
+	return (ret);
+}
+
+/*
  * A safe form of realloc(), which also zeroes newly allocated space.
  */
 void *
@@ -579,7 +608,7 @@ libzfs_init(void)
 {
 	libzfs_handle_t *hdl;
 
-	if ((hdl = calloc(sizeof (libzfs_handle_t), 1)) == NULL) {
+	if ((hdl = calloc(1, sizeof (libzfs_handle_t))) == NULL) {
 		return (NULL);
 	}
 
@@ -692,7 +721,7 @@ int
 zcmd_alloc_dst_nvlist(libzfs_handle_t *hdl, zfs_cmd_t *zc, size_t len)
 {
 	if (len == 0)
-		len = 4*1024;
+		len = 16 * 1024;
 	zc->zc_nvlist_dst_size = len;
 	if ((zc->zc_nvlist_dst = (uint64_t)(uintptr_t)
 	    zfs_alloc(hdl, zc->zc_nvlist_dst_size)) == NULL)