Prebaked scripts for zpool status/iostat -c

This patch updates the "zpool status/iostat -c" commands to only run
"pre-baked" scripts from the /etc/zfs/zpool.d directory (or wherever
you install to).  The scripts can only be run from -c as an unprivileged
user (unless the ZPOOL_SCRIPTS_AS_ROOT environment var is
set by root).  This was done to encourage scripts to be written is such
a way that normal users can use them, and to be cautious.  If your
script needs to run a privileged command, consider adding the
appropriate line in /etc/sudoers.  See zpool(8) for an example of how
to do this.

The patch also allows the scripts to output custom column names.  If
the script outputs a line like:

name=value

then "name" is used for the column name, and "value" is its value.
Multiple columns can be specified by outputting multiple lines.  Column
names and values can have spaces.  If the value is empty, a dash (-) is
printed instead.

After all the "name=value" lines are read (if any), zpool will take the
next the next line of output (if any) and print it without a column
header.  After that, no more lines will be processed. This can be
useful for printing errors.

Lastly, this patch also disables the -c option with the latency and
request size histograms, since it produced awkward output and made the
code harder to maintain.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Giuseppe Di Natale <[email protected]>
Signed-off-by: Tony Hutter <[email protected]>
Closes #5852 

22 files changed, 816 insertions, 65 deletions

diff --git a/cmd/zpool/Makefile.am b/cmd/zpool/Makefile.am
index b4ff106e1..f905fb73d 100644
--- a/cmd/zpool/Makefile.am
+++ b/cmd/zpool/Makefile.am
@@ -20,3 +20,54 @@ zpool_LDADD = \
 	$(top_builddir)/lib/libzfs/libzfs.la \
 	$(top_builddir)/lib/libzfs_core/libzfs_core.la \
 	-lm $(LIBBLKID)
+
+zpoolconfdir = $(sysconfdir)/zfs/zpool.d
+zpoolexecdir = $(libexecdir)/zfs/zpool.d
+
+EXTRA_DIST = zpool.d/README
+
+dist_zpoolexec_SCRIPTS = \
+	zpool.d/enc \
+	zpool.d/encdev \
+	zpool.d/fault_led \
+	zpool.d/iostat \
+	zpool.d/iostat-1s \
+	zpool.d/iostat-10s \
+	zpool.d/label \
+	zpool.d/locate_led \
+	zpool.d/lsblk \
+	zpool.d/model \
+	zpool.d/serial \
+	zpool.d/ses \
+	zpool.d/size \
+	zpool.d/slaves \
+	zpool.d/slot \
+	zpool.d/upath \
+	zpool.d/vendor
+
+zpoolconfdefaults = \
+	enc \
+	encdev \
+	fault_led \
+	iostat \
+	iostat-1s \
+	iostat-10s \
+	label \
+	locate_led \
+	lsblk \
+	model \
+	serial \
+	ses \
+	size \
+	slaves \
+	slot \
+	upath \
+	vendor
+
+install-data-hook:
+	$(MKDIR_P) "$(DESTDIR)$(zpoolconfdir)"
+	for f in $(zpoolconfdefaults); do \
+	  test -f "$(DESTDIR)$(zpoolconfdir)/$${f}" -o \
+	       -L "$(DESTDIR)$(zpoolconfdir)/$${f}" || \
+	    ln -s "$(zpoolexecdir)/$${f}" "$(DESTDIR)$(zpoolconfdir)"; \
+	done
diff --git a/cmd/zpool/zpool.d/README b/cmd/zpool/zpool.d/README
new file mode 100644
index 000000000..033b7c363
--- /dev/null
+++ b/cmd/zpool/zpool.d/README
@@ -0,0 +1,9 @@
+This directory contains scripts that can be run the zpool status/iostat
+-c option:
+
+	zpool status -c script1,script2, ...
+
+	zpool iostat -vc script1,script2, ...
+
+Some scripts output different values depending on the symlink name that is
+used to run them.  See the zpool(8) man page for more details.
diff --git a/cmd/zpool/zpool.d/enc b/cmd/zpool/zpool.d/enc
new file mode 120000
index 000000000..478d1e896
--- /dev/null
+++ b/cmd/zpool/zpool.d/enc
@@ -0,0 +1 @@
+ses
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/encdev b/cmd/zpool/zpool.d/encdev
new file mode 120000
index 000000000..478d1e896
--- /dev/null
+++ b/cmd/zpool/zpool.d/encdev
@@ -0,0 +1 @@
+ses
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/fault_led b/cmd/zpool/zpool.d/fault_led
new file mode 120000
index 000000000..478d1e896
--- /dev/null
+++ b/cmd/zpool/zpool.d/fault_led
@@ -0,0 +1 @@
+ses
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/iostat b/cmd/zpool/zpool.d/iostat
new file mode 100755
index 000000000..f6452fb25
--- /dev/null
+++ b/cmd/zpool/zpool.d/iostat
@@ -0,0 +1,65 @@
+#!/bin/sh
+#
+# Display most relevant iostat bandwidth/latency numbers.  The output is
+# dependent on the name of the script/symlink used to call it.
+#
+
+helpstr="
+iostat:		Show iostat values since boot (summary page).
+iostat-1s:	Do a single 1-second iostat sample and show values.
+iostat-10s:	Do a single 10-second iostat sample and show values."
+
+script=$(basename "$0")
+if [ "$1" = "-h" ] ; then
+	echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
+	exit
+fi
+
+if [ "$script" = "iostat-1s" ] ; then
+	# Do a single one-second sample
+	extra="1 1"
+	# Don't show summary stats
+	y="-y"
+elif [ "$script" = "iostat-10s" ] ; then
+	# Do a single ten-second sample
+	extra="10 1"
+	# Don't show summary stats
+	y="-y"
+fi
+
+if [ -f "$VDEV_UPATH" ] ; then
+	# We're a file-based vdev, iostat doesn't work on us.  Do nothing.
+	exit
+fi
+
+out=$(eval "iostat $y -k -x $VDEV_UPATH $extra")
+
+# Sample output (we want the last two lines):
+#
+# Linux 2.6.32-642.13.1.el6.x86_64 (centos68) 	03/09/2017 	_x86_64_	(6 CPU)
+#
+# avg-cpu:  %user   %nice %system %iowait  %steal   %idle
+#           0.00    0.00    0.00    0.00    0.00  100.00
+#
+# Device:         rrqm/s   wrqm/s     r/s     w/s    rkB/s    wkB/s avgrq-sz avgqu-sz   await r_await w_await  svctm  %util
+# sdb               0.00     0.00    0.00    0.00     0.00     0.00     0.00     0.00    0.00    0.00    0.00   0.00   0.00
+#
+
+# Get the column names
+cols=$(echo "$out" | grep Device)
+
+# Get the values and tab separate them to make them cut-able.
+vals="$(echo "$out" | grep -A1 Device | tail -n 1 | sed -r 's/[[:blank:]]+/\t/g')"
+
+i=0
+for col in $cols ; do
+	i=$((i+1))
+	# Skip the first column since it's just the device name
+	if [ "$col" = "Device:" ] ; then
+		continue
+	fi
+
+	# Get i'th value
+	val=$(echo "$vals" | cut -f "$i")
+	echo "$col=$val"
+done
diff --git a/cmd/zpool/zpool.d/iostat-10s b/cmd/zpool/zpool.d/iostat-10s
new file mode 120000
index 000000000..084278d99
--- /dev/null
+++ b/cmd/zpool/zpool.d/iostat-10s
@@ -0,0 +1 @@
+iostat
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/iostat-1s b/cmd/zpool/zpool.d/iostat-1s
new file mode 120000
index 000000000..084278d99
--- /dev/null
+++ b/cmd/zpool/zpool.d/iostat-1s
@@ -0,0 +1 @@
+iostat
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/label b/cmd/zpool/zpool.d/label
new file mode 120000
index 000000000..7d1e766ad
--- /dev/null
+++ b/cmd/zpool/zpool.d/label
@@ -0,0 +1 @@
+lsblk
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/locate_led b/cmd/zpool/zpool.d/locate_led
new file mode 120000
index 000000000..478d1e896
--- /dev/null
+++ b/cmd/zpool/zpool.d/locate_led
@@ -0,0 +1 @@
+ses
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/lsblk b/cmd/zpool/zpool.d/lsblk
new file mode 100755
index 000000000..e38a71941
--- /dev/null
+++ b/cmd/zpool/zpool.d/lsblk
@@ -0,0 +1,84 @@
+#!/bin/sh
+#
+# Print some common lsblk values
+#
+# Any (lowercased) name symlinked to the lsblk script will be passed to lsblk
+# as one of its --output names.  Here's a partial list of --output names
+# from the lsblk binary:
+#
+# Available columns (for --output):
+#        NAME  device name
+#       KNAME  internal kernel device name
+#     MAJ:MIN  major:minor device number
+#      FSTYPE  filesystem type
+#  MOUNTPOINT  where the device is mounted
+#       LABEL  filesystem LABEL
+#        UUID  filesystem UUID
+#          RA  read-ahead of the device
+#          RO  read-only device
+#          RM  removable device
+#       MODEL  device identifier
+#        SIZE  size of the device
+#       STATE  state of the device
+#       OWNER  user name
+#       GROUP  group name
+#        MODE  device node permissions
+#   ALIGNMENT  alignment offset
+#      MIN-IO  minimum I/O size
+#      OPT-IO  optimal I/O size
+#     PHY-SEC  physical sector size
+#     LOG-SEC  logical sector size
+#        ROTA  rotational device
+#       SCHED  I/O scheduler name
+#     RQ-SIZE  request queue size
+#        TYPE  device type
+#    DISC-ALN  discard alignment offset
+#   DISC-GRAN  discard granularity
+#    DISC-MAX  discard max bytes
+#   DISC-ZERO  discard zeroes data
+#
+# If the script is run as just 'lsblk' then print out disk size, vendor,
+# model number and serial number.
+
+
+helpstr="
+label:	Show filesystem label.
+model:	Show disk model number.
+serial:	Show disk serial number.
+size:	Show the disk capacity.
+vendor:	Show the disk vendor.
+lsblk:	Show the disk size, vendor, model number, and serial number."
+
+script=$(basename "$0")
+
+if [ "$1" = "-h" ] ; then
+        echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
+        exit
+fi
+
+if [ "$script" = "lsblk" ] ; then
+	list="size vendor model serial"
+else
+	list=$(echo "$script" | tr '[:upper:]' '[:lower:]')
+fi
+
+# Older versions of lsblk don't support all these values (like SERIAL).
+for i in $list ; do
+
+	# Special case: Looking up the size of a file-based vdev can't
+	# be done with lsblk.
+	if [ "$i" = "size" ] && [ -f "$VDEV_UPATH" ] ; then
+		size="$(du -h --apparent-size $VDEV_UPATH | cut -f 1)"
+		echo "size=$size"
+		continue
+	fi
+
+
+	val=""
+	if val=$(eval "lsblk -dl -n -o $i $VDEV_UPATH 2>/dev/null") ; then
+		# Remove leading/trailing whitespace from value
+		val=$(echo "$val" | sed -e 's/^[[:space:]]*//' \
+		     -e 's/[[:space:]]*$//')
+	fi
+	echo "$i=$val"
+done
diff --git a/cmd/zpool/zpool.d/model b/cmd/zpool/zpool.d/model
new file mode 120000
index 000000000..7d1e766ad
--- /dev/null
+++ b/cmd/zpool/zpool.d/model
@@ -0,0 +1 @@
+lsblk
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/serial b/cmd/zpool/zpool.d/serial
new file mode 120000
index 000000000..7d1e766ad
--- /dev/null
+++ b/cmd/zpool/zpool.d/serial
@@ -0,0 +1 @@
+lsblk
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/ses b/cmd/zpool/zpool.d/ses
new file mode 100755
index 000000000..10d5dcfd3
--- /dev/null
+++ b/cmd/zpool/zpool.d/ses
@@ -0,0 +1,52 @@
+#!/bin/sh
+#
+# Print SCSI Enclosure Services (SES) info. The output is dependent on the name
+# of the script/symlink used to call it.
+#
+helpstr="
+enc:		Show disk enclosure w:x:y:z value.
+slot:		Show disk slot number as reported by the enclosure.
+encdev:		Show the /dev/sg* device for the enclosure associated with the disk slot.
+fault_led:	Show the value of the disk enclosure slot fault LED.
+locate_led:	Show the value of the disk enclosure slot locate LED.
+ses:		Show disk's enclosure, enclosure dev, slot number, and fault/locate LED values."
+
+script=$(basename "$0")
+if [ "$1" = "-h" ] ; then
+	echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
+	exit
+fi
+
+if [ "$script" = "ses" ] ; then
+	scripts='enc encdev slot fault_led locate_led'
+else
+	scripts="$script"
+fi
+
+for i in $scripts ; do
+	if [ -z "$VDEV_ENC_SYSFS_PATH" ] ; then
+		echo "$i="
+		continue
+	fi
+
+	val=""
+	case $i in
+	enc)
+		val=$(ls "$VDEV_ENC_SYSFS_PATH/../../" 2>/dev/null)
+		;;
+	slot)
+		val=$(cat "$VDEV_ENC_SYSFS_PATH/slot" 2>/dev/null)
+		;;
+	encdev)
+		val=$(ls "$VDEV_ENC_SYSFS_PATH/../device/scsi_generic" 2>/dev/null)
+		;;
+	fault_led)
+		val=$(cat "$VDEV_ENC_SYSFS_PATH/fault" 2>/dev/null)
+		;;
+	locate_led)
+		val=$(cat "$VDEV_ENC_SYSFS_PATH/locate" 2>/dev/null)
+		;;
+	esac
+	echo "$i=$val"
+done
+
diff --git a/cmd/zpool/zpool.d/size b/cmd/zpool/zpool.d/size
new file mode 120000
index 000000000..7d1e766ad
--- /dev/null
+++ b/cmd/zpool/zpool.d/size
@@ -0,0 +1 @@
+lsblk
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/slaves b/cmd/zpool/zpool.d/slaves
new file mode 100755
index 000000000..9c16d6c4e
--- /dev/null
+++ b/cmd/zpool/zpool.d/slaves
@@ -0,0 +1,32 @@
+#!/bin/sh
+#
+# Show device mapper slave devices.  This is useful for looking up the
+# /dev/sd* devices associated with a dm or multipath device.  For example:
+#
+# $ ls /sys/block/dm-113/slaves/
+# sddt  sdjw
+#
+
+if [ "$1" = "-h" ] ; then
+	echo "Show device mapper slave devices."
+	exit
+fi
+
+dev="$VDEV_PATH"
+
+# If the VDEV path is a symlink, resolve it to a real device
+if [ -L "$dev" ] ; then
+	dev=$(readlink "$dev")
+fi
+
+dev=$(basename "$dev")
+val=""
+if [ -d "/sys/class/block/$dev/slaves" ] ; then
+	# ls -C: output in columns, no newlines
+	val=$(ls -C "/sys/class/block/$dev/slaves")
+
+	# ls -C will print two spaces between files; change to one space.
+	val=$(echo "$val" | sed -r 's/[[:blank:]]+/ /g')
+fi
+
+echo "slaves=$val"
diff --git a/cmd/zpool/zpool.d/slot b/cmd/zpool/zpool.d/slot
new file mode 120000
index 000000000..478d1e896
--- /dev/null
+++ b/cmd/zpool/zpool.d/slot
@@ -0,0 +1 @@
+ses
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/upath b/cmd/zpool/zpool.d/upath
new file mode 100755
index 000000000..16a4327d4
--- /dev/null
+++ b/cmd/zpool/zpool.d/upath
@@ -0,0 +1,7 @@
+#!/bin/sh
+if [ "$1" = "-h" ] ; then
+	echo "Show the underlying path for a device."
+	exit
+fi
+
+echo upath="$VDEV_UPATH"
diff --git a/cmd/zpool/zpool.d/vendor b/cmd/zpool/zpool.d/vendor
new file mode 120000
index 000000000..7d1e766ad
--- /dev/null
+++ b/cmd/zpool/zpool.d/vendor
@@ -0,0 +1 @@
+lsblk
\ No newline at end of file
diff --git a/cmd/zpool/zpool_iter.c b/cmd/zpool/zpool_iter.c
index 7ce0ccf9e..94777f076 100644
--- a/cmd/zpool/zpool_iter.c
+++ b/cmd/zpool/zpool_iter.c
@@ -36,6 +36,7 @@
 
 #include <libzfs.h>
 #include <sys/zfs_context.h>
+#include <sys/wait.h>
 
 #include "zpool_util.h"
 
@@ -321,41 +322,229 @@ for_each_vdev(zpool_handle_t *zhp, pool_vdev_iter_f func, void *data)
 	return (for_each_vdev_cb(zhp, nvroot, func, data));
 }
 
+/*
+ * Process the vcdl->vdev_cmd_data[] array to figure out all the unique column
+ * names and their widths.  When this function is done, vcdl->uniq_cols,
+ * vcdl->uniq_cols_cnt, and vcdl->uniq_cols_width will be filled in.
+ */
+static void
+process_unique_cmd_columns(vdev_cmd_data_list_t *vcdl)
+{
+	char **uniq_cols = NULL, **tmp = NULL;
+	int *uniq_cols_width;
+	vdev_cmd_data_t *data;
+	int cnt = 0;
+	int k;
+
+	/* For each vdev */
+	for (int i = 0; i < vcdl->count; i++) {
+		data = &vcdl->data[i];
+		/* For each column the vdev reported */
+		for (int j = 0; j < data->cols_cnt; j++) {
+			/* Is this column in our list of unique column names? */
+			for (k = 0; k < cnt; k++) {
+				if (strcmp(data->cols[j], uniq_cols[k]) == 0)
+					break; /* yes it is */
+			}
+			if (k == cnt) {
+				/* No entry for column, add to list */
+				tmp = realloc(uniq_cols, sizeof (*uniq_cols) *
+				    (cnt + 1));
+				if (tmp == NULL)
+					break; /* Nothing we can do... */
+				uniq_cols = tmp;
+				uniq_cols[cnt] = data->cols[j];
+				cnt++;
+			}
+		}
+	}
+
+	/*
+	 * We now have a list of all the unique column names.  Figure out the
+	 * max width of each column by looking at the column name and all its
+	 * values.
+	 */
+	uniq_cols_width = safe_malloc(sizeof (*uniq_cols_width) * cnt);
+	for (int i = 0; i < cnt; i++) {
+		/* Start off with the column title's width */
+		uniq_cols_width[i] = strlen(uniq_cols[i]);
+		/* For each vdev */
+		for (int j = 0; j < vcdl->count; j++) {
+			/* For each of the vdev's values in a column */
+			data = &vcdl->data[j];
+			for (k = 0; k < data->cols_cnt; k++) {
+				/* Does this vdev have a value for this col? */
+				if (strcmp(data->cols[k], uniq_cols[i]) == 0) {
+					/* Is the value width larger? */
+					uniq_cols_width[i] =
+					    MAX(uniq_cols_width[i],
+					    strlen(data->lines[k]));
+				}
+			}
+		}
+	}
+
+	vcdl->uniq_cols = uniq_cols;
+	vcdl->uniq_cols_cnt = cnt;
+	vcdl->uniq_cols_width = uniq_cols_width;
+}
+
+
+/*
+ * Process a line of command output
+ *
+ * When running 'zpool iostat|status -c' the lines of output can either be
+ * in the form of:
+ *
+ *	column_name=value
+ *
+ * Or just:
+ *
+ *	value
+ *
+ * Process the column_name (if any) and value.
+ *
+ * Returns 0 if line was processed, and there are more lines can still be
+ * processed.
+ *
+ * Returns 1 if this was the last line to process, or error.
+ */
+static int
+vdev_process_cmd_output(vdev_cmd_data_t *data, char *line)
+{
+	char *col = NULL;
+	char *val = line;
+	char *equals;
+	char **tmp;
+
+	if (line == NULL)
+		return (1);
+
+	equals = strchr(line, '=');
+	if (equals != NULL) {
+		/*
+		 * We have a 'column=value' type line.  Split it into the
+		 * column and value strings by turning the '=' into a '\0'.
+		 */
+		*equals = '\0';
+		col = line;
+		val = equals + 1;
+	} else {
+		val = line;
+	}
+
+	/* Do we already have a column by this name?  If so, skip it. */
+	if (col != NULL) {
+		for (int i = 0; i < data->cols_cnt; i++) {
+			if (strcmp(col, data->cols[i]) == 0)
+				return (0); /* Duplicate, skip */
+		}
+	}
+
+	if (val != NULL) {
+		tmp = realloc(data->lines,
+		    (data->lines_cnt + 1) * sizeof (*data->lines));
+		if (tmp == NULL)
+			return (1);
+
+		data->lines = tmp;
+		data->lines[data->lines_cnt] = strdup(val);
+		data->lines_cnt++;
+	}
+
+	if (col != NULL) {
+		tmp = realloc(data->cols,
+		    (data->cols_cnt + 1) * sizeof (*data->cols));
+		if (tmp == NULL)
+			return (1);
+
+		data->cols = tmp;
+		data->cols[data->cols_cnt] = strdup(col);
+		data->cols_cnt++;
+	}
+
+	if (val != NULL && col == NULL)
+		return (1);
+
+	return (0);
+}
+
+/*
+ * Run the cmd and store results in *data.
+ */
+static void
+vdev_run_cmd(vdev_cmd_data_t *data, char *cmd)
+{
+	int rc;
+	char *argv[2] = {cmd, 0};
+	char *env[5] = {"PATH=/bin:/sbin:/usr/bin:/usr/sbin", NULL, NULL, NULL,
+	    NULL};
+	char **lines = NULL;
+	int lines_cnt = 0;
+	int i;
+
+	/* Setup our custom environment variables */
+	rc = asprintf(&env[1], "VDEV_PATH=%s",
+	    data->path ? data->path : "");
+	if (rc == -1)
+		goto out;
+
+	rc = asprintf(&env[2], "VDEV_UPATH=%s",
+	    data->upath ? data->upath : "");
+	if (rc == -1)
+		goto out;
+
+	rc = asprintf(&env[3], "VDEV_ENC_SYSFS_PATH=%s",
+	    data->vdev_enc_sysfs_path ?
+	    data->vdev_enc_sysfs_path : "");
+	if (rc == -1)
+		goto out;
+
+	/* Run the command */
+	rc = libzfs_run_process_get_stdout_nopath(cmd, argv, env, &lines,
+	    &lines_cnt);
+	if (rc != 0)
+		goto out;
+
+	/* Process the output we got */
+	for (i = 0; i < lines_cnt; i++)
+		if (vdev_process_cmd_output(data, lines[i]) != 0)
+			break;
+
+out:
+	if (lines != NULL)
+		libzfs_free_str_array(lines, lines_cnt);
+
+	/* Start with i = 1 since env[0] was statically allocated */
+	for (i = 1; i < ARRAY_SIZE(env); i++)
+		if (env[i] != NULL)
+			free(env[i]);
+}
+
 /* Thread function run for each vdev */
 static void
 vdev_run_cmd_thread(void *cb_cmd_data)
 {
 	vdev_cmd_data_t *data = cb_cmd_data;
-	char *pos = NULL;
-	FILE *fp;
-	size_t len = 0;
-	char cmd[_POSIX_ARG_MAX];
-
-	/* Set our VDEV_PATH and VDEV_UPATH env vars and run command */
-	if (snprintf(cmd, sizeof (cmd), "VDEV_PATH=%s && VDEV_UPATH=\"%s\" && "
-	    "VDEV_ENC_SYSFS_PATH=\"%s\" && %s", data->path ? data->path : "",
-	    data->upath ? data->upath : "",
-	    data->vdev_enc_sysfs_path ? data->vdev_enc_sysfs_path : "",
-	    data->cmd) >= sizeof (cmd)) {
-		/* Our string was truncated */
-		return;
-	}
+	const char *sep = ",";
+	char *cmd = NULL, *cmddup, *rest;
+	char fullpath[MAXPATHLEN];
 
-	fp = popen(cmd, "r");
-	if (fp == NULL)
+	cmddup = strdup(data->cmd);
+	if (cmddup == NULL)
 		return;
 
-	data->line = NULL;
+	rest = cmddup;
+	while ((cmd = strtok_r(rest, sep, &rest))) {
+		if (snprintf(fullpath, sizeof (fullpath), "%s/%s",
+		    ZPOOL_SCRIPTS_DIR, cmd) == -1)
+			continue;
 
-	/* Save the first line of output from the command */
-	if (getline(&data->line, &len, fp) != -1) {
-		/* Success.  Remove newline from the end, if necessary. */
-		if ((pos = strchr(data->line, '\n')) != NULL)
-			*pos = '\0';
-	} else {
-		data->line = NULL;
+		/* Does the script exist in our zpool scripts dir? */
+		if (access(fullpath, X_OK) == 0)
+			vdev_run_cmd(data, fullpath);
 	}
-	pclose(fp);
+	free(cmddup);
 }
 
 /* For each vdev in the pool run a command */
@@ -412,6 +601,8 @@ for_each_vdev_run_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_vcdl)
 	data->path = strdup(path);
 	data->upath = zfs_get_underlying_path(path);
 	data->cmd = vcdl->cmd;
+	data->lines = data->cols = NULL;
+	data->lines_cnt = data->cols_cnt = 0;
 	if (vdev_enc_sysfs_path)
 		data->vdev_enc_sysfs_path = strdup(vdev_enc_sysfs_path);
 	else
@@ -463,6 +654,7 @@ all_pools_for_each_vdev_run_vcdl(vdev_cmd_data_list_t *vcdl)
 	taskq_wait(t);
 	taskq_destroy(t);
 	thread_fini();
+
 }
 
 /*
@@ -495,6 +687,13 @@ all_pools_for_each_vdev_run(int argc, char **argv, char *cmd,
 	/* Run command on all vdevs in all pools */
 	all_pools_for_each_vdev_run_vcdl(vcdl);
 
+	/*
+	 * vcdl->data[] now contains all the column names and values for each
+	 * vdev.  We need to process that into a master list of unique column
+	 * names, and figure out the width of each column.
+	 */
+	process_unique_cmd_columns(vcdl);
+
 	return (vcdl);
 }
 
@@ -504,12 +703,23 @@ all_pools_for_each_vdev_run(int argc, char **argv, char *cmd,
 void
 free_vdev_cmd_data_list(vdev_cmd_data_list_t *vcdl)
 {
-	int i;
-	for (i = 0; i < vcdl->count; i++) {
+	free(vcdl->uniq_cols);
+	free(vcdl->uniq_cols_width);
+
+	for (int i = 0; i < vcdl->count; i++) {
 		free(vcdl->data[i].path);
 		free(vcdl->data[i].pool);
 		free(vcdl->data[i].upath);
-		free(vcdl->data[i].line);
+
+		for (int j = 0; j < vcdl->data[i].lines_cnt; j++)
+			free(vcdl->data[i].lines[j]);
+
+		free(vcdl->data[i].lines);
+
+		for (int j = 0; j < vcdl->data[i].cols_cnt; j++)
+			free(vcdl->data[i].cols[j]);
+
+		free(vcdl->data[i].cols);
 		free(vcdl->data[i].vdev_enc_sysfs_path);
 	}
 	free(vcdl->data);
diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c
index cc6c18eed..4b67bfde0 100644
--- a/cmd/zpool/zpool_main.c
+++ b/cmd/zpool/zpool_main.c
@@ -45,6 +45,7 @@
 #include <unistd.h>
 #include <pwd.h>
 #include <zone.h>
+#include <sys/wait.h>
 #include <zfs_prop.h>
 #include <sys/fs/zfs.h>
 #include <sys/stat.h>
@@ -52,6 +53,7 @@
 #include <sys/fm/util.h>
 #include <sys/fm/protocol.h>
 #include <sys/zfs_ioctl.h>
+
 #include <math.h>
 
 #include <libzfs.h>
@@ -314,10 +316,10 @@ get_usage(zpool_help_t idx)
 		    "[-R root] [-F [-n]]\n"
 		    "\t    <pool | id> [newpool]\n"));
 	case HELP_IOSTAT:
-		return (gettext("\tiostat [-c CMD] [-T d | u] [-ghHLpPvy] "
-		    "[[-lq]|[-r|-w]]\n"
-		    "\t    [[pool ...]|[pool vdev ...]|[vdev ...]] "
-		    "[interval [count]]\n"));
+		return (gettext("\tiostat [[[-c [script1,script2,...]"
+		    "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
+		    "\t    [[pool ...]|[pool vdev ...]|[vdev ...]]"
+		    " [interval [count]]\n"));
 	case HELP_LABELCLEAR:
 		return (gettext("\tlabelclear [-f] <vdev>\n"));
 	case HELP_LIST:
@@ -337,8 +339,8 @@ get_usage(zpool_help_t idx)
 	case HELP_SCRUB:
 		return (gettext("\tscrub [-s] <pool> ...\n"));
 	case HELP_STATUS:
-		return (gettext("\tstatus [-c CMD] [-gLPvxD] [-T d|u] [pool]"
-		    " ... [interval [count]]\n"));
+		return (gettext("\tstatus [-c [script1,script2,...]] [-gLPvxD]"
+		    "[-T d|u] [pool] ... [interval [count]]\n"));
 	case HELP_UPGRADE:
 		return (gettext("\tupgrade\n"
 		    "\tupgrade -v\n"
@@ -1542,17 +1544,68 @@ typedef struct status_cbdata {
 	vdev_cmd_data_list_t	*vcdl;
 } status_cbdata_t;
 
-/* Print output line for specific vdev in a specific pool */
+/* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
+static int
+is_blank_str(char *str)
+{
+	while (str != NULL && *str != '\0') {
+		if (!isblank(*str))
+			return (0);
+		str++;
+	}
+	return (1);
+}
+
+/* Print command output lines for specific vdev in a specific pool */
 static void
 zpool_print_cmd(vdev_cmd_data_list_t *vcdl, const char *pool, char *path)
 {
-	int i;
+	vdev_cmd_data_t *data;
+	int i, j;
+	char *val;
+
 	for (i = 0; i < vcdl->count; i++) {
-		if ((strcmp(vcdl->data[i].path, path) == 0) &&
-		    (strcmp(vcdl->data[i].pool, pool) == 0)) {
-			printf("%s", vcdl->data[i].line);
-			break;
+		if ((strcmp(vcdl->data[i].path, path) != 0) ||
+		    (strcmp(vcdl->data[i].pool, pool) != 0)) {
+			/* Not the vdev we're looking for */
+			continue;
 		}
+
+		data = &vcdl->data[i];
+		/* Print out all the output values for this vdev */
+		for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
+			val = NULL;
+			/* Does this vdev have values for this column? */
+			for (int k = 0; k < data->cols_cnt; k++) {
+				if (strcmp(data->cols[k],
+				    vcdl->uniq_cols[j]) == 0) {
+					/* yes it does, record the value */
+					val = data->lines[k];
+					break;
+				}
+			}
+			/*
+			 * Mark empty values with dashes to make output
+			 * awk-able.
+			 */
+			if (is_blank_str(val))
+				val = "-";
+
+			printf("%*s", vcdl->uniq_cols_width[j], val);
+			if (j < vcdl->uniq_cols_cnt - 1)
+				printf("  ");
+		}
+
+		/* Print out any values that aren't in a column at the end */
+		for (j = data->cols_cnt; j < data->lines_cnt; j++) {
+			/* Did we have any columns?  If so print a spacer. */
+			if (vcdl->uniq_cols_cnt > 0)
+				printf("  ");
+
+			val = data->lines[j];
+			printf("%s", val ? val : "");
+		}
+		break;
 	}
 }
 
@@ -2799,9 +2852,54 @@ print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
 
 		}
 	}
-	printf("\n");
 }
 
+
+/*
+ * print_cmd_columns - Print custom column titles from -c
+ *
+ * If the user specified the "zpool status|iostat -c" then print their custom
+ * column titles in the header.  For example, print_cmd_columns() would print
+ * the "  col1  col2" part of this:
+ *
+ * $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
+ * ...
+ *	      capacity     operations     bandwidth
+ * pool        alloc   free   read  write   read  write  col1  col2
+ * ----------  -----  -----  -----  -----  -----  -----  ----  ----
+ * mypool       269K  1008M      0      0    107    946
+ *   mirror     269K  1008M      0      0    107    946
+ *     sdb         -      -      0      0    102    473  val1  val2
+ *     sdc         -      -      0      0      5    473  val1  val2
+ * ----------  -----  -----  -----  -----  -----  -----  ----  ----
+ */
+void
+print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
+{
+	int i, j;
+	vdev_cmd_data_t *data = &vcdl->data[0];
+
+	if (vcdl->count == 0 || data == NULL)
+		return;
+
+	/*
+	 * Each vdev cmd should have the same column names unless the user did
+	 * something weird with their cmd.  Just take the column names from the
+	 * first vdev and assume it works for all of them.
+	 */
+	for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
+		printf("  ");
+		if (use_dashes) {
+			for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
+				printf("-");
+		} else {
+			printf("%*s", vcdl->uniq_cols_width[i],
+			    vcdl->uniq_cols[i]);
+		}
+	}
+}
+
+
 /*
  * Utility function to print out a line of dashes like:
  *
@@ -2870,7 +2968,6 @@ print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
 				    "--------------------");
 		}
 	}
-	printf("\n");
 }
 
 
@@ -2912,12 +3009,22 @@ print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
 
 
 	print_iostat_labels(cb, force_column_width, iostat_top_labels);
+	printf("\n");
 
 	printf("%-*s", namewidth, title);
 
 	print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
+	if (cb->vcdl != NULL)
+		print_cmd_columns(cb->vcdl, 0);
+
+	printf("\n");
 
 	print_iostat_separator_impl(cb, force_column_width);
+
+	if (cb->vcdl != NULL)
+		print_cmd_columns(cb->vcdl, 1);
+
+	printf("\n");
 }
 
 static void
@@ -3451,11 +3558,8 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
 		char *path;
 		if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
 		    &path) == 0) {
-			if (!(cb->cb_flags & IOS_ANYHISTO_M))
-				printf("  ");
+			printf("  ");
 			zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
-			if (cb->cb_flags & IOS_ANYHISTO_M)
-				printf("\n");
 		}
 	}
 
@@ -3602,6 +3706,10 @@ print_iostat(zpool_handle_t *zhp, void *data)
 	if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
 	    !cb->cb_scripted && cb->cb_verbose && !cb->cb_vdev_names_count) {
 		print_iostat_separator(cb);
+		if (cb->vcdl != NULL) {
+			print_cmd_columns(cb->vcdl, 1);
+		}
+		printf("\n");
 	}
 
 	return (ret);
@@ -3990,11 +4098,72 @@ fsleep(float sec)
 	nanosleep(&req, NULL);
 }
 
+/*
+ * Run one of the zpool status/iostat -c scripts with the help (-h) option and
+ * print the result.
+ *
+ * name:	Short name of the script ('iostat').
+ * path:	Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
+ */
+static void
+print_zpool_script_help(char *name, char *path)
+{
+	char *argv[] = {path, "-h", NULL};
+	char **lines = NULL;
+	int lines_cnt = 0;
+	int rc;
+
+	rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
+	    &lines_cnt);
+	if (rc != 0 || lines == NULL || lines_cnt <= 0)
+		return;
+
+	for (int i = 0; i < lines_cnt; i++)
+		if (!is_blank_str(lines[i]))
+			printf("  %-14s  %s\n", name, lines[i]);
+
+	libzfs_free_str_array(lines, lines_cnt);
+}
+
 
 /*
- * zpool iostat [-c CMD] [-ghHLpPvy] [[-lq]|[-r|-w]] [-n name] [-T d|u]
- *		[[ pool ...]|[pool vdev ...]|[vdev ...]]
- *		[interval [count]]
+ * Go though the list of all the zpool status/iostat -c scripts, run their
+ * help option (-h), and print out the results.
+ */
+static void
+print_zpool_script_list(void)
+{
+	DIR *dir;
+	struct dirent *ent;
+	char fullpath[MAXPATHLEN];
+	struct stat dir_stat;
+
+	if ((dir = opendir(ZPOOL_SCRIPTS_DIR)) != NULL) {
+		printf("\n");
+		/* print all the files and directories within directory */
+		while ((ent = readdir(dir)) != NULL) {
+			sprintf(fullpath, "%s/%s", ZPOOL_SCRIPTS_DIR,
+			    ent->d_name);
+
+			/* Print the scripts */
+			if (stat(fullpath, &dir_stat) == 0)
+				if (dir_stat.st_mode & S_IXUSR &&
+				    S_ISREG(dir_stat.st_mode))
+					print_zpool_script_help(ent->d_name,
+					    fullpath);
+		}
+		printf("\n");
+		closedir(dir);
+	} else {
+		fprintf(stderr, gettext("Can't open %s scripts dir\n"),
+		    ZPOOL_SCRIPTS_DIR);
+	}
+}
+
+/*
+ * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
+ *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
+ *              [interval [count]]
  *
  *	-c CMD  For each vdev, run command CMD
  *	-g	Display guid for individual vdev name.
@@ -4046,7 +4215,20 @@ zpool_do_iostat(int argc, char **argv)
 	while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwH")) != -1) {
 		switch (c) {
 		case 'c':
+			if (cmd != NULL) {
+				fprintf(stderr,
+				    gettext("Can't set -c flag twice\n"));
+				exit(1);
+			}
+			if ((getuid() <= 0 || geteuid() <= 0) &&
+			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
+				fprintf(stderr, gettext(
+				    "Can't run -c with root privileges "
+				    "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
+				exit(1);
+			}
 			cmd = optarg;
+			verbose = B_TRUE;
 			break;
 		case 'g':
 			guid = B_TRUE;
@@ -4089,8 +4271,11 @@ zpool_do_iostat(int argc, char **argv)
 			break;
 		case '?':
 			if (optopt == 'c') {
-				fprintf(stderr,
-				    gettext("Missing CMD for -c\n"));
+				fprintf(stderr, gettext(
+				    "Current scripts in %s:\n"),
+				    ZPOOL_SCRIPTS_DIR);
+				print_zpool_script_list();
+				exit(0);
 			} else {
 				fprintf(stderr,
 				    gettext("invalid option '%c'\n"), optopt);
@@ -4185,10 +4370,10 @@ zpool_do_iostat(int argc, char **argv)
 		return (1);
 	}
 
-	if ((l_histo || rq_histo) && (queues || latency)) {
+	if ((l_histo || rq_histo) && (cmd != NULL || latency || queues)) {
 		pool_list_free(list);
 		(void) fprintf(stderr,
-		    gettext("[-r|-w] isn't allowed with [-q|-l]\n"));
+		    gettext("[-r|-w] isn't allowed with [-c|-l|-q]\n"));
 		usage(B_FALSE);
 		return (1);
 	}
@@ -4276,6 +4461,15 @@ zpool_do_iostat(int argc, char **argv)
 			if (timestamp_fmt != NODATE)
 				print_timestamp(timestamp_fmt);
 
+			if (cmd != NULL && cb.cb_verbose &&
+			    !(cb.cb_flags & IOS_ANYHISTO_M)) {
+				cb.vcdl = all_pools_for_each_vdev_run(argc,
+				    argv, cmd, g_zfs, cb.cb_vdev_names,
+				    cb.cb_vdev_names_count, cb.cb_name_flags);
+			} else {
+				cb.vcdl = NULL;
+			}
+
 			/*
 			 * If it's the first time and we're not skipping it,
 			 * or either skip or verbose mode, print the header.
@@ -4294,16 +4488,9 @@ zpool_do_iostat(int argc, char **argv)
 				continue;
 			}
 
-			if (cmd != NULL && cb.cb_verbose)
-				cb.vcdl = all_pools_for_each_vdev_run(argc,
-				    argv, cmd, g_zfs, cb.cb_vdev_names,
-				    cb.cb_vdev_names_count, cb.cb_name_flags);
 
 			pool_list_iter(list, B_FALSE, print_iostat, &cb);
 
-			if (cb.vcdl != NULL)
-				free_vdev_cmd_data_list(cb.vcdl);
-
 			/*
 			 * If there's more than one pool, and we're not in
 			 * verbose mode (which prints a separator for us),
@@ -4318,7 +4505,14 @@ zpool_do_iostat(int argc, char **argv)
 			    cb.cb_vdev_names_count)) &&
 			    !cb.cb_scripted) {
 				print_iostat_separator(&cb);
+				if (cb.vcdl != NULL)
+					print_cmd_columns(cb.vcdl, 1);
+				printf("\n");
 			}
+
+			if (cb.vcdl != NULL)
+				free_vdev_cmd_data_list(cb.vcdl);
+
 		}
 
 		/*
@@ -6036,9 +6230,14 @@ status_callback(zpool_handle_t *zhp, void *data)
 			cbp->cb_namewidth = 10;
 
 		(void) printf(gettext("config:\n\n"));
-		(void) printf(gettext("\t%-*s  %-8s %5s %5s %5s\n"),
+		(void) printf(gettext("\t%-*s  %-8s %5s %5s %5s"),
 		    cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
 		    "CKSUM");
+
+		if (cbp->vcdl != NULL)
+			print_cmd_columns(cbp->vcdl, 0);
+
+		printf("\n");
 		print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
 		    B_FALSE);
 
@@ -6098,7 +6297,8 @@ status_callback(zpool_handle_t *zhp, void *data)
 }
 
 /*
- * zpool status [-c CMD] [-gLPvx] [-T d|u] [pool] ... [interval [count]]
+ * zpool status [-c [script1,script2,...]] [-gLPvx] [-T d|u] [pool] ...
+ *              [interval [count]]
  *
  *	-c CMD	For each vdev, run command CMD
  *	-g	Display guid for individual vdev name.
@@ -6125,6 +6325,18 @@ zpool_do_status(int argc, char **argv)
 	while ((c = getopt(argc, argv, "c:gLPvxDT:")) != -1) {
 		switch (c) {
 		case 'c':
+			if (cmd != NULL) {
+				fprintf(stderr,
+				    gettext("Can't set -c flag twice\n"));
+				exit(1);
+			}
+			if ((getuid() <= 0 || geteuid() <= 0) &&
+			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
+				fprintf(stderr, gettext(
+				    "Can't run -c with root privileges "
+				    "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
+				exit(1);
+			}
 			cmd = optarg;
 			break;
 		case 'g':
@@ -6150,8 +6362,11 @@ zpool_do_status(int argc, char **argv)
 			break;
 		case '?':
 			if (optopt == 'c') {
-				fprintf(stderr,
-				    gettext("Missing CMD for -c\n"));
+				fprintf(stderr, gettext(
+				    "Current scripts in %s:\n"),
+				    ZPOOL_SCRIPTS_DIR);
+				print_zpool_script_list();
+				exit(0);
 			} else {
 				fprintf(stderr,
 				    gettext("invalid option '%c'\n"), optopt);
diff --git a/cmd/zpool/zpool_util.h b/cmd/zpool/zpool_util.h
index 61ed33c15..ab7662cfa 100644
--- a/cmd/zpool/zpool_util.h
+++ b/cmd/zpool/zpool_util.h
@@ -32,6 +32,9 @@
 extern "C" {
 #endif
 
+/* Path to scripts you can run with "zpool status/iostat -c" */
+#define	ZPOOL_SCRIPTS_DIR SYSCONFDIR"/zfs/zpool.d"
+
 /*
  * Basic utility functions
  */
@@ -75,7 +78,13 @@ libzfs_handle_t *g_zfs;
 
 typedef	struct vdev_cmd_data
 {
-	char *line;	/* cmd output */
+	char **lines;	/* Array of lines of output, minus the column name */
+	int lines_cnt;	/* Number of lines in the array */
+
+	char **cols;	/* Array of column names */
+	int cols_cnt;	/* Number of column names */
+
+
 	char *path;	/* vdev path */
 	char *upath;	/* vdev underlying path */
 	char *pool;	/* Pool name */
@@ -96,6 +105,11 @@ typedef struct vdev_cmd_data_list
 
 	vdev_cmd_data_t *data;	/* Array of vdevs */
 
+	/* List of unique column names and widths */
+	char **uniq_cols;
+	int uniq_cols_cnt;
+	int *uniq_cols_width;
+
 } vdev_cmd_data_list_t;
 
 vdev_cmd_data_list_t *all_pools_for_each_vdev_run(int argc, char **argv,