zpool iostat/status -c improvements

Users can now provide their own scripts to be run
with 'zpool iostat/status -c'. User scripts should be
placed in ~/.zpool.d to be included in zpool's
default search path.

Provide a script which can be used with
'zpool iostat|status -c' that will return the type of
device (hdd, sdd, file).

Provide a script to get various values from smartctl
when using 'zpool iostat/status -c'.

Allow users to define the ZPOOL_SCRIPTS_PATH
environment variable which can be used to override
the default 'zpool iostat/status -c' search path.

Allow the ZPOOL_SCRIPTS_ENABLED environment
variable to enable or disable 'zpool status/iostat -c'
functionality.

Use the new smart script to provide the serial command.

Install /etc/sudoers.d/zfs file which contains the sudoer
rule for smartctl as a sample.

Allow 'zpool iostat/status -c' tests to run in tree.

Reviewed-by: Tony Hutter <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Giuseppe Di Natale <[email protected]>
Closes #6121 
Closes #6153 

26 files changed, 313 insertions, 38 deletions

diff --git a/cmd/zpool/Makefile.am b/cmd/zpool/Makefile.am
index f905fb73d..6eff1d143 100644
--- a/cmd/zpool/Makefile.am
+++ b/cmd/zpool/Makefile.am
@@ -36,12 +36,31 @@ dist_zpoolexec_SCRIPTS = \
 	zpool.d/label \
 	zpool.d/locate_led \
 	zpool.d/lsblk \
+	zpool.d/media \
 	zpool.d/model \
 	zpool.d/serial \
 	zpool.d/ses \
 	zpool.d/size \
 	zpool.d/slaves \
 	zpool.d/slot \
+	zpool.d/smart \
+	zpool.d/smartx \
+	zpool.d/temp \
+	zpool.d/health \
+	zpool.d/r_proc \
+	zpool.d/w_proc \
+	zpool.d/r_ucor \
+	zpool.d/w_ucor \
+	zpool.d/nonmed \
+	zpool.d/defect \
+	zpool.d/hours_on \
+	zpool.d/realloc \
+	zpool.d/rep_ucor \
+	zpool.d/cmd_to \
+	zpool.d/pend_sec \
+	zpool.d/off_ucor \
+	zpool.d/ata_err \
+	zpool.d/pwr_cyc \
 	zpool.d/upath \
 	zpool.d/vendor
 
@@ -55,12 +74,31 @@ zpoolconfdefaults = \
 	label \
 	locate_led \
 	lsblk \
+	media \
 	model \
 	serial \
 	ses \
 	size \
 	slaves \
 	slot \
+	smart \
+	smartx \
+	temp \
+	health \
+	r_proc \
+	w_proc \
+	r_ucor \
+	w_ucor \
+	nonmed \
+	defect \
+	hours_on \
+	realloc \
+	rep_ucor \
+	cmd_to \
+	pend_sec \
+	off_ucor \
+	ata_err \
+	pwr_cyc \
 	upath \
 	vendor
 
diff --git a/cmd/zpool/zpool.d/ata_err b/cmd/zpool/zpool.d/ata_err
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/ata_err
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/cmd_to b/cmd/zpool/zpool.d/cmd_to
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/cmd_to
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/defect b/cmd/zpool/zpool.d/defect
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/defect
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/health b/cmd/zpool/zpool.d/health
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/health
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/hours_on b/cmd/zpool/zpool.d/hours_on
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/hours_on
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/lsblk b/cmd/zpool/zpool.d/lsblk
index fc0394a3d..1cdef4049 100755
--- a/cmd/zpool/zpool.d/lsblk
+++ b/cmd/zpool/zpool.d/lsblk
@@ -38,16 +38,15 @@
 #   DISC-ZERO  discard zeroes data
 #
 # If the script is run as just 'lsblk' then print out disk size, vendor,
-# model number and serial number.
+# and model number.
 
 
 helpstr="
 label:	Show filesystem label.
 model:	Show disk model number.
-serial:	Show disk serial number.
 size:	Show the disk capacity.
 vendor:	Show the disk vendor.
-lsblk:	Show the disk size, vendor, model number, and serial number."
+lsblk:	Show the disk size, vendor, and model number."
 
 script=$(basename "$0")
 
@@ -57,7 +56,7 @@ if [ "$1" = "-h" ] ; then
 fi
 
 if [ "$script" = "lsblk" ] ; then
-	list="size vendor model serial"
+	list="size vendor model"
 else
 	list=$(echo "$script" | tr '[:upper:]' '[:lower:]')
 fi
diff --git a/cmd/zpool/zpool.d/media b/cmd/zpool/zpool.d/media
new file mode 100755
index 000000000..05bc15918
--- /dev/null
+++ b/cmd/zpool/zpool.d/media
@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# Print out the type of device
+#
+
+if [ "$1" = "-h" ] ; then
+	echo "Show whether a vdev is a file, hdd, or ssd."
+	exit
+fi
+
+if [ -b "$VDEV_UPATH" ]; then
+	device=$(basename "$VDEV_UPATH")
+	val=$(cat "/sys/block/$device/queue/rotational" 2>/dev/null)
+	if [ "$val" = "0" ]; then
+		MEDIA="ssd"
+	fi
+
+	if [ "$val" = "1" ]; then
+		MEDIA="hdd"
+	fi
+else
+	if [ -f "$VDEV_UPATH" ]; then
+		MEDIA="file"
+	fi
+fi
+
+echo "media=$MEDIA"
diff --git a/cmd/zpool/zpool.d/nonmed b/cmd/zpool/zpool.d/nonmed
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/nonmed
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/off_ucor b/cmd/zpool/zpool.d/off_ucor
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/off_ucor
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/pend_sec b/cmd/zpool/zpool.d/pend_sec
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/pend_sec
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/pwr_cyc b/cmd/zpool/zpool.d/pwr_cyc
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/pwr_cyc
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/r_proc b/cmd/zpool/zpool.d/r_proc
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/r_proc
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/r_ucor b/cmd/zpool/zpool.d/r_ucor
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/r_ucor
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/realloc b/cmd/zpool/zpool.d/realloc
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/realloc
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/rep_ucor b/cmd/zpool/zpool.d/rep_ucor
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/rep_ucor
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/serial b/cmd/zpool/zpool.d/serial
index 7d1e766ad..94f22861f 120000
--- a/cmd/zpool/zpool.d/serial
+++ b/cmd/zpool/zpool.d/serial
@@ -1 +1 @@
-lsblk
\ No newline at end of file
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/ses b/cmd/zpool/zpool.d/ses
index 10d5dcfd3..f6b7520df 100755
--- a/cmd/zpool/zpool.d/ses
+++ b/cmd/zpool/zpool.d/ses
@@ -6,10 +6,10 @@
 helpstr="
 enc:		Show disk enclosure w:x:y:z value.
 slot:		Show disk slot number as reported by the enclosure.
-encdev:		Show the /dev/sg* device for the enclosure associated with the disk slot.
-fault_led:	Show the value of the disk enclosure slot fault LED.
-locate_led:	Show the value of the disk enclosure slot locate LED.
-ses:		Show disk's enclosure, enclosure dev, slot number, and fault/locate LED values."
+encdev:		Show /dev/sg* device associated with the enclosure disk slot.
+fault_led:	Show value of the disk enclosure slot fault LED.
+locate_led:	Show value of the disk enclosure slot locate LED.
+ses:		Show disk's enc, enc device, slot, and fault/locate LED values."
 
 script=$(basename "$0")
 if [ "$1" = "-h" ] ; then
diff --git a/cmd/zpool/zpool.d/smart b/cmd/zpool/zpool.d/smart
new file mode 100755
index 000000000..3721f30ed
--- /dev/null
+++ b/cmd/zpool/zpool.d/smart
@@ -0,0 +1,123 @@
+#!/bin/sh
+#
+# Show SMART stats
+#
+
+helpstr="
+smart:		Show SMART temperature and error stats (specific to drive type)
+smartx:		Show SMART extended drive stats (specific to drive type).
+temp:		Show SMART drive temperature in celsius (all drives).
+health:		Show reported SMART status (all drives).
+r_proc:		Show SMART read GBytes processed over drive lifetime (SAS).
+w_proc:		Show SMART write GBytes processed over drive lifetime (SAS).
+r_ucor:		Show SMART read uncorrectable errors (SAS).
+w_ucor:		Show SMART write uncorrectable errors (SAS).
+nonmed:		Show SMART non-medium errors (SAS).
+defect:		Show SMART grown defect list (SAS).
+hours_on:	Show number of hours drive powered on (all drives).
+realloc:	Show SMART reallocated sectors count (ATA).
+rep_ucor:	Show SMART reported uncorrectable count (ATA).
+cmd_to:		Show SMART command timeout count (ATA).
+pend_sec:	Show SMART current pending sector count (ATA).
+off_ucor:	Show SMART offline uncorrectable errors (ATA).
+ata_err:	Show SMART ATA errors (ATA).
+pwr_cyc:	Show SMART power cycle count (ATA).
+serial:		Show disk serial number.
+"
+
+script=$(basename "$0")
+
+if [ "$1" = "-h" ] ; then
+        echo "$helpstr" | grep "$script:" | tr -s '\t' | cut -f 2-
+        exit
+fi
+
+smartctl_path=$(which smartctl)
+
+if [ -b "$VDEV_UPATH" ] && [ -x "$smartctl_path" ]; then
+	raw_out=$(eval "sudo $smartctl_path -a $VDEV_UPATH")
+
+	# Are we a SAS or ATA drive?  Look for the right line in smartctl:
+	#
+	# SAS:
+	#	Transport protocol:   SAS
+	#
+	# SATA:
+	#	ATA Version is:   8
+	#
+	type=$(echo "$raw_out" | grep -m 1 -Eo '^ATA|SAS$')
+	out=$(echo "$raw_out" | awk '
+# SAS specific
+/read:/{print "rrd="$4"\nr_cor="$5"\nr_proc="$7"\nr_ucor="$8}
+/write:/{print "rwr="$4"\nw_cor="$5"\nw_proc="$7"\nw_ucor="$8}
+/Non-medium error count/{print "nonmed="$4}
+/Elements in grown defect list/{print "defect="$6}
+
+# SAS common
+/Drive Temperature:/{print "temp="$4}
+# Status can be a long string, substitute spaces for '_'
+/SMART Health Status:/{printf "health="; for(i=4;i<=NF-1;i++){printf "%s_", $i}; printf "%s\n", $i}
+/number of hours powered up/{print "hours_on="$7}
+/Serial number:/{print "serial="$3}
+
+# SATA specific
+/Reallocated_Sector_Ct/{print "realloc="$10}
+/Reported_Uncorrect/{print "rep_ucor="$10}
+/Command_Timeout/{print "cmd_to="$10}
+/Current_Pending_Sector/{print "pend_sec="$10}
+/Offline_Uncorrectable/{print "off_ucor="$10}
+/ATA Error Count:/{print "ata_err="$4}
+/Power_Cycle_Count/{print "pwr_cyc="$10}
+
+# SATA common
+/Temperature_Celsius/{print "temp="$10}
+/SMART overall-health self-assessment test result:/{print "health="$6}
+/Power_On_Hours/{print "hours_on="$10}
+/Serial Number:/{print "serial="$3}
+
+END {ORS="\n"; print ""}
+');
+fi
+
+# if type is not set by now, either we don't have a block device
+# or smartctl failed. Either way, default to ATA and set out to
+# nothing
+if [ -z "$type" ]; then
+	type="ATA"
+	out=
+fi
+
+case $script in
+smart)
+	# Print temperature plus common predictors of drive failure
+	if [ "$type" = "SAS" ] ; then
+		scripts="temp|health|r_ucor|w_ucor"
+	elif [ "$type" = "ATA" ] ; then
+		scripts="temp|health|ata_err|realloc|rep_ucor|cmd_to|pend_sec|off_ucor"
+	fi
+	;;
+smartx)
+	# Print some other interesting stats
+	if [ "$type" = "SAS" ] ; then
+		scripts="hours_on|defect|nonmed|r_proc|w_proc"
+	elif [ "$type" = "ATA" ] ; then
+		scripts="hours_on|pwr_cyc"
+	fi
+	;;
+*)
+	scripts="$script"
+esac
+
+with_vals=$(echo "$out" | grep -E "$scripts")
+if [ ! -z "$with_vals" ]; then
+	echo "$with_vals"
+	without_vals=$(echo "$scripts" | tr "|" "\n" |
+		grep -v -E "$(echo "$with_vals" |
+		awk -F "=" '{print $1}')" | awk '{print $0"="}')
+else
+	without_vals=$(echo "$scripts" | tr "|" "\n" | awk '{print $0"="}')
+fi
+
+if [ ! -z "$without_vals" ]; then
+	echo "$without_vals"
+fi
diff --git a/cmd/zpool/zpool.d/smartx b/cmd/zpool/zpool.d/smartx
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/smartx
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/temp b/cmd/zpool/zpool.d/temp
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/temp
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/w_proc b/cmd/zpool/zpool.d/w_proc
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/w_proc
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool.d/w_ucor b/cmd/zpool/zpool.d/w_ucor
new file mode 120000
index 000000000..94f22861f
--- /dev/null
+++ b/cmd/zpool/zpool.d/w_ucor
@@ -0,0 +1 @@
+smart
\ No newline at end of file
diff --git a/cmd/zpool/zpool_iter.c b/cmd/zpool/zpool_iter.c
index 94777f076..abb1b1798 100644
--- a/cmd/zpool/zpool_iter.c
+++ b/cmd/zpool/zpool_iter.c
@@ -521,28 +521,66 @@ out:
 			free(env[i]);
 }
 
+/*
+ * Generate the search path for zpool iostat/status -c scripts.
+ * The string returned must be freed.
+ */
+char *
+zpool_get_cmd_search_path(void)
+{
+	const char *env;
+	char *sp = NULL;
+
+	env = getenv("ZPOOL_SCRIPTS_PATH");
+	if (env != NULL)
+		return (strdup(env));
+
+	env = getenv("HOME");
+	if (env != NULL) {
+		if (asprintf(&sp, "%s/.zpool.d:%s",
+		    env, ZPOOL_SCRIPTS_DIR) != -1) {
+			return (sp);
+		}
+	}
+
+	if (asprintf(&sp, "%s", ZPOOL_SCRIPTS_DIR) != -1)
+		return (sp);
+
+	return (NULL);
+}
+
 /* Thread function run for each vdev */
 static void
 vdev_run_cmd_thread(void *cb_cmd_data)
 {
 	vdev_cmd_data_t *data = cb_cmd_data;
-	const char *sep = ",";
-	char *cmd = NULL, *cmddup, *rest;
-	char fullpath[MAXPATHLEN];
+	char *cmd = NULL, *cmddup, *cmdrest;
 
 	cmddup = strdup(data->cmd);
 	if (cmddup == NULL)
 		return;
 
-	rest = cmddup;
-	while ((cmd = strtok_r(rest, sep, &rest))) {
-		if (snprintf(fullpath, sizeof (fullpath), "%s/%s",
-		    ZPOOL_SCRIPTS_DIR, cmd) == -1)
+	cmdrest = cmddup;
+	while ((cmd = strtok_r(cmdrest, ",", &cmdrest))) {
+		char *dir = NULL, *sp, *sprest;
+		char fullpath[MAXPATHLEN];
+
+		sp = zpool_get_cmd_search_path();
+		if (sp == NULL)
 			continue;
 
-		/* Does the script exist in our zpool scripts dir? */
-		if (access(fullpath, X_OK) == 0)
-			vdev_run_cmd(data, fullpath);
+		sprest = sp;
+		while ((dir = strtok_r(sprest, ":", &sprest))) {
+			if (snprintf(fullpath, sizeof (fullpath),
+			    "%s/%s", dir, cmd) == -1)
+				continue;
+
+			if (access(fullpath, X_OK) == 0) {
+				vdev_run_cmd(data, fullpath);
+				break;
+			}
+		}
+		free(sp);
 	}
 	free(cmddup);
 }
diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c
index f99145da9..5fccd754b 100644
--- a/cmd/zpool/zpool_main.c
+++ b/cmd/zpool/zpool_main.c
@@ -4190,25 +4190,22 @@ print_zpool_script_help(char *name, char *path)
 	libzfs_free_str_array(lines, lines_cnt);
 }
 
-
 /*
- * Go though the list of all the zpool status/iostat -c scripts, run their
+ * Go though the zpool status/iostat -c scripts in the user's path, run their
  * help option (-h), and print out the results.
  */
 static void
-print_zpool_script_list(void)
+print_zpool_dir_scripts(char *dirpath)
 {
 	DIR *dir;
 	struct dirent *ent;
 	char fullpath[MAXPATHLEN];
 	struct stat dir_stat;
 
-	if ((dir = opendir(ZPOOL_SCRIPTS_DIR)) != NULL) {
-		printf("\n");
+	if ((dir = opendir(dirpath)) != NULL) {
 		/* print all the files and directories within directory */
 		while ((ent = readdir(dir)) != NULL) {
-			sprintf(fullpath, "%s/%s", ZPOOL_SCRIPTS_DIR,
-			    ent->d_name);
+			sprintf(fullpath, "%s/%s", dirpath, ent->d_name);
 
 			/* Print the scripts */
 			if (stat(fullpath, &dir_stat) == 0)
@@ -4217,15 +4214,34 @@ print_zpool_script_list(void)
 					print_zpool_script_help(ent->d_name,
 					    fullpath);
 		}
-		printf("\n");
 		closedir(dir);
-	} else {
-		fprintf(stderr, gettext("Can't open %s scripts dir\n"),
-		    ZPOOL_SCRIPTS_DIR);
 	}
 }
 
 /*
+ * Print out help text for all zpool status/iostat -c scripts.
+ */
+static void
+print_zpool_script_list(char *subcommand)
+{
+	char *dir, *sp;
+
+	printf(gettext("Available 'zpool %s -c' commands:\n"), subcommand);
+
+	sp = zpool_get_cmd_search_path();
+	if (sp == NULL)
+		return;
+
+	dir = strtok(sp, ":");
+	while (dir != NULL) {
+		print_zpool_dir_scripts(dir);
+		dir = strtok(NULL, ":");
+	}
+
+	free(sp);
+}
+
+/*
  * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
  *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
  *              [interval [count]]
@@ -4285,6 +4301,15 @@ zpool_do_iostat(int argc, char **argv)
 				    gettext("Can't set -c flag twice\n"));
 				exit(1);
 			}
+
+			if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
+			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
+				fprintf(stderr, gettext(
+				    "Can't run -c, disabled by "
+				    "ZPOOL_SCRIPTS_ENABLED.\n"));
+				exit(1);
+			}
+
 			if ((getuid() <= 0 || geteuid() <= 0) &&
 			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
 				fprintf(stderr, gettext(
@@ -4336,10 +4361,7 @@ zpool_do_iostat(int argc, char **argv)
 			break;
 		case '?':
 			if (optopt == 'c') {
-				fprintf(stderr, gettext(
-				    "Current scripts in %s:\n"),
-				    ZPOOL_SCRIPTS_DIR);
-				print_zpool_script_list();
+				print_zpool_script_list("iostat");
 				exit(0);
 			} else {
 				fprintf(stderr,
@@ -6427,6 +6449,15 @@ zpool_do_status(int argc, char **argv)
 				    gettext("Can't set -c flag twice\n"));
 				exit(1);
 			}
+
+			if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
+			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
+				fprintf(stderr, gettext(
+				    "Can't run -c, disabled by "
+				    "ZPOOL_SCRIPTS_ENABLED.\n"));
+				exit(1);
+			}
+
 			if ((getuid() <= 0 || geteuid() <= 0) &&
 			    !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
 				fprintf(stderr, gettext(
@@ -6459,10 +6490,7 @@ zpool_do_status(int argc, char **argv)
 			break;
 		case '?':
 			if (optopt == 'c') {
-				fprintf(stderr, gettext(
-				    "Current scripts in %s:\n"),
-				    ZPOOL_SCRIPTS_DIR);
-				print_zpool_script_list();
+				print_zpool_script_list("status");
 				exit(0);
 			} else {
 				fprintf(stderr,
diff --git a/cmd/zpool/zpool_util.h b/cmd/zpool/zpool_util.h
index ab7662cfa..aef2cff27 100644
--- a/cmd/zpool/zpool_util.h
+++ b/cmd/zpool/zpool_util.h
@@ -45,6 +45,11 @@ uint64_t array64_max(uint64_t array[], unsigned int len);
 int isnumber(char *str);
 
 /*
+ * Misc utility functions
+ */
+char *zpool_get_cmd_search_path(void);
+
+/*
  * Virtual device functions
  */