Add scrub after resilver zed script

* Add a zed script to kick off a scrub after a resilver.  The script is
disabled by default.

* Add a optional $PATH (-P) option to zed to allow it to use a custom
$PATH for its zedlets.  This is needed when you're running zed under
the ZTS in a local workspace.

* Update test scripts to not copy in all-debug.sh and all-syslog.sh by
default.  They can be optionally copied in as part of zed_setup().
These scripts slow down zed considerably under heavy events loads and
can cause events to be dropped or their delivery delayed. This was
causing some sporadic failures in the 'fault' tests.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Richard Laager <[email protected]>
Signed-off-by: Tony Hutter <[email protected]>
Closes #4662
Closes #7086

2 files changed, 44 insertions, 7 deletions

diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg
index f6fd239de..936e54c1a 100644
--- a/tests/zfs-tests/include/commands.cfg
+++ b/tests/zfs-tests/include/commands.cfg
@@ -83,6 +83,7 @@ export SYSTEM_FILES='arp
     pgrep
     ping
     pkill
+    printenv
     printf
     ps
     pwd
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index 86f172a6d..48fb5e7c5 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -3340,8 +3340,31 @@ function wait_replacing #pool
 }
 
 #
+# Wait for a pool to be scrubbed
+#
+# $1 pool name
+# $2 number of seconds to wait (optional)
+#
+# Returns true when pool has been scrubbed, or false if there's a timeout or if
+# no scrub was done.
+#
+function wait_scrubbed
+{
+	typeset pool=${1:-$TESTPOOL}
+	typeset iter=${2:-10}
+	for i in {1..$iter} ; do
+		if is_pool_scrubbed $pool ; then
+			return 0
+		fi
+		sleep 1
+	done
+	return 1
+}
+
+#
 # Setup custom environment for the ZED.
 #
+# $@ Optional list of zedlets to run under zed.
 function zed_setup
 {
 	if ! is_linux; then
@@ -3359,6 +3382,7 @@ function zed_setup
 	if [[ -e $VDEVID_CONF_ETC ]]; then
 		log_fail "Must not have $VDEVID_CONF_ETC file present on system"
 	fi
+	EXTRA_ZEDLETS=$@
 
 	# Create a symlink for /etc/zfs/vdev_id.conf file.
 	log_must ln -s $VDEVID_CONF $VDEVID_CONF_ETC
@@ -3368,32 +3392,44 @@ function zed_setup
 	log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR
 	log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR
 
+	# Scripts must only be user writable.
+	if [[ -n "$EXTRA_ZEDLETS" ]] ; then
+		saved_umask=$(umask)
+		log_must umask 0022
+		for i in $EXTRA_ZEDLETS ; do
+			log_must cp ${ZEDLET_LIBEXEC_DIR}/$i $ZEDLET_DIR
+		done
+		log_must umask $saved_umask
+	fi
+
 	# Customize the zed.rc file to enable the full debug log.
 	log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc
 	echo "ZED_DEBUG_LOG=$ZED_DEBUG_LOG" >>$ZEDLET_DIR/zed.rc
 
-	# Scripts must only be user writable.
-	saved_umask=$(umask)
-	log_must umask 0022
-	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-syslog.sh $ZEDLET_DIR
-	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-debug.sh $ZEDLET_DIR
-	log_must umask $saved_umask
 }
 
 #
 # Cleanup custom ZED environment.
 #
+# $@ Optional list of zedlets to remove from our test zed.d directory.
 function zed_cleanup
 {
 	if ! is_linux; then
 		return
 	fi
+	EXTRA_ZEDLETS=$@
 
 	log_must rm -f ${ZEDLET_DIR}/zed.rc
 	log_must rm -f ${ZEDLET_DIR}/zed-functions.sh
 	log_must rm -f ${ZEDLET_DIR}/all-syslog.sh
 	log_must rm -f ${ZEDLET_DIR}/all-debug.sh
 	log_must rm -f ${ZEDLET_DIR}/state
+
+	if [[ -n "$EXTRA_ZEDLETS" ]] ; then
+		for i in $EXTRA_ZEDLETS ; do
+			log_must rm -f ${ZEDLET_DIR}/$i
+		done
+	fi
 	log_must rm -f $ZED_LOG
 	log_must rm -f $ZED_DEBUG_LOG
 	log_must rm -f $VDEVID_CONF_ETC
@@ -3425,7 +3461,7 @@ function zed_start
 	# run ZED in the background and redirect foreground logging
 	# output to $ZED_LOG.
 	log_must truncate -s 0 $ZED_DEBUG_LOG
-	log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid" \
+	log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid -P $PATH" \
 	    "-s $ZEDLET_DIR/state 2>$ZED_LOG &"
 
 	return 0