Add scrub after resilver zed script

* Add a zed script to kick off a scrub after a resilver.  The script is
disabled by default.

* Add a optional $PATH (-P) option to zed to allow it to use a custom
$PATH for its zedlets.  This is needed when you're running zed under
the ZTS in a local workspace.

* Update test scripts to not copy in all-debug.sh and all-syslog.sh by
default.  They can be optionally copied in as part of zed_setup().
These scripts slow down zed considerably under heavy events loads and
can cause events to be dropped or their delivery delayed. This was
causing some sporadic failures in the 'fault' tests.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Richard Laager <[email protected]>
Signed-off-by: Tony Hutter <[email protected]>
Closes #4662 
Closes #7086 

8 files changed, 115 insertions, 12 deletions

diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg
index 4aede9f09..47926abfa 100644
--- a/tests/zfs-tests/include/commands.cfg
+++ b/tests/zfs-tests/include/commands.cfg
@@ -86,6 +86,7 @@ export SYSTEM_FILES='arp
     pgrep
     ping
     pkill
+    printenv
     printf
     ps
     pwd
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index 6a0d51d17..da5456b6f 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -3054,8 +3054,31 @@ function wait_replacing #pool
 }
 
 #
+# Wait for a pool to be scrubbed
+#
+# $1 pool name
+# $2 number of seconds to wait (optional)
+#
+# Returns true when pool has been scrubbed, or false if there's a timeout or if
+# no scrub was done.
+#
+function wait_scrubbed
+{
+	typeset pool=${1:-$TESTPOOL}
+	typeset iter=${2:-10}
+	for i in {1..$iter} ; do
+		if is_pool_scrubbed $pool ; then
+			return 0
+		fi
+		sleep 1
+	done
+	return 1
+}
+
+#
 # Setup custom environment for the ZED.
 #
+# $@ Optional list of zedlets to run under zed.
 function zed_setup
 {
 	if ! is_linux; then
@@ -3073,6 +3096,7 @@ function zed_setup
 	if [[ -e $VDEVID_CONF_ETC ]]; then
 		log_fail "Must not have $VDEVID_CONF_ETC file present on system"
 	fi
+	EXTRA_ZEDLETS=$@
 
 	# Create a symlink for /etc/zfs/vdev_id.conf file.
 	log_must ln -s $VDEVID_CONF $VDEVID_CONF_ETC
@@ -3082,32 +3106,44 @@ function zed_setup
 	log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR
 	log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR
 
+	# Scripts must only be user writable.
+	if [[ -n "$EXTRA_ZEDLETS" ]] ; then
+		saved_umask=$(umask)
+		log_must umask 0022
+		for i in $EXTRA_ZEDLETS ; do
+			log_must cp ${ZEDLET_LIBEXEC_DIR}/$i $ZEDLET_DIR
+		done
+		log_must umask $saved_umask
+	fi
+
 	# Customize the zed.rc file to enable the full debug log.
 	log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc
 	echo "ZED_DEBUG_LOG=$ZED_DEBUG_LOG" >>$ZEDLET_DIR/zed.rc
 
-	# Scripts must only be user writable.
-	saved_umask=$(umask)
-	log_must umask 0022
-	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-syslog.sh $ZEDLET_DIR
-	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-debug.sh $ZEDLET_DIR
-	log_must umask $saved_umask
 }
 
 #
 # Cleanup custom ZED environment.
 #
+# $@ Optional list of zedlets to remove from our test zed.d directory.
 function zed_cleanup
 {
 	if ! is_linux; then
 		return
 	fi
+	EXTRA_ZEDLETS=$@
 
 	log_must rm -f ${ZEDLET_DIR}/zed.rc
 	log_must rm -f ${ZEDLET_DIR}/zed-functions.sh
 	log_must rm -f ${ZEDLET_DIR}/all-syslog.sh
 	log_must rm -f ${ZEDLET_DIR}/all-debug.sh
 	log_must rm -f ${ZEDLET_DIR}/state
+
+	if [[ -n "$EXTRA_ZEDLETS" ]] ; then
+		for i in $EXTRA_ZEDLETS ; do
+			log_must rm -f ${ZEDLET_DIR}/$i
+		done
+	fi
 	log_must rm -f $ZED_LOG
 	log_must rm -f $ZED_DEBUG_LOG
 	log_must rm -f $VDEVID_CONF_ETC
@@ -3139,7 +3175,7 @@ function zed_start
 	# run ZED in the background and redirect foreground logging
 	# output to $ZED_LOG.
 	log_must truncate -s 0 $ZED_DEBUG_LOG
-	log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid" \
+	log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid -P $PATH" \
 	    "-s $ZEDLET_DIR/state 2>$ZED_LOG &"
 
 	return 0
diff --git a/tests/zfs-tests/tests/functional/events/cleanup.ksh b/tests/zfs-tests/tests/functional/events/cleanup.ksh
index bc536e260..4905342b7 100755
--- a/tests/zfs-tests/tests/functional/events/cleanup.ksh
+++ b/tests/zfs-tests/tests/functional/events/cleanup.ksh
@@ -26,6 +26,6 @@
 
 . $STF_SUITE/include/libtest.shlib
 
-zed_cleanup
+zed_cleanup all-debug.sh all-syslog.sh
 
 default_cleanup
diff --git a/tests/zfs-tests/tests/functional/events/setup.ksh b/tests/zfs-tests/tests/functional/events/setup.ksh
index 7113c1f39..2f81d16b1 100755
--- a/tests/zfs-tests/tests/functional/events/setup.ksh
+++ b/tests/zfs-tests/tests/functional/events/setup.ksh
@@ -28,6 +28,6 @@
 
 DISK=${DISKS%% *}
 
-zed_setup
+zed_setup all-debug.sh all-syslog.sh
 
 default_setup $DISK
diff --git a/tests/zfs-tests/tests/functional/fault/Makefile.am b/tests/zfs-tests/tests/functional/fault/Makefile.am
index ef4380835..8b8221147 100644
--- a/tests/zfs-tests/tests/functional/fault/Makefile.am
+++ b/tests/zfs-tests/tests/functional/fault/Makefile.am
@@ -8,4 +8,5 @@ dist_pkgdata_SCRIPTS = \
 	auto_spare_001_pos.ksh \
 	auto_spare_002_pos.ksh \
 	auto_spare_ashift.ksh \
-	auto_spare_multiple.ksh
+	auto_spare_multiple.ksh \
+	scrub_after_resilver.ksh
diff --git a/tests/zfs-tests/tests/functional/fault/cleanup.ksh b/tests/zfs-tests/tests/functional/fault/cleanup.ksh
index 9d354f30e..45b94723a 100755
--- a/tests/zfs-tests/tests/functional/fault/cleanup.ksh
+++ b/tests/zfs-tests/tests/functional/fault/cleanup.ksh
@@ -31,6 +31,6 @@ verify_runnable "global"
 cleanup_devices $DISKS
 
 zed_stop
-zed_cleanup
+zed_cleanup resilver_finish-start-scrub.sh
 
 log_pass
diff --git a/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh b/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh
new file mode 100755
index 000000000..558cb065f
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh
@@ -0,0 +1,65 @@
+#!/bin/ksh -p
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+# All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/fault/fault.cfg
+
+#
+# DESCRIPTION:
+# Test the scrub after resilver zedlet
+#
+# STRATEGY:
+# 1. Create a mirrored pool
+# 2. Fault a disk
+# 3. Replace the disk, starting a resilver
+# 4. Verify that a scrub happens after the resilver finishes
+#
+
+log_assert "Testing the scrub after resilver zedlet"
+
+# Backup our zed.rc
+zedrc_backup="$(mktemp)"
+log_must cp $ZEDLET_DIR/zed.rc $zedrc_backup
+
+# Enable ZED_SCRUB_AFTER_RESILVER
+eval "sed -i 's/\#ZED_SCRUB_AFTER_RESILVER/ZED_SCRUB_AFTER_RESILVER/g' $ZEDLET_DIR/zed.rc"
+
+function cleanup
+{
+	# Restore our zed.rc
+	log_must mv $zedrc_backup $ZEDLET_DIR/zed.rc
+	default_cleanup_noexit
+}
+
+log_onexit cleanup
+
+verify_disk_count "$DISKS" 3
+default_mirror_setup_noexit $DISK1 $DISK2
+
+log_must zpool offline -f $TESTPOOL $DISK1
+
+# Write to our degraded pool so we have some data to resilver
+log_must mkfile 16M $TESTDIR/file1
+
+# Replace the failed disks, forcing a resilver
+log_must zpool replace $TESTPOOL $DISK1 $DISK3
+
+# Wait for the resilver to finish, and then the subsequent scrub to finish.
+# Waiting for the scrub has the effect of waiting for both.  Timeout after 10
+# seconds if nothing is happening.
+log_must wait_scrubbed $TESTPOOL 10
+log_pass "Successfully ran the scrub after resilver zedlet"
diff --git a/tests/zfs-tests/tests/functional/fault/setup.ksh b/tests/zfs-tests/tests/functional/fault/setup.ksh
index 3d3cbc9e5..b78ee8ccd 100755
--- a/tests/zfs-tests/tests/functional/fault/setup.ksh
+++ b/tests/zfs-tests/tests/functional/fault/setup.ksh
@@ -28,7 +28,7 @@
 
 verify_runnable "global"
 
-zed_setup
+zed_setup resilver_finish-start-scrub.sh
 zed_start
 
 log_pass