Add zpool events tests

* events_001_pos - Verify the expected events are generated when
  invoking the various zpool sub-commands.  These events must
  appear in `zpool event` and be consumed by the ZED.

* events_002_pos - Verify the ZED consumes events which were
  generated while it wasn't running when it is started.
  Additionally, verify that events are only processed once.

As part of this change the default.cfg used by the test suite
was changed to a default.cfg.in file.  This was needed so the
install location of all zed scripts, not only the enabled ones,
could be reliably determined.

Signed-off-by: Brian Behlendorf <[email protected]>
Closes #6128

15 files changed, 479 insertions, 10 deletions

diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index e83a9ed0f..7e64c6714 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -344,6 +344,9 @@ tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos',
 [tests/functional/devices]
 tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos']
 
+[tests/functional/events]
+tests = ['events_001_pos', 'events_002_pos']
+
 [tests/functional/exec]
 tests = ['exec_001_pos', 'exec_002_neg']
 
diff --git a/tests/zfs-tests/include/.gitignore b/tests/zfs-tests/include/.gitignore
new file mode 100644
index 000000000..30d8a77c1
--- /dev/null
+++ b/tests/zfs-tests/include/.gitignore
@@ -0,0 +1 @@
+/default.cfg
diff --git a/tests/zfs-tests/include/Makefile.am b/tests/zfs-tests/include/Makefile.am
index d6fb32b61..579e1356e 100644
--- a/tests/zfs-tests/include/Makefile.am
+++ b/tests/zfs-tests/include/Makefile.am
@@ -6,3 +6,8 @@ dist_pkgdata_SCRIPTS = \
 	math.shlib \
 	properties.shlib \
 	zpool_script.shlib
+
+EXTRA_DIST=default.cfg.in
+
+distclean-local::
+	-$(RM) $(dist_pkgdata_SCRIPTS)
diff --git a/tests/zfs-tests/include/default.cfg b/tests/zfs-tests/include/default.cfg.in
index 84edce737..9ac74f29c 100644
--- a/tests/zfs-tests/include/default.cfg
+++ b/tests/zfs-tests/include/default.cfg.in
@@ -33,8 +33,11 @@
 . $STF_SUITE/include/libtest.shlib
 
 # ZFS Directories
-export ZEDLETDIR=${ZEDLETDIR:-/etc/zfs/zed.d}
-export ZPOOLSCRIPTDIR=${ZPOOLSCRIPTDIR:-/etc/zfs/zpool.d}
+export prefix=@prefix@
+export exec_prefix=@exec_prefix@
+export ZEDLET_ETC_DIR=${ZEDLET_ETC_DIR:-@sysconfdir@/zfs/zed.d}
+export ZEDLET_LIBEXEC_DIR=${ZEDLET_LIBEXEC_DIR:-@libexecdir@/zfs/zed.d}
+export ZPOOL_SCRIPT_DIR=${ZPOOL_SCRIPT_DIR:-@sysconfdir@/zfs/zpool.d}
 
 # Define run length constants
 export RT_LONG="3"
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index 56f765d20..df6b94f58 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -3280,11 +3280,16 @@ function zed_setup
 
 	# Setup minimal ZED configuration.  Individual test cases should
 	# add additional ZEDLETs as needed for their specific test.
-	log_must cp ${ZEDLETDIR}/zed.rc $ZEDLET_DIR
-	log_must cp ${ZEDLETDIR}/zed-functions.sh $ZEDLET_DIR
-	log_must cp ${ZEDLETDIR}/all-syslog.sh $ZEDLET_DIR
+	log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR
+	log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR
 
-	log_must zpool events -c
+	# Customize the zed.rc file to enable the full debug log.
+	log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc
+	echo "ZED_DEBUG_LOG=$ZEDLET_DIR/zed.debug.log" >>$ZEDLET_DIR/zed.rc
+
+	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-syslog.sh $ZEDLET_DIR
+	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-debug.sh $ZEDLET_DIR
+	log_must touch $ZEDLET_DIR/zed.debug.log
 }
 
 #
@@ -3299,8 +3304,10 @@ function zed_cleanup
 	log_must rm -f ${ZEDLET_DIR}/zed.rc
 	log_must rm -f ${ZEDLET_DIR}/zed-functions.sh
 	log_must rm -f ${ZEDLET_DIR}/all-syslog.sh
+	log_must rm -f ${ZEDLET_DIR}/all-debug.sh
 	log_must rm -f ${ZEDLET_DIR}/zed.pid
 	log_must rm -f ${ZEDLET_DIR}/zedlog
+	log_must rm -f ${ZEDLET_DIR}/zed.debug.log
 	log_must rm -f ${ZEDLET_DIR}/state
 	log_must rm -f $VDEVID_CONF_ETC
 	log_must rm -f $VDEVID_CONF
@@ -3332,6 +3339,8 @@ function zed_start
 	# output to zedlog
 	log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid" \
 	    "-s $ZEDLET_DIR/state 2>${ZEDLET_DIR}/zedlog &"
+
+	return 0
 }
 
 #
@@ -3343,10 +3352,13 @@ function zed_stop
 		return
 	fi
 
+	log_note "Stopping ZED"
 	if [[ -f ${ZEDLET_DIR}/zed.pid ]]; then
 		zedpid=$(cat ${ZEDLET_DIR}/zed.pid)
 		log_must kill $zedpid
 	fi
+
+	return 0
 }
 
 #
diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am
index a2d78f520..1c3f4b1a8 100644
--- a/tests/zfs-tests/tests/functional/Makefile.am
+++ b/tests/zfs-tests/tests/functional/Makefile.am
@@ -14,6 +14,7 @@ SUBDIRS = \
 	ctime \
 	delegate \
 	devices \
+	events \
 	exec \
 	fault \
 	features \
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh
index cf7959161..c5e0c6e47 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh
@@ -53,10 +53,10 @@ else
 	testpool="${TESTPOOL%%/*}"
 fi
 
-files="$(ls $ZPOOLSCRIPTDIR)"
+files="$(ls $ZPOOL_SCRIPT_DIR)"
 scripts=""
 for i in $files ; do
-	if [ ! -x "$ZPOOLSCRIPTDIR/$i" ] ; then
+	if [ ! -x "$ZPOOL_SCRIPT_DIR/$i" ] ; then
 		# Skip non-executables
 		continue
 	fi
diff --git a/tests/zfs-tests/tests/functional/cli_user/zpool_iostat/zpool_iostat_005_pos.ksh b/tests/zfs-tests/tests/functional/cli_user/zpool_iostat/zpool_iostat_005_pos.ksh
index 0528e0c0a..1ae91c1a8 100755
--- a/tests/zfs-tests/tests/functional/cli_user/zpool_iostat/zpool_iostat_005_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_user/zpool_iostat/zpool_iostat_005_pos.ksh
@@ -53,10 +53,10 @@ else
 	testpool=${TESTPOOL%%/*}
 fi
 
-files="$(ls $ZPOOLSCRIPTDIR)"
+files="$(ls $ZPOOL_SCRIPT_DIR)"
 scripts=""
 for i in $files ; do
-	if [ ! -x "$ZPOOLSCRIPTDIR/$i" ] ; then
+	if [ ! -x "$ZPOOL_SCRIPT_DIR/$i" ] ; then
 		# Skip non-executables
 		continue
 	fi
diff --git a/tests/zfs-tests/tests/functional/events/Makefile.am b/tests/zfs-tests/tests/functional/events/Makefile.am
new file mode 100644
index 000000000..7813c18be
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/events/Makefile.am
@@ -0,0 +1,8 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/events
+dist_pkgdata_SCRIPTS = \
+	setup.ksh \
+	cleanup.ksh \
+	events.cfg \
+	events_common.kshlib \
+	events_001_pos.ksh \
+	events_002_pos.ksh
diff --git a/tests/zfs-tests/tests/functional/events/cleanup.ksh b/tests/zfs-tests/tests/functional/events/cleanup.ksh
new file mode 100755
index 000000000..bc536e260
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/events/cleanup.ksh
@@ -0,0 +1,31 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+zed_cleanup
+
+default_cleanup
diff --git a/tests/zfs-tests/tests/functional/events/events.cfg b/tests/zfs-tests/tests/functional/events/events.cfg
new file mode 100644
index 000000000..3fba819b6
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/events/events.cfg
@@ -0,0 +1,38 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+export MPOOL=mpool.$$
+
+VDEV1=$TEST_BASE_DIR/vdev1
+VDEV2=$TEST_BASE_DIR/vdev2
+VDEV3=$TEST_BASE_DIR/vdev3
+VDEV4=$TEST_BASE_DIR/vdev4
+
+export TMP_EVENTS=/tmp/tmp_events.$$
+export TMP_EVENTS_FULL=/tmp/tmp_events_full.$$
+export TMP_EVENT_FULL=/tmp/tmp_event_full.$$
+export TMP_EVENTS_ZED=/tmp/tmp_events_zed.$$
+export TMP_EVENT_ZED=/tmp/tmp_event_zed.$$
diff --git a/tests/zfs-tests/tests/functional/events/events_001_pos.ksh b/tests/zfs-tests/tests/functional/events/events_001_pos.ksh
new file mode 100755
index 000000000..aaf2ad86c
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/events/events_001_pos.ksh
@@ -0,0 +1,134 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+# DESCRIPTION:
+# Verify zpool events command logs events.
+#
+# STRATEGY:
+# 1. Execute zpool sub-commands on a pool.
+# 2. Verify the expected events are logged in 'zpool events'.
+# 3. Verify the expected events are logged by the ZED.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+	if poolexists $MPOOL; then
+		destroy_pool $MPOOL
+	fi
+
+	for file in $VDEV1 $VDEV2 $VDEV3 $VDEV4; do
+		[[ -f $file ]] && rm -f $file
+	done
+
+	log_must zed_stop
+}
+
+log_assert "Verify zpool sub-commands generate expected events"
+log_onexit cleanup
+
+log_must truncate -s $MINVDEVSIZE $VDEV1 $VDEV2 $VDEV3 $VDEV4
+log_must zed_start
+
+# Create a mirrored pool with two devices.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.config_sync" \
+    -e "sysevent.fs.zfs.pool_create" \
+    "zpool create $MPOOL mirror $VDEV1 $VDEV2"
+
+# Add a cache device then remove it.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.config_sync" \
+    -e "sysevent.fs.zfs.vdev_add" \
+    "zpool add -f $MPOOL spare $VDEV3"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_remove_aux" \
+    "zpool remove $MPOOL $VDEV3"
+
+# Add a log device then remove it.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.config_sync" \
+    -e "sysevent.fs.zfs.vdev_add" \
+    "zpool add -f $MPOOL log $VDEV3"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_remove_dev" \
+    "zpool remove $MPOOL $VDEV3"
+
+# Offline then online a device.
+run_and_verify -p "$MPOOL"\
+    -e "resource.fs.zfs.statechange" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool offline $MPOOL $VDEV1"
+run_and_verify -p "$MPOOL" \
+    -e "resource.fs.zfs.statechange" \
+    -e "sysevent.fs.zfs.vdev_online" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool online $MPOOL $VDEV1"
+
+# Attach then detach a device from the mirror.
+run_and_verify -p "$MPOOL" \
+     -e "sysevent.fs.zfs.vdev_attach" \
+    "zpool attach $MPOOL $VDEV1 $VDEV4"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_remove" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool detach $MPOOL $VDEV4"
+
+# Replace a device
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_attach" \
+    -e "sysevent.fs.zfs.resilver_start" \
+    -e "sysevent.fs.zfs.resilver_finish" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool replace -f $MPOOL $VDEV1 $VDEV4"
+
+# Scrub a pool.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.scrub_start" \
+    -e "sysevent.fs.zfs.scrub_finish" \
+    "zpool scrub $MPOOL"
+
+# Export then import a pool (may change to a pool_export event)
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.pool_destroy" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool export $MPOOL"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.pool_import" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool import -d $TEST_BASE_DIR $MPOOL"
+
+# Destroy the pool
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.pool_destroy" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool destroy $MPOOL"
+
+log_pass "Verify zpool sub-commands generate expected events"
diff --git a/tests/zfs-tests/tests/functional/events/events_002_pos.ksh b/tests/zfs-tests/tests/functional/events/events_002_pos.ksh
new file mode 100755
index 000000000..7769b45c8
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/events/events_002_pos.ksh
@@ -0,0 +1,102 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+# DESCRIPTION:
+# Verify ZED handles missed events from a pool when starting.
+#
+# STRATEGY:
+# 1. Create a pool and generate some events.
+# 2. Start the ZED and verify it handles missed events.
+# 3. Stop the ZED
+# 4. Generate additional events.
+# 5. Start the ZED and verify it only handles the new missed events.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+	if poolexists $MPOOL; then
+		destroy_pool $MPOOL
+	fi
+
+	for file in $VDEV1 $VDEV2; do
+		[[ -f $file ]] && rm -f $file
+	done
+
+	log_must rm -f $TMP_EVENTS_ZED $TMP_EVENTS_ZED
+	log_must rm -f $ZEDLET_DIR/zed.debug.log.old
+	log_must zed_stop
+}
+
+log_assert "Verify ZED handles missed events on when starting"
+log_onexit cleanup
+
+log_must truncate -s $MINVDEVSIZE $VDEV1 $VDEV2
+
+# 1. Create a pool and generate some events.
+log_must cp -f $ZEDLET_DIR/zed.debug.log $ZEDLET_DIR/zed.debug.log.old
+log_must zpool create $MPOOL mirror $VDEV1 $VDEV2
+
+# 2. Start the ZED and verify it handles missed events.
+log_must zed_start
+log_must sleep 1
+diff $ZEDLET_DIR/zed.debug.log.old $ZEDLET_DIR/zed.debug.log | \
+    grep "^> " | sed 's/^> //g' >$TMP_EVENTS_ZED
+log_must awk -v event="sysevent.fs.zfs.pool_create" \
+    'BEGIN{FS="\n"; RS=""} $0 ~ event { print $0 }' \
+    $TMP_EVENTS_ZED >$TMP_EVENT_ZED
+log_must grep -q "^ZEVENT_POOL=$MPOOL" $TMP_EVENT_ZED
+
+# 3. Stop the ZED
+zed_stop
+
+# 4. Generate additional events.
+log_must cp -f $ZEDLET_DIR/zed.debug.log $ZEDLET_DIR/zed.debug.log.old
+log_must zpool offline $MPOOL $VDEV1
+log_must zpool online $MPOOL $VDEV1
+log_must zpool scrub $MPOOL
+
+# Wait for the scrub to wrap, or is_healthy will be wrong.
+while ! is_pool_scrubbed $MPOOL; do
+	sleep 1
+done
+
+# 5. Start the ZED and verify it only handled the new missed events.
+log_must zed_start
+log_must sleep 3
+diff $ZEDLET_DIR/zed.debug.log.old $ZEDLET_DIR/zed.debug.log | \
+    grep "^> " | sed 's/^> //g' >$TMP_EVENTS_ZED
+
+log_mustnot grep -q "sysevent.fs.zfs.pool_create" $TMP_EVENTS_ZED
+log_must grep -q "sysevent.fs.zfs.vdev_online" $TMP_EVENTS_ZED
+log_must grep -q "sysevent.fs.zfs.resilver_start" $TMP_EVENTS_ZED
+log_must grep -q "sysevent.fs.zfs.resilver_finish" $TMP_EVENTS_ZED
+
+log_pass "Verify ZED handles missed events on when starting"
diff --git a/tests/zfs-tests/tests/functional/events/events_common.kshlib b/tests/zfs-tests/tests/functional/events/events_common.kshlib
new file mode 100755
index 000000000..afefef1f0
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/events/events_common.kshlib
@@ -0,0 +1,98 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events.cfg
+
+function run_and_verify
+{
+	typeset event pool
+	set -A events
+
+	while getopts "e:p:z:" opt; do
+		case $opt in
+		e)
+			events[${#events[*]}+1]=$OPTARG
+			;;
+		p)
+			pool=$OPTARG
+			;;
+		z)
+			zedlog=$OPTARG
+			;;
+		esac
+	done
+	shift $(($OPTIND - 1))
+
+	pool=${pool:-$TESTPOOL}
+	zedlog=${zedlog:-$ZEDLET_DIR/zed.debug.log}
+	fullcmd="$1"
+	cmd=$(echo $fullcmd | awk '{print $1}')
+	subcmd=$(echo $fullcmd | awk '{print $2}')
+
+	# If we aren't running zpool or zfs, something is wrong
+	[[ $cmd == "zpool" || $cmd == "zfs" ]] || \
+	    log_fail "run_and_verify called with \"$cmd ($fullcmd)\""
+
+	# Run the command as provided and collect the new events.
+	log_must zpool events -c
+	if [[ -f $zedlog ]]; then
+		cp -f $zedlog $zedlog.old
+	fi
+
+	log_must eval "$fullcmd"
+
+	log_must zpool events > $TMP_EVENTS 2>/dev/null
+	log_must zpool events -v > $TMP_EVENTS_FULL 2>/dev/null
+	if [[ -f $zedlog ]]; then
+		sleep 1 # Brief delay for the ZED to handle the event.
+		diff $zedlog.old $zedlog | grep "^> " | sed 's/^> //g' \
+		    >$TMP_EVENTS_ZED
+	fi
+
+	# Verify all the expected events appear in the log.
+	for event in ${events[*]}; do
+
+		# Verify the event is in in the short output.
+		log_must grep -q "$event" $TMP_EVENTS
+
+		# Verify the event is in the verbose output with pool name.
+		log_must awk -v event="$event" \
+		    'BEGIN{FS="\n"; RS=""} $0 ~ event { print $0 }' \
+		    $TMP_EVENTS_FULL >$TMP_EVENT_FULL
+		log_must grep -q "pool = \"$pool\"" $TMP_EVENT_FULL
+
+		# Verify the event was received by the ZED and logged.
+		log_must awk -v event="$event" \
+		    'BEGIN{FS="\n"; RS=""} $0 ~ event { print $0 }' \
+		    $TMP_EVENTS_ZED >$TMP_EVENT_ZED
+		log_must grep -q "^ZEVENT_POOL=$pool" $TMP_EVENT_ZED
+
+	done
+
+	log_must rm -f $TMP_EVENTS $TMP_EVENTS_FULL $TMP_EVENT_FULL \
+	    $TMP_EVENTS_ZED $TMP_EVENT_ZED $zedlog.old
+}
diff --git a/tests/zfs-tests/tests/functional/events/setup.ksh b/tests/zfs-tests/tests/functional/events/setup.ksh
new file mode 100755
index 000000000..7113c1f39
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/events/setup.ksh
@@ -0,0 +1,33 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+DISK=${DISKS%% *}
+
+zed_setup
+
+default_setup $DISK