Added auto-replace FMA test for the ZFS Test Suite

Also included are updates to auto-online test

Automated auto-replace test to go along with ZED FMA integration
(PR 4673) auto-replace_001.pos works using a scsi_debug device
(the only usable virtual device currently due to whole_disk var
needing to be set)

Functionality for automated FMA auto-replace test to work with
scsi_debug devs:  Some functionality/exceptions needed to be
added for automation of auto-replace to work correctly.

In the test an alias vdev_id rule is added for any scsi_debug
device which sets the phys_path="scsidebug" after a udevadm
trigger command.

A symlink is created for the vdev_id.conf file (in /etc/zfs/ by
default) to be used in-tree for the test suite
(/var/tmp/zfs/vdev_id.conf).  "./scripts/zfs-helpers.sh -i" needs
to be run before fault tests in the ZTS (to use udev rules in-tree)

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Don Brady <[email protected]>
Reviewed-by: David Quigley <[email protected]>
Signed-off-by: Sydney Vanda <[email protected]>
Closes #5944

11 files changed, 371 insertions, 114 deletions

diff --git a/cmd/zed/agents/zfs_mod.c b/cmd/zed/agents/zfs_mod.c
index d297ab294..7ebf708ae 100644
--- a/cmd/zed/agents/zfs_mod.c
+++ b/cmd/zed/agents/zfs_mod.c
@@ -22,7 +22,7 @@
  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012 by Delphix. All rights reserved.
  * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2016, Intel Corporation.
+ * Copyright (c) 2016, 2017, Intel Corporation.
  */
 
 /*
@@ -89,6 +89,7 @@
 
 #define	DEV_BYID_PATH	"/dev/disk/by-id/"
 #define	DEV_BYPATH_PATH	"/dev/disk/by-path/"
+#define	DEV_BYVDEV_PATH	"/dev/disk/by-vdev/"
 
 typedef void (*zfs_process_func_t)(zpool_handle_t *, nvlist_t *, boolean_t);
 
@@ -190,6 +191,7 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	char devpath[PATH_MAX];
 	int ret;
 	int is_dm = 0;
+	int is_sd = 0;
 	uint_t c;
 	vdev_stat_t *vs;
 
@@ -259,6 +261,13 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	}
 
 	/*
+	 * vdev_id alias rule for using scsi_debug devices (FMA automated
+	 * testing)
+	 */
+	if (strcmp("scsidebug", physpath) == 0)
+		is_sd = 1;
+
+	/*
 	 * If the pool doesn't have the autoreplace property set, then use
 	 * vdev online to trigger a FMA fault by posting an ereport.
 	 */
@@ -272,10 +281,13 @@ zfs_process_add(zpool_handle_t *zhp, nvlist_t *vdev, boolean_t labeled)
 	}
 
 	/*
-	 * convert physical path into its current device node
+	 * Convert physical path into its current device node.  Rawpath
+	 * needs to be /dev/disk/by-vdev for a scsi_debug device since
+	 * /dev/disk/by-path will not be present.
 	 */
-	(void) snprintf(rawpath, sizeof (rawpath), "%s%s", DEV_BYPATH_PATH,
-	    physpath);
+	(void) snprintf(rawpath, sizeof (rawpath), "%s%s",
+	    is_sd ? DEV_BYVDEV_PATH : DEV_BYPATH_PATH, physpath);
+
 	if (realpath(rawpath, devpath) == NULL && !is_dm) {
 		zed_log_msg(LOG_INFO, "  realpath: %s failed (%s)",
 		    rawpath, strerror(errno));
diff --git a/cmd/zed/zed_disk_event.c b/cmd/zed/zed_disk_event.c
index ae39bd6a9..32eefb2f3 100644
--- a/cmd/zed/zed_disk_event.c
+++ b/cmd/zed/zed_disk_event.c
@@ -11,7 +11,7 @@
  */
 
 /*
- * Copyright (c) 2016, Intel Corporation.
+ * Copyright (c) 2016, 2017, Intel Corporation.
  */
 
 #ifdef HAVE_LIBUDEV
@@ -312,6 +312,31 @@ zed_udev_monitor(void *arg)
 			free(tmp2);
 		}
 
+		/*
+		 * Special case an EC_DEV_ADD for scsi_debug devices
+		 *
+		 * These devices require a udevadm trigger command after
+		 * creation in order to register the vdev_id scsidebug alias
+		 * rule (adds a persistent path (phys_path) used for fault
+		 * management automated tests in the ZFS test suite.
+		 *
+		 * After udevadm trigger command, event registers as a "change"
+		 * event but needs to instead be handled as another "add" event
+		 * to allow for disk labeling and partitioning to occur.
+		 */
+		if (strcmp(class, EC_DEV_STATUS) == 0 &&
+		    udev_device_get_property_value(dev, "ID_VDEV") &&
+		    udev_device_get_property_value(dev, "ID_MODEL")) {
+			const char *id_model, *id_model_sd = "scsi_debug";
+
+			id_model = udev_device_get_property_value(dev,
+			    "ID_MODEL");
+			if (strcmp(id_model, id_model_sd) == 0) {
+				class = EC_DEV_ADD;
+				subclass = ESC_DISK;
+			}
+		}
+
 		if ((nvl = dev_event_nvlist(dev)) != NULL) {
 			zed_udev_event(class, subclass, nvl);
 			nvlist_free(nvl);
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index 73f19ecfa..d8e39d01c 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -412,7 +412,7 @@ tests = ['devices_003_pos']
 tests = ['exec_001_pos']
 
 [tests/functional/fault]
-tests = ['auto_online_001_pos']
+tests = ['auto_online_001_pos', 'auto_replace_001_pos']
 
 [tests/functional/features/async_destroy]
 tests = ['async_destroy_001_pos']
diff --git a/tests/zfs-tests/include/default.cfg.in b/tests/zfs-tests/include/default.cfg.in
index f2241b541..4df1d844a 100644
--- a/tests/zfs-tests/include/default.cfg.in
+++ b/tests/zfs-tests/include/default.cfg.in
@@ -209,6 +209,7 @@ if is_linux; then
 	ZVOL_RDEVDIR="/dev/zvol"
 	DEV_RDSKDIR="/dev"
 	DEV_MPATHDIR="/dev/mapper"
+	ZEDLET_DIR="/var/tmp/zed"
 
 	NEWFS_DEFAULT_FS="ext2"
 else
@@ -226,4 +227,5 @@ else
 	NEWFS_DEFAULT_FS="ufs"
 fi
 export unpack_opts pack_opts verbose unpack_preserve pack_preserve \
-       ZVOL_DEVDIR ZVOL_RDEVDIR NEWFS_DEFAULT_FS DEV_RDSKDIR DEV_MPATHDIR
+       ZVOL_DEVDIR ZVOL_RDEVDIR NEWFS_DEFAULT_FS DEV_RDSKDIR DEV_MPATHDIR \
+       ZEDLET_DIR
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index d9aa1090d..159b4c79b 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -1679,13 +1679,18 @@ function scan_scsi_hosts
 {
 	typeset hostnum=${1}
 
-	if [[ -z $hostnum ]]; then
-		for host in /sys/class/scsi_host/host*; do
-			echo '- - -' > $host/scan
-		done
-	else
-		echo "/sys/class/scsi_host/host$hostnum/scan"
-		echo '- - -' > "/sys/class/scsi_host/host$hostnum/scan"
+	if is_linux; then
+		if [[ -z $hostnum ]]; then
+			for host in /sys/class/scsi_host/host*; do
+				log_must eval "$ECHO '- - -' > $host/scan"
+			done
+		else
+			log_must eval \
+			    "$ECHO /sys/class/scsi_host/host$hostnum/scan" \
+			    > /dev/null
+			log_must eval \
+			    "$ECHO '- - -' > /sys/class/scsi_host/host$hostnum/scan"
+		fi
 	fi
 }
 #
@@ -1758,7 +1763,7 @@ function on_off_disk # disk state{online,offline} host
 			fi
 		elif [[ $state == "online" ]]; then
 			#force a full rescan
-			log_must scan_scsi_hosts $host
+			scan_scsi_hosts $host
 			block_device_wait
 			if is_mpath_device $disk; then
 				dm_name="$($READLINK $DEV_DSKDIR/$disk \
@@ -3028,6 +3033,40 @@ function get_persistent_disk_name #device
 }
 
 #
+# Load scsi_debug module with specified parameters
+#
+function load_scsi_debug # dev_size_mb add_host num_tgts max_luns
+{
+	typeset devsize=$1
+	typeset hosts=$2
+	typeset tgts=$3
+	typeset luns=$4
+
+	[[ -z $devsize ]] || [[ -z $hosts ]] || [[ -z $tgts ]] || \
+	    [[ -z $luns ]] && log_fail "Arguments invalid or missing"
+
+	if is_linux; then
+		$MODLOAD -n scsi_debug
+		if (($? != 0)); then
+			log_unsupported "Platform does not have scsi_debug"
+			    "module"
+		fi
+		$LSMOD | $EGREP scsi_debug > /dev/zero
+		if (($? == 0)); then
+			log_fail "scsi_debug module already installed"
+		else
+			log_must $MODLOAD scsi_debug dev_size_mb=$devsize \
+			    add_host=$hosts num_tgts=$tgts max_luns=$luns
+			block_device_wait
+			$LSSCSI | $EGREP scsi_debug > /dev/null
+			if (($? == 1)); then
+				log_fail "scsi_debug module install failed"
+			fi
+		fi
+	fi
+}
+
+#
 # Get the package name
 #
 function get_package_name
@@ -3222,3 +3261,49 @@ function wait_freeing #pool
 		log_must $SLEEP 1
 	done
 }
+
+#
+# Check if ZED is currently running, if not start ZED.
+#
+function zed_start
+{
+	if is_linux; then
+		# ZEDLET_DIR=/var/tmp/zed
+		if [[ ! -d $ZEDLET_DIR ]]; then
+			log_must $MKDIR $ZEDLET_DIR
+		fi
+
+		# Verify the ZED is not already running.
+		$PGREP -x zed > /dev/null
+		if (($? == 0)); then
+			log_fail "ZED already running"
+		fi
+
+		log_must $CP ${ZEDLETDIR}/all-syslog.sh $ZEDLET_DIR
+
+		log_note "Starting ZED"
+		# run ZED in the background and redirect foreground logging
+		# output to zedlog
+		log_must eval "$ZED -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid" \
+		    "-s $ZEDLET_DIR/state 2>${ZEDLET_DIR}/zedlog &"
+	fi
+}
+
+#
+# Kill ZED process
+#
+function zed_stop
+{
+	if is_linux; then
+		if [[ -f ${ZEDLET_DIR}/zed.pid ]]; then
+			zedpid=$($CAT ${ZEDLET_DIR}/zed.pid)
+			log_must $KILL $zedpid
+		fi
+		log_must $RM -f ${ZEDLET_DIR}/all-syslog.sh
+		log_must $RM -f ${ZEDLET_DIR}/zed.pid
+		log_must $RM -f ${ZEDLET_DIR}/zedlog
+		log_must $RM -f ${ZEDLET_DIR}/state
+		log_must $RMDIR $ZEDLET_DIR
+
+	fi
+}
diff --git a/tests/zfs-tests/tests/functional/fault/Makefile.am b/tests/zfs-tests/tests/functional/fault/Makefile.am
index 19bb9428c..eeff31261 100644
--- a/tests/zfs-tests/tests/functional/fault/Makefile.am
+++ b/tests/zfs-tests/tests/functional/fault/Makefile.am
@@ -3,4 +3,5 @@ dist_pkgdata_SCRIPTS = \
 	fault.cfg \
 	setup.ksh \
 	cleanup.ksh \
-	auto_online_001_pos.ksh
+	auto_online_001_pos.ksh \
+	auto_replace_001_pos.ksh
diff --git a/tests/zfs-tests/tests/functional/fault/auto_online_001_pos.ksh b/tests/zfs-tests/tests/functional/fault/auto_online_001_pos.ksh
index 5bbc57dff..3321b5a60 100755
--- a/tests/zfs-tests/tests/functional/fault/auto_online_001_pos.ksh
+++ b/tests/zfs-tests/tests/functional/fault/auto_online_001_pos.ksh
@@ -20,7 +20,7 @@
 # CDDL HEADER END
 #
 #
-# Copyright (c) 2016 by Intel Corporation. All rights reserved.
+# Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -28,21 +28,23 @@
 
 #
 # DESCRIPTION:
-# Tesing auto-online FMA ZED logic.
+# Testing Fault Management Agent ZED Logic - Automated Auto-Online Test.
 #
 # STRATEGY:
 # 1. Create a pool
-# 2. export a pool
-# 3. offline disk
-# 4. import pool with missing disk
-# 5. online disk
+# 2. Export a pool
+# 3. Offline disk
+# 4. Import pool with missing disk
+# 5. Online disk
 # 6. ZED polls for an event change for online disk to be automatically
 #    added back to the pool.
-# 7. Creates a raidz1 zpool using persistent disk path names
-#    (ie not /dev/sdc).
-# 8. Tests import using pool guid and cache file.
+#
+# Creates a raidz1 zpool using persistent disk path names
+# (ie not /dev/sdc).
 #
 # If loop devices are used, then a scsi_debug device is added to the pool.
+# otherwise just an sd device is used as the auto-online device.
+# Auto-online matches by devid.
 #
 verify_runnable "both"
 
@@ -53,16 +55,15 @@ fi
 function cleanup
 {
 	#online last disk before fail
-	on_off_disk $offline_disk "online"
+	on_off_disk $offline_disk "online" $host
 	poolexists $TESTPOOL && destroy_pool $TESTPOOL
 }
 
-log_assert "Testing auto-online FMA ZED logic"
+log_assert "Testing automated auto-online FMA test"
 
 log_onexit cleanup
 
-target=$TESTPOOL
-
+# If using the default loop devices, need a scsi_debug device for auto-online
 if is_loop_device $DISK1; then
 	SD=$($LSSCSI | $NAWK '/scsi_debug/ {print $6; exit}')
 	SDDEVICE=$($ECHO $SD | $NAWK -F / '{print $3}')
@@ -79,57 +80,61 @@ do
 done
 
 if is_loop_device $DISK1; then
-	#create a pool with one scsi_debug device and 3 loop devices
+	# create a pool with one scsi_debug device and 3 loop devices
 	log_must $ZPOOL create -f $TESTPOOL raidz1 $SDDEVICE_ID $DISK1 \
 	    $DISK2 $DISK3
 elif ( is_real_device $DISK1 || is_mpath_device $DISK1 ); then
+	# else use the persistent names for sd devices
 	log_must $ZPOOL create -f $TESTPOOL raidz1 ${devs_id[0]} \
 	    ${devs_id[1]} ${devs_id[2]}
 else
 	log_fail "Disks are not supported for this test"
 fi
 
-#add some data to the pool
+# Add some data to the pool
 log_must $MKFILE $FSIZE /$TESTPOOL/data
 
-#pool guid import
-typeset guid=$(get_config $TESTPOOL pool_guid)
-if (( RANDOM % 2 == 0 )) ; then
-	target=$guid
-fi
-
 for offline_disk in $autoonline_disks
 do
 	log_must $ZPOOL export -F $TESTPOOL
 
-	host=$($LS /sys/block/$offline_disk/device/scsi_device | $NAWK -F : '{ print $1}')
-	#offline disk
+	host=$($LS /sys/block/$offline_disk/device/scsi_device \
+	    | $NAWK -F : '{ print $1}')
+
+	# Offline disk
 	on_off_disk $offline_disk "offline"
 
-	#reimport pool with drive missing
-	log_must $ZPOOL import $target
+	# Reimport pool with drive missing
+	log_must $ZPOOL import $TESTPOOL
 	check_state $TESTPOOL "" "degraded"
 	if (($? != 0)); then
 		log_fail "$TESTPOOL is not degraded"
 	fi
 
-	#online disk
+	# Clear zpool events
+	$ZPOOL events -c $TESTPOOL
+
+	# Online disk
 	on_off_disk $offline_disk "online" $host
-	
+
 	log_note "Delay for ZED auto-online"
 	typeset -i timeout=0
-	$CAT ${ZEDLET_DIR}/zedlog | \
-	    $EGREP "zfs_iter_vdev: matched devid" > /dev/null
-	while (($? != 0)); do
+	while true; do
 		if ((timeout == $MAXTIMEOUT)); then
 			log_fail "Timeout occured"
 		fi
 		((timeout++))
 		$SLEEP 1
-		$CAT ${ZEDLET_DIR}/zedlog | \
-		    $EGREP "zfs_iter_vdev: matched devid" > /dev/null
+		$ZPOOL events $TESTPOOL \
+		    | $EGREP sysevent.fs.zfs.resilver_finish > /dev/null
+		if (($? == 0)); then
+			log_note "Auto-online of $offline_disk is complete"
+			$SLEEP 1
+			break
+		fi
 	done
 
+	# Validate auto-online was successful
 	check_state $TESTPOOL "" "online"
 	if (($? != 0)); then
 		log_fail "$TESTPOOL is not back online"
@@ -138,5 +143,4 @@ do
 done
 log_must $ZPOOL destroy $TESTPOOL
 
-
 log_pass "Auto-online test successful"
diff --git a/tests/zfs-tests/tests/functional/fault/auto_replace_001_pos.ksh b/tests/zfs-tests/tests/functional/fault/auto_replace_001_pos.ksh
new file mode 100755
index 000000000..b822b109d
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/fault/auto_replace_001_pos.ksh
@@ -0,0 +1,156 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+#
+# Copyright (c) 2017 by Intel Corporation. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/fault/fault.cfg
+
+#
+# DESCRIPTION:
+# Testing Fault Management Agent ZED Logic - Automated Auto-Replace Test.
+#
+# STRATEGY:
+# 1. Update /etc/zfs/vdev_id.conf with scsidebug alias rule for a persistent
+#    path. This creates keys ID_VDEV and ID_VDEV_PATH and sets
+#    phys_path="scsidebug".
+# 2. Create a pool & set autoreplace=on (auto-replace is opt-in)
+# 2. Export a pool
+# 3. Offline disk by removing scsi_debug module
+# 4. Import pool with missing disk
+# 5. Online disk by loading scsi_debug module again and re-registering vdev_id
+#    rule.
+# 6. ZED polls for an event change for new disk to be automatically
+#    added back to the pool
+#
+# Creates a raidz1 zpool using persistent disk path names
+# (ie not /dev/sdc)
+#
+# Auto-replace is opt in, and matches by phys_path.
+#
+
+verify_runnable "both"
+
+if ! is_physical_device $DISKS; then
+	log_unsupported "Unsupported disks for this test."
+fi
+
+function setup
+{
+	$LSMOD | $EGREP scsi_debug > /dev/null
+	if (($? == 1)); then
+		load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
+	fi
+	# Register vdev_id alias rule for scsi_debug device to create a
+	# persistent path
+	SD=$($LSSCSI | $NAWK '/scsi_debug/ {print $6; exit}' \
+	    | $NAWK -F / '{print $3}')
+	SDDEVICE_ID=$(get_persistent_disk_name $SD)
+	log_must eval "$ECHO "alias scsidebug /dev/disk/by-id/$SDDEVICE_ID" \
+	    >> $VDEVID_CONF"
+	block_device_wait
+
+	SDDEVICE=$($UDEVADM info -q all -n $DEV_DSKDIR/$SD | $EGREP ID_VDEV \
+	    | $NAWK '{print $2; exit}' | $NAWK -F = '{print $2; exit}')
+	[[ -z $SDDEVICE ]] && log_fail "vdev rule was not registered properly"
+}
+
+function cleanup
+{
+	poolexists $TESTPOOL && destroy_pool $TESTPOOL
+}
+
+log_assert "Testing automated auto-replace FMA test"
+
+log_onexit cleanup
+
+# Clear disk labels
+for i in {0..2}
+do
+	log_must $ZPOOL labelclear -f /dev/disk/by-id/"${devs_id[i]}"
+done
+
+setup
+if is_loop_device $DISK1; then
+	log_must $ZPOOL create -f $TESTPOOL raidz1 $SDDEVICE $DISK1 $DISK2 \
+	    $DISK3
+elif ( is_real_device $DISK1 || is_mpath_device $DISK1 ); then
+	log_must $ZPOOL create -f $TESTPOOL raidz1 $SDDEVICE ${devs_id[0]} \
+	    ${devs_id[1]} ${devs_id[2]}
+else
+	log_fail "Disks are not supported for this test"
+fi
+
+# Auto-replace is opt-in so need to set property
+log_must $ZPOOL set autoreplace=on $TESTPOOL
+
+# Add some data to the pool
+log_must $MKFILE $FSIZE /$TESTPOOL/data
+
+log_must $ZPOOL export -F $TESTPOOL
+
+# Offline disk
+on_off_disk $SD "offline"
+block_device_wait
+log_must $MODUNLOAD scsi_debug
+
+# Reimport pool with drive missing
+log_must $ZPOOL import $TESTPOOL
+check_state $TESTPOOL "" "degraded"
+if (($? != 0)); then
+	log_fail "$TESTPOOL is not degraded"
+fi
+
+# Clear zpool events
+$ZPOOL events -c $TESTPOOL
+
+# Create another scsi_debug device
+setup
+
+log_note "Delay for ZED auto-replace"
+typeset -i timeout=0
+while true; do
+	if ((timeout == $MAXTIMEOUT)); then
+		log_fail "Timeout occured"
+	fi
+	((timeout++))
+	$SLEEP 1
+	$ZPOOL events $TESTPOOL | $EGREP sysevent.fs.zfs.resilver_finish \
+	    > /dev/null
+	if (($? == 0)); then
+		log_note "Auto-replace should be complete"
+		$SLEEP 1
+		break
+	fi
+done
+
+# Validate auto-replace was successful
+check_state $TESTPOOL "" "online"
+if (($? != 0)); then
+	log_fail "$TESTPOOL is not back online"
+fi
+$SLEEP 2
+
+log_must $ZPOOL destroy $TESTPOOL
+
+log_pass "Auto-replace test successful"
diff --git a/tests/zfs-tests/tests/functional/fault/cleanup.ksh b/tests/zfs-tests/tests/functional/fault/cleanup.ksh
index 600dae16c..233213c9f 100755
--- a/tests/zfs-tests/tests/functional/fault/cleanup.ksh
+++ b/tests/zfs-tests/tests/functional/fault/cleanup.ksh
@@ -20,7 +20,7 @@
 # CDDL HEADER END
 
 #
-# Copyright (c) 2016 by Intel Corporation. All rights reserved.
+# Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -30,29 +30,24 @@ verify_runnable "global"
 
 cleanup_devices $DISKS
 
-if [[ -f ${ZEDLET_DIR}/zed.pid ]]; then
-	zedpid=$($CAT ${ZEDLET_DIR}/zed.pid)
-	log_must $KILL $zedpid
-fi
+# Remove symlink and vdev_id.conf in-tree file
+$RM -f $VDEVID_CONF_ETC
+$RM -f $VDEVID_CONF
+zed_stop
 
-log_must $RM ${ZEDLET_DIR}/all-syslog.sh
-log_must $RM ${ZEDLET_DIR}/zed.pid
-log_must $RM ${ZEDLET_DIR}/zedlog
-log_must $RM ${ZEDLET_DIR}/state
-log_must $RMDIR $ZEDLET_DIR
+SD=$($LSSCSI | $NAWK '/scsi_debug/ {print $6; exit}')
+SDDEVICE=$($ECHO $SD | $NAWK -F / '{print $3}')
 
-if is_loop_device $DISK1; then
-	SD=$($LSSCSI | $NAWK '/scsi_debug/ {print $6; exit}')
-	SDDEVICE=$($ECHO $SD | $NAWK -F / '{print $3}')
+if [[ -z $SDDEVICE ]]; then
+	log_pass
+fi
 
-	if [[ -z $SDDEVICE ]]; then
-		log_pass
-	fi
-	#offline disk
+# Offline disk and remove scsi_debug module
+if is_linux; then
 	on_off_disk $SDDEVICE "offline"
 	block_device_wait
-
 	log_must $MODUNLOAD scsi_debug
 fi
 
+
 log_pass
diff --git a/tests/zfs-tests/tests/functional/fault/fault.cfg b/tests/zfs-tests/tests/functional/fault/fault.cfg
index a7fe9de93..8552e8702 100644
--- a/tests/zfs-tests/tests/functional/fault/fault.cfg
+++ b/tests/zfs-tests/tests/functional/fault/fault.cfg
@@ -20,7 +20,7 @@
 # CDDL HEADER END
 
 #
-# Copyright (c) 2016 by Intel Corporation. All rights reserved.
+# Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -32,11 +32,17 @@ export DISKSARRAY=$DISKS
 export FSIZE=10M
 export MAXTIMEOUT=20
 
+export SDSIZE=256
+export SDHOSTS=1
+export SDTGTS=1
+export SDLUNS=1
+
 export DISK1=$($ECHO $DISKS | $NAWK '{print $1}')
 export DISK2=$($ECHO $DISKS | $NAWK '{print $2}')
 export DISK3=$($ECHO $DISKS | $NAWK '{print $3}')
 
-export ZEDLET_DIR=/var/tmp/zed
+export VDEVID_CONF=$ZEDLET_DIR/vdev_id.conf
+export VDEVID_CONF_ETC=/etc/zfs/vdev_id.conf
 
 if is_linux; then
 	set_slice_prefix
diff --git a/tests/zfs-tests/tests/functional/fault/setup.ksh b/tests/zfs-tests/tests/functional/fault/setup.ksh
index f2bc1d016..d25cdffe2 100755
--- a/tests/zfs-tests/tests/functional/fault/setup.ksh
+++ b/tests/zfs-tests/tests/functional/fault/setup.ksh
@@ -20,66 +20,37 @@
 # CDDL HEADER END
 
 #
-# Copyright (c) 2016 by Intel Corporation. All rights reserved.
+# Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/fault/fault.cfg
 
-typeset SDSIZE=256
-typeset SDHOSTS=1
-typeset SDTGTS=1
-typeset SDLUNS=1
-
 [[ -z $UDEVADM ]] && log_fail "Missing UDEVADM command"
-[[ -z $NAWK ]]  && log_fail "Missing NAWK command"
-[[ -z $EGREP ]] && log_fail "Missing EGREP command"
+[[ -z $LSMOD ]]  && log_fail "Missing LSMOD command"
 [[ -z $LSSCSI ]] && log_fail "Missing LSSCSI command"
 [[ -z $MODUNLOAD ]] && log_fail "Missing MODUNLOAD command"
-[[ -z $PGREP ]] && log_fail "Missing PGREP command"
+[[ -z $MODLOAD ]] && log_fail "Missing MODLOAD command"
 
 verify_runnable "global"
-if [[ ! -d /var/tmp/zed ]]; then
-	log_must $MKDIR  /var/tmp/zed
-fi
 
-modprobe -n scsi_debug
-if (($? != 0)); then
-	log_unsupported "Platform does not have scsi_debug module"
+if [[ ! -d $ZEDLET_DIR ]]; then
+	log_must $MKDIR $ZEDLET_DIR
 fi
-
-# Verify the ZED is not already running.
-$PGREP -x zed > /dev/null
-if (($? == 0)); then
-	log_fail "ZED already running"
+if [[ ! -e $VDEVID_CONF ]]; then
+	log_must $TOUCH $VDEVID_CONF
+fi
+if [[ -e $VDEVID_CONF_ETC ]]; then
+	log_fail "Must not have $VDEVID_CONF_ETC file present on system"
 fi
 
-log_must $CP ${ZEDLETDIR}/all-syslog.sh $ZEDLET_DIR
+# Create a symlink for /etc/zfs/vdev_id.conf file
+log_must ln -s $VDEVID_CONF $VDEVID_CONF_ETC
 
-log_note "Starting ZED"
-#run ZED in the background and redirect foreground logging output to zedlog
-log_must eval "$ZED -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid -s" \
-    "$ZEDLET_DIR/state 2>${ZEDLET_DIR}/zedlog &"
+zed_start
 
-#if using loop devices, create a scsi_debug device to be used with
-#auto-online test
-if is_loop_device $DISK1; then
-	$LSMOD | $EGREP scsi_debug > /dev/zero
-	if (($? == 0)); then
-		log_fail "SCSI_DEBUG module already installed"
-	else
-		log_must $MODLOAD scsi_debug dev_size_mb=$SDSIZE \
-		    add_host=$SDHOSTS num_tgts=$SDTGTS max_luns=$SDLUNS
-		block_device_wait
-		$LSSCSI | $EGREP scsi_debug > /dev/null
-		if (($? == 1)); then
-			log_fail "scsi_debug failed"
-		else
-			SDDEVICE=$($LSSCSI \
-			    | $NAWK '/scsi_debug/ {print $6; exit}')
-			log_must  $FORMAT -s $SDDEVICE mklabel gpt
-		fi
-	fi
-fi
+# Create a scsi_debug device to be used with auto-online (if using loop devices)
+# and auto-replace regardless of other devices
+load_scsi_debug $SDSIZE $SDHOSTS $SDTGTS $SDLUNS
 
 log_pass