Add device rebuild feature

The device_rebuild feature enables sequential reconstruction when
resilvering.  Mirror vdevs can be rebuilt in LBA order which may
more quickly restore redundancy depending on the pools average block
size, overall fragmentation and the performance characteristics
of the devices.  However, block checksums cannot be verified
as part of the rebuild thus a scrub is automatically started after
the sequential resilver completes.

The new '-s' option has been added to the `zpool attach` and
`zpool replace` command to request sequential reconstruction
instead of healing reconstruction when resilvering.

    zpool attach -s <pool> <existing vdev> <new vdev>
    zpool replace -s <pool> <old vdev> <new vdev>

The `zpool status` output has been updated to report the progress
of sequential resilvering in the same way as healing resilvering.
The one notable difference is that multiple sequential resilvers
may be in progress as long as they're operating on different
top-level vdevs.

The `zpool wait -t resilver` command was extended to wait on
sequential resilvers.  From this perspective they are no different
than healing resilvers.

Sequential resilvers cannot be supported for RAIDZ, but are
compatible with the dRAID feature being developed.

As part of this change the resilver_restart_* tests were moved
in to the functional/replacement directory.  Additionally, the
replacement tests were renamed and extended to verify both
resilvering and rebuilding.

Original-patch-by: Isaac Huang <[email protected]>
Reviewed-by: Tony Hutter <[email protected]>
Reviewed-by: John Poduska <[email protected]>
Co-authored-by: Mark Maybee <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #10349 

26 files changed, 1112 insertions, 178 deletions

diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run
index 765ffea8a..f6478dd0d 100644
--- a/tests/runfiles/common.run
+++ b/tests/runfiles/common.run
@@ -487,7 +487,8 @@ tests = ['zpool_wait_discard', 'zpool_wait_freeing',
 tags = ['functional', 'cli_root', 'zpool_wait']
 
 [tests/functional/cli_root/zpool_wait/scan]
-tests = ['zpool_wait_replace_cancel', 'zpool_wait_resilver', 'zpool_wait_scrub_cancel',
+tests = ['zpool_wait_replace_cancel', 'zpool_wait_rebuild',
+    'zpool_wait_resilver', 'zpool_wait_scrub_cancel',
     'zpool_wait_replace', 'zpool_wait_scrub_basic', 'zpool_wait_scrub_flag']
 tags = ['functional', 'cli_root', 'zpool_wait']
 
@@ -748,7 +749,11 @@ tests = ['rename_dirs_001_pos']
 tags = ['functional', 'rename_dirs']
 
 [tests/functional/replacement]
-tests = ['replacement_001_pos', 'replacement_002_pos', 'replacement_003_pos']
+tests = ['attach_import', 'attach_multiple', 'attach_rebuild',
+    'attach_resilver', 'detach', 'rebuild_disabled_feature',
+    'rebuild_multiple', 'rebuild_raidz', 'replace_import', 'replace_rebuild',
+    'replace_resilver', 'resilver_restart_001', 'resilver_restart_002',
+    'scrub_cancel']
 tags = ['functional', 'replacement']
 
 [tests/functional/reservation]
@@ -762,10 +767,6 @@ tests = ['reservation_001_pos', 'reservation_002_pos', 'reservation_003_pos',
     'reservation_022_pos']
 tags = ['functional', 'reservation']
 
-[tests/functional/resilver]
-tests = ['resilver_restart_001', 'resilver_restart_002']
-tags = ['functional', 'resilver']
-
 [tests/functional/rootpool]
 tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
 tags = ['functional', 'rootpool']
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index 9fbcc37c6..5e07cda4d 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -2222,26 +2222,27 @@ function check_pool_status # pool token keyword <verbose>
 	if [[ $verbose == true ]]; then
 		log_note $scan
 	fi
-	echo $scan | grep -i "$keyword" > /dev/null 2>&1
+	echo $scan | egrep -i "$keyword" > /dev/null 2>&1
 
 	return $?
 }
 
 #
 # The following functions are instance of check_pool_status()
-#	is_pool_resilvering - to check if the pool is resilver in progress
-#	is_pool_resilvered - to check if the pool is resilver completed
-#	is_pool_scrubbing - to check if the pool is scrub in progress
-#	is_pool_scrubbed - to check if the pool is scrub completed
-#	is_pool_scrub_stopped - to check if the pool is scrub stopped
-#	is_pool_scrub_paused - to check if the pool has scrub paused
-#	is_pool_removing - to check if the pool is removing a vdev
-#	is_pool_removed - to check if the pool is remove completed
-#	is_pool_discarding - to check if the pool has checkpoint being discarded
+#	is_pool_resilvering - to check if the pool resilver is in progress
+#	is_pool_resilvered - to check if the pool resilver is completed
+#	is_pool_scrubbing - to check if the pool scrub is in progress
+#	is_pool_scrubbed - to check if the pool scrub is completed
+#	is_pool_scrub_stopped - to check if the pool scrub is stopped
+#	is_pool_scrub_paused - to check if the pool scrub has paused
+#	is_pool_removing - to check if the pool removing is a vdev
+#	is_pool_removed - to check if the pool remove is completed
+#	is_pool_discarding - to check if the pool checkpoint is being discarded
 #
 function is_pool_resilvering #pool <verbose>
 {
-	check_pool_status "$1" "scan" "resilver in progress since " $2
+	check_pool_status "$1" "scan" \
+	    "resilver[ ()0-9A-Za-z_-]* in progress since" $2
 	return $?
 }
 
@@ -3487,7 +3488,7 @@ function wait_scrubbed
 	typeset pool=${1:-$TESTPOOL}
 	while true ; do
 		is_pool_scrubbed $pool && break
-		log_must sleep 1
+		sleep 1
 	done
 }
 
diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am
index 24f3e50bb..c56518c55 100644
--- a/tests/zfs-tests/tests/functional/Makefile.am
+++ b/tests/zfs-tests/tests/functional/Makefile.am
@@ -65,7 +65,6 @@ SUBDIRS = \
 	rename_dirs \
 	replacement \
 	reservation \
-	resilver \
 	rootpool \
 	rsend \
 	scrub_mirror \
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg
index ee5b2b4e1..4991b76bf 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg
@@ -79,6 +79,7 @@ typeset -a properties=(
     "feature@redacted_datasets"
     "feature@bookmark_written"
     "feature@log_spacemap"
+    "feature@device_rebuild"
 )
 
 if is_linux || is_freebsd; then
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/Makefile.am
index 6a21cac4f..451d83a79 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/Makefile.am
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/Makefile.am
@@ -4,6 +4,7 @@ dist_pkgdata_SCRIPTS = \
 	cleanup.ksh \
 	zpool_wait_replace.ksh  \
 	zpool_wait_replace_cancel.ksh \
+	zpool_wait_rebuild.ksh \
 	zpool_wait_resilver.ksh  \
 	zpool_wait_scrub_basic.ksh \
 	zpool_wait_scrub_cancel.ksh \
diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/zpool_wait_rebuild.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/zpool_wait_rebuild.ksh
new file mode 100755
index 000000000..8cd586459
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/zpool_wait_rebuild.ksh
@@ -0,0 +1,64 @@
+#!/bin/ksh -p
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2018 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_wait/zpool_wait.kshlib
+
+#
+# DESCRIPTION:
+# 'zpool wait' works when waiting for sequential resilvering to complete.
+#
+# STRATEGY:
+# 1. Attach a device to the pool so that sequential resilvering starts.
+# 2. Start 'zpool wait'.
+# 3. Monitor the waiting process to make sure it returns neither too soon nor
+#    too late.
+# 4. Repeat 1-3, except using the '-w' flag with 'zpool attach' instead of using
+#    'zpool wait'.
+#
+
+function cleanup
+{
+	remove_io_delay
+	kill_if_running $pid
+	get_disklist $TESTPOOL | grep $DISK2 >/dev/null && \
+	    log_must zpool detach $TESTPOOL $DISK2
+}
+
+typeset -r IN_PROGRESS_CHECK="is_pool_resilvering $TESTPOOL"
+typeset pid
+
+log_onexit cleanup
+
+add_io_delay $TESTPOOL
+
+# Test 'zpool wait -t resilver'
+log_must zpool attach -s $TESTPOOL $DISK1 $DISK2
+log_bkgrnd zpool wait -t resilver $TESTPOOL
+pid=$!
+check_while_waiting $pid "$IN_PROGRESS_CHECK"
+
+log_must zpool detach $TESTPOOL $DISK2
+
+# Test 'zpool attach -w'
+log_bkgrnd zpool attach -sw $TESTPOOL $DISK1 $DISK2
+pid=$!
+while ! is_pool_resilvering $TESTPOOL && proc_exists $pid; do
+	log_must sleep .5
+done
+check_while_waiting $pid "$IN_PROGRESS_CHECK"
+
+log_pass "'zpool wait -t resilver' and 'zpool attach -w' work."
diff --git a/tests/zfs-tests/tests/functional/replacement/Makefile.am b/tests/zfs-tests/tests/functional/replacement/Makefile.am
index d47fcd5e1..fe6e49121 100644
--- a/tests/zfs-tests/tests/functional/replacement/Makefile.am
+++ b/tests/zfs-tests/tests/functional/replacement/Makefile.am
@@ -2,9 +2,20 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/replacement
 dist_pkgdata_SCRIPTS = \
 	setup.ksh \
 	cleanup.ksh \
-	replacement_001_pos.ksh \
-	replacement_002_pos.ksh \
-	replacement_003_pos.ksh
+	attach_import.ksh \
+	attach_multiple.ksh \
+	attach_rebuild.ksh \
+	attach_resilver.ksh \
+	detach.ksh \
+	rebuild_disabled_feature.ksh \
+	rebuild_multiple.ksh \
+	rebuild_raidz.ksh \
+	replace_import.ksh \
+	replace_rebuild.ksh \
+	replace_resilver.ksh \
+	resilver_restart_001.ksh \
+	resilver_restart_002.ksh \
+	scrub_cancel.ksh
 
 dist_pkgdata_DATA = \
 	replacement.cfg
diff --git a/tests/zfs-tests/tests/functional/replacement/attach_import.ksh b/tests/zfs-tests/tests/functional/replacement/attach_import.ksh
new file mode 100755
index 000000000..e2749b164
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/replacement/attach_import.ksh
@@ -0,0 +1,67 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+# Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# Description:
+# Verify that on import an in progress attach operation is resumed.
+#
+# Strategy:
+# 1. For both healing and sequential resilvering.
+#    a. Create a pool
+#    b. Add a vdev with 'zpool attach' and resilver (-s) it.
+#    c. Export the pool
+#    d. Import the pool
+#    e. Verify the 'zpool attach' resumed resilvering
+#    f. Destroy the pool
+#
+
+function cleanup
+{
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS \
+	    $ORIG_SCAN_SUSPEND_PROGRESS
+	destroy_pool $TESTPOOL1
+	rm -f ${VDEV_FILES[@]}
+}
+
+log_assert "Verify attach is resumed on import"
+
+ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS)
+
+log_onexit cleanup
+
+log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]}
+
+# Verify healing and sequential resilver resume on import.
+for arg in "" "-s"; do
+	log_must zpool create -f $TESTPOOL1 ${VDEV_FILES[0]}
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1
+	log_must zpool attach $arg $TESTPOOL1 ${VDEV_FILES[0]} ${VDEV_FILES[1]}
+	log_must is_pool_resilvering $TESTPOOL1
+	log_must zpool export $TESTPOOL1
+	log_must zpool import -d $TEST_BASE_DIR $TESTPOOL1
+	log_must is_pool_resilvering $TESTPOOL1
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS
+	log_must zpool wait -t resilver $TESTPOOL1
+	log_must is_pool_resilvered $TESTPOOL1
+	destroy_pool $TESTPOOL1
+done
+
+log_pass "Verify attach is resumed on import"
diff --git a/tests/zfs-tests/tests/functional/replacement/attach_multiple.ksh b/tests/zfs-tests/tests/functional/replacement/attach_multiple.ksh
new file mode 100755
index 000000000..b3192b2bf
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/replacement/attach_multiple.ksh
@@ -0,0 +1,111 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+# Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# Description:
+# Verify that attach/detach work while resilvering and attaching
+# multiple vdevs.
+#
+# Strategy:
+# 1. Create a single vdev pool
+# 2. While healing or sequential resilvering:
+#    a. Attach a vdev to convert the pool to a mirror.
+#    b. Attach a vdev to convert the pool to a 3-way mirror.
+#    c. Verify the original vdev cannot be removed (no redundant copies)
+#    d. Detach a vdev.  Healing and sequential resilver remain running.
+#    e. Detach a vdev.  Healing resilver remains running, sequential
+#       resilver is canceled.
+#    f. Wait for resilver to complete.
+#
+
+function cleanup
+{
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS \
+	    $ORIG_SCAN_SUSPEND_PROGRESS
+	destroy_pool $TESTPOOL1
+	rm -f ${VDEV_FILES[@]}
+}
+
+log_assert "Verify attach/detech with multiple vdevs"
+
+ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS)
+
+log_onexit cleanup
+
+log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]}
+
+# Verify resilver resumes on import.
+log_must zpool create -f $TESTPOOL1 ${VDEV_FILES[0]}
+
+for replace_mode in "healing" "sequential"; do
+        #
+        # Resilvers abort the dsl_scan and reconfigure it for resilvering.
+        # Rebuilds cancel the dsl_scan and start the vdev_rebuild thread.
+        #
+        if [[ "$replace_mode" = "healing" ]]; then
+                flags=""
+        else
+                flags="-s"
+        fi
+
+	log_mustnot is_pool_resilvering $TESTPOOL1
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1
+
+	# Attach first vdev (stripe -> mirror)
+	log_must zpool attach $flags $TESTPOOL1 \
+	    ${VDEV_FILES[0]} ${VDEV_FILES[1]}
+	log_must is_pool_resilvering $TESTPOOL1
+
+	# Attach second vdev (2-way -> 3-way mirror)
+	log_must zpool attach $flags $TESTPOOL1 \
+	    ${VDEV_FILES[1]} ${VDEV_FILES[2]}
+	log_must is_pool_resilvering $TESTPOOL1
+
+	# Original vdev cannot be detached until there is sufficent redundancy.
+	log_mustnot zpool detach $TESTPOOL1 ${VDEV_FILES[0]}
+
+	# Detach first vdev (resilver keeps running)
+	log_must zpool detach $TESTPOOL1 ${VDEV_FILES[1]}
+	log_must is_pool_resilvering $TESTPOOL1
+
+	#
+	# Detach second vdev.  There's a difference in behavior between
+	# healing and sequential resilvers.  A healing resilver will not be
+	# cancelled even though there's nothing on the original vdev which
+	# needs to be rebuilt.  A sequential resilver on the otherhand is
+	# canceled when returning to a non-redundant striped layout.  At
+	# some point the healing resilver behavior should be updated to match
+	# the sequential resilver behavior.
+	#
+	log_must zpool detach $TESTPOOL1 ${VDEV_FILES[2]}
+
+        if [[ "$replace_mode" = "healing" ]]; then
+		log_must is_pool_resilvering $TESTPOOL1
+        else
+		log_mustnot is_pool_resilvering $TESTPOOL1
+        fi
+
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS \
+	    $ORIG_SCAN_SUSPEND_PROGRESS
+	log_must zpool wait $TESTPOOL1
+done
+
+log_pass "Verify attach/detech with multiple vdevs"
diff --git a/tests/zfs-tests/tests/functional/replacement/attach_rebuild.ksh b/tests/zfs-tests/tests/functional/replacement/attach_rebuild.ksh
new file mode 100755
index 000000000..e9427c7ad
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/replacement/attach_rebuild.ksh
@@ -0,0 +1,173 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# DESCRIPTION:
+# 	Attaching disks during I/O should pass for supported pools.
+#
+# STRATEGY:
+#	1. Create multidisk pools (stripe/mirror/raidz) and
+#	   start some random I/O
+#	2. Attach a disk to the pool.
+#	3. Verify the integrity of the file system and the resilvering.
+#
+# NOTE: Raidz does not support the sequential resilver (-s) option.
+#
+
+verify_runnable "global"
+
+function cleanup
+{
+	if [[ -n "$child_pids" ]]; then
+		for wait_pid in $child_pids; do
+			kill $wait_pid
+		done
+	fi
+
+	if poolexists $TESTPOOL1; then
+		destroy_pool $TESTPOOL1
+	fi
+
+	[[ -e $TESTDIR ]] && log_must rm -rf $TESTDIR/*
+}
+
+log_assert "Replacing a disk during I/O completes."
+
+options=""
+options_display="default options"
+
+log_onexit cleanup
+
+[[ -n "$HOLES_FILESIZE" ]] && options=" $options -f $HOLES_FILESIZE "
+
+[[ -n "$HOLES_BLKSIZE" ]] && options="$options -b $HOLES_BLKSIZE "
+
+[[ -n "$HOLES_COUNT" ]] && options="$options -c $HOLES_COUNT "
+
+[[ -n "$HOLES_SEED" ]] && options="$options -s $HOLES_SEED "
+
+[[ -n "$HOLES_FILEOFFSET" ]] && options="$options -o $HOLES_FILEOFFSET "
+
+options="$options -r "
+
+[[ -n "$options" ]] && options_display=$options
+
+child_pids=""
+
+function attach_test
+{
+	typeset -i iters=2
+	typeset -i index=0
+	typeset opt=$1
+	typeset disk1=$2
+	typeset disk2=$3
+
+	typeset i=0
+	while [[ $i -lt $iters ]]; do
+		log_note "Invoking file_trunc with: $options_display"
+		file_trunc $options $TESTDIR/$TESTFILE.$i &
+		typeset pid=$!
+
+		sleep 1
+
+		child_pids="$child_pids $pid"
+		((i = i + 1))
+	done
+
+	log_must zpool attach -sw $opt $TESTPOOL1 $disk1 $disk2
+
+	for wait_pid in $child_pids; do
+		kill $wait_pid
+	done
+	child_pids=""
+
+	log_must zpool export $TESTPOOL1
+	log_must zpool import -d $TESTDIR $TESTPOOL1
+	log_must zfs umount $TESTPOOL1/$TESTFS1
+	log_must zdb -cdui $TESTPOOL1/$TESTFS1
+	log_must zfs mount $TESTPOOL1/$TESTFS1
+	verify_pool $TESTPOOL1
+}
+
+specials_list=""
+i=0
+while [[ $i != 3 ]]; do
+	truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i
+	specials_list="$specials_list $TESTDIR/$TESTFILE1.$i"
+
+	((i = i + 1))
+done
+
+#
+# Create a replacement disk special file.
+#
+truncate -s $MINVDEVSIZE $TESTDIR/$REPLACEFILE
+
+for op in "" "-f"; do
+	create_pool $TESTPOOL1 mirror $specials_list
+	log_must zfs create $TESTPOOL1/$TESTFS1
+	log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1
+
+	attach_test "$opt" $TESTDIR/$TESTFILE1.1 $TESTDIR/$REPLACEFILE
+
+	zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE"
+	if [[ $? -ne 0 ]]; then
+		log_fail "$REPLACEFILE is not present."
+	fi
+
+	destroy_pool $TESTPOOL1
+done
+
+log_note "Verify 'zpool attach' fails with non-mirrors."
+
+for type in "" "raidz" "raidz1"; do
+	for op in "" "-f"; do
+		create_pool $TESTPOOL1 $type $specials_list
+		log_must zfs create $TESTPOOL1/$TESTFS1
+		log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1
+
+		log_mustnot zpool attach -s "$opt" $TESTDIR/$TESTFILE1.1 \
+		    $TESTDIR/$REPLACEFILE
+
+		zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE"
+		if [[ $? -eq 0 ]]; then
+			log_fail "$REPLACEFILE should not be present."
+		fi
+
+		destroy_pool $TESTPOOL1
+	done
+done
+
+log_pass
diff --git a/tests/zfs-tests/tests/functional/replacement/replacement_002_pos.ksh b/tests/zfs-tests/tests/functional/replacement/attach_resilver.ksh
index 391aa5cf0..4261d4d67 100755
--- a/tests/zfs-tests/tests/functional/replacement/replacement_002_pos.ksh
+++ b/tests/zfs-tests/tests/functional/replacement/attach_resilver.ksh
@@ -104,9 +104,7 @@ function attach_test
 		((i = i + 1))
 	done
 
-	log_must zpool attach $opt $TESTPOOL1 $disk1 $disk2
-
-	sleep 10
+	log_must zpool attach -w $opt $TESTPOOL1 $disk1 $disk2
 
 	for wait_pid in $child_pids
 	do
@@ -119,13 +117,13 @@ function attach_test
         log_must zfs umount $TESTPOOL1/$TESTFS1
         log_must zdb -cdui $TESTPOOL1/$TESTFS1
         log_must zfs mount $TESTPOOL1/$TESTFS1
-
+	verify_pool $TESTPOOL1
 }
 
 specials_list=""
 i=0
-while [[ $i != 2 ]]; do
-	mkfile $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i
+while [[ $i != 3 ]]; do
+	truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i
 	specials_list="$specials_list $TESTDIR/$TESTFILE1.$i"
 
 	((i = i + 1))
@@ -134,7 +132,7 @@ done
 #
 # Create a replacement disk special file.
 #
-mkfile $MINVDEVSIZE $TESTDIR/$REPLACEFILE
+truncate -s $MINVDEVSIZE $TESTDIR/$REPLACEFILE
 
 for op in "" "-f"; do
 	create_pool $TESTPOOL1 mirror $specials_list
@@ -143,7 +141,7 @@ for op in "" "-f"; do
 
 	attach_test "$opt" $TESTDIR/$TESTFILE1.1 $TESTDIR/$REPLACEFILE
 
-	zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$REPLACEFILE"
+	zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE"
 	if [[ $? -ne 0 ]]; then
 		log_fail "$REPLACEFILE is not present."
 	fi
@@ -162,7 +160,7 @@ for type in "" "raidz" "raidz1"; do
 		log_mustnot zpool attach "$opt" $TESTDIR/$TESTFILE1.1 \
 		    $TESTDIR/$REPLACEFILE
 
-		zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$REPLACEFILE"
+		zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE"
 		if [[ $? -eq 0 ]]; then
 		        log_fail "$REPLACEFILE should not be present."
 		fi
diff --git a/tests/zfs-tests/tests/functional/replacement/replacement_003_pos.ksh b/tests/zfs-tests/tests/functional/replacement/detach.ksh
index 71b9602ee..aa3ec4f7a 100755
--- a/tests/zfs-tests/tests/functional/replacement/replacement_003_pos.ksh
+++ b/tests/zfs-tests/tests/functional/replacement/detach.ksh
@@ -121,8 +121,8 @@ function detach_test
 
 specials_list=""
 i=0
-while [[ $i != 2 ]]; do
-	mkfile $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i
+while [[ $i != 3 ]]; do
+	truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i
 	specials_list="$specials_list $TESTDIR/$TESTFILE1.$i"
 
 	((i = i + 1))
@@ -134,7 +134,7 @@ log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1
 
 detach_test $TESTDIR/$TESTFILE1.1
 
-zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$TESTFILE1.1"
+zpool iostat -v $TESTPOOL1 | grep "$TESTFILE1.1"
 if [[ $? -eq 0 ]]; then
 	log_fail "$TESTFILE1.1 should no longer be present."
 fi
@@ -143,14 +143,14 @@ destroy_pool $TESTPOOL1
 
 log_note "Verify 'zpool detach' fails with non-mirrors."
 
-for type in "" "raidz" "raidz1" ; do
+for type in "" "raidz" "raidz1"; do
 	create_pool $TESTPOOL1 $type $specials_list
 	log_must zfs create $TESTPOOL1/$TESTFS1
 	log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1
 
 	log_mustnot zpool detach $TESTDIR/$TESTFILE1.1
 
-	zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$TESTFILE1.1"
+	zpool iostat -v $TESTPOOL1 | grep "$TESTFILE1.1"
 	if [[ $? -ne 0 ]]; then
 	        log_fail "$TESTFILE1.1 is not present."
 	fi
diff --git a/tests/zfs-tests/tests/functional/replacement/rebuild_disabled_feature.ksh b/tests/zfs-tests/tests/functional/replacement/rebuild_disabled_feature.ksh
new file mode 100755
index 000000000..d17d83b78
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/replacement/rebuild_disabled_feature.ksh
@@ -0,0 +1,78 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+# Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# Description:
+# Verify device_rebuild feature flags.
+#
+# Strategy:
+# 1. Create a pool with all features disabled.
+# 2. Verify 'zpool replace -s' fails and the feature is disabled.
+# 3. Enable the device_rebuild feature.
+# 4. Verify 'zpool replace -s' works and the feature is active.
+# 5. Wait for the feature to return to enabled.
+#
+
+function cleanup
+{
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS \
+	    $ORIG_SCAN_SUSPEND_PROGRESS
+	destroy_pool $TESTPOOL1
+	rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+}
+
+function check_feature_flag
+{
+	feature=$1
+	pool=$2
+	expected_value=$3
+
+	value="$(zpool get -H -o property,value all $pool | \
+	    egrep "$feature" | awk '{print $2}')"
+	if [ "$value" = "$expected_value" ]; then
+		log_note "$feature verified to be $value"
+	else
+		log_fail "$feature should be $expected_value but is $value"
+	fi
+}
+
+log_assert "Verify device_rebuild feature flags."
+
+ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS)
+
+log_onexit cleanup
+
+log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+log_must zpool create -d $TESTPOOL1 ${VDEV_FILES[@]}
+
+log_mustnot zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE
+check_feature_flag "feature@device_rebuild" "$TESTPOOL1" "disabled"
+
+log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1
+log_must zpool set feature@device_rebuild=enabled $TESTPOOL1
+log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE
+check_feature_flag "feature@device_rebuild" "$TESTPOOL1" "active"
+
+log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS
+log_must zpool wait -t resilver $TESTPOOL1
+check_feature_flag "feature@device_rebuild" "$TESTPOOL1" "enabled"
+
+log_pass "Verify device_rebuild feature flags."
diff --git a/tests/zfs-tests/tests/functional/replacement/rebuild_multiple.ksh b/tests/zfs-tests/tests/functional/replacement/rebuild_multiple.ksh
new file mode 100755
index 000000000..7775cbff4
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/replacement/rebuild_multiple.ksh
@@ -0,0 +1,126 @@
+#!/bin/ksh -p
+
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+# Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# DESCRIPTION:
+# Sequential reconstruction (unlike healing reconstruction) operate on the
+# top-level vdev.  This means that a sequential resilver operation can be
+# started/stopped on a different top-level vdev without impacting other
+# sequential resilvers.
+#
+# STRATEGY:
+# 1. Create a mirrored pool.
+#
+
+function cleanup
+{
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS \
+	    $ORIG_SCAN_SUSPEND_PROGRESS
+	destroy_pool $TESTPOOL1
+	rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE $SPARE_VDEV_FILE2
+}
+
+function check_history
+{
+	pool=$1
+	msg=$2
+	exp=$3
+
+	count=$(zpool history -i $pool | grep "rebuild" | grep -c "$msg")
+	if [[ "$count" -ne "$exp" ]]; then
+		log_fail "Expected $exp rebuild '$msg' messages, found $count"
+	else
+		log_note "Found $count/$exp rebuild '$msg' messages"
+	fi
+}
+
+log_assert "Rebuilds operate on the top-level vdevs"
+
+ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS)
+
+log_onexit cleanup
+
+log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} \
+    $SPARE_VDEV_FILE $SPARE_VDEV_FILE2
+
+# Verify two sequential resilvers can run concurrently.
+log_must zpool create -f $TESTPOOL1 \
+    mirror ${VDEV_FILES[0]} ${VDEV_FILES[1]} \
+    mirror ${VDEV_FILES[2]} ${VDEV_FILES[3]}
+log_must zfs create $TESTPOOL1/$TESTFS
+
+mntpnt=$(get_prop mountpoint $TESTPOOL1/$TESTFS)
+log_must dd if=/dev/urandom of=$mntpnt/file bs=1M count=32
+log_must zpool sync $TESTPOOL1
+
+log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1
+
+log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE
+log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[3]} $SPARE_VDEV_FILE2
+
+check_history $TESTPOOL1 "started" 2
+check_history $TESTPOOL1 "reset" 0
+check_history $TESTPOOL1 "complete" 0
+check_history $TESTPOOL1 "canceled" 0
+
+log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS
+log_must zpool wait -t resilver $TESTPOOL1
+
+check_history $TESTPOOL1 "complete" 2
+destroy_pool $TESTPOOL1
+
+# Verify canceling one resilver (zpool detach) does not impact others.
+log_must zpool create -f $TESTPOOL1 \
+    mirror ${VDEV_FILES[0]} ${VDEV_FILES[1]} \
+    mirror ${VDEV_FILES[2]} ${VDEV_FILES[3]}
+log_must zfs create $TESTPOOL1/$TESTFS
+
+mntpnt=$(get_prop mountpoint $TESTPOOL1/$TESTFS)
+log_must dd if=/dev/urandom of=$mntpnt/file bs=1M count=32
+log_must zpool sync $TESTPOOL1
+
+log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1
+
+log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE
+log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[3]} $SPARE_VDEV_FILE2
+
+check_history $TESTPOOL1 "started" 2
+check_history $TESTPOOL1 "reset" 0
+check_history $TESTPOOL1 "complete" 0
+check_history $TESTPOOL1 "canceled" 0
+
+log_must zpool detach $TESTPOOL1 $SPARE_VDEV_FILE2
+
+check_history $TESTPOOL1 "complete" 0
+check_history $TESTPOOL1 "canceled" 1
+
+log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS
+log_must zpool wait -t resilver $TESTPOOL1
+
+check_history $TESTPOOL1 "complete" 1
+check_history $TESTPOOL1 "canceled" 1
+destroy_pool $TESTPOOL1
+
+log_pass "Rebuilds operate on the top-level vdevs"
diff --git a/tests/zfs-tests/tests/functional/replacement/rebuild_raidz.ksh b/tests/zfs-tests/tests/functional/replacement/rebuild_raidz.ksh
new file mode 100755
index 000000000..c919b44b2
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/replacement/rebuild_raidz.ksh
@@ -0,0 +1,70 @@
+#!/bin/ksh -p
+
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+# Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# DESCRIPTION:
+# Executing 'zpool replace -s' for raidz vdevs failed.  Sequential
+# resilvers are only allowed for stripe/mirror pools.
+#
+# STRATEGY:
+# 1. Create a raidz pool, verify 'zpool replace -s' fails
+# 2. Create a stripe/mirror pool, verify 'zpool replace -s' passes
+#
+
+function cleanup
+{
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS \
+	    $ORIG_SCAN_SUSPEND_PROGRESS
+	destroy_pool $TESTPOOL1
+	rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+}
+
+log_assert "Sequential resilver is not allowed for raidz vdevs"
+
+ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS)
+
+log_onexit cleanup
+
+log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+
+# raidz[1-3]
+for vdev_type in "raidz" "raidz2" "raidz3"; do
+	log_must zpool create -f $TESTPOOL1 $vdev_type ${VDEV_FILES[@]}
+	log_mustnot zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} \
+	    $SPARE_VDEV_FILE
+	destroy_pool $TESTPOOL1
+done
+
+# stripe
+log_must zpool create $TESTPOOL1 ${VDEV_FILES[@]}
+log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE
+destroy_pool $TESTPOOL1
+
+# mirror
+log_must zpool create $TESTPOOL1 mirror ${VDEV_FILES[0]} ${VDEV_FILES[1]}
+log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]}  $SPARE_VDEV_FILE
+destroy_pool $TESTPOOL1
+
+log_pass "Sequential resilver is not allowed for raidz vdevs"
diff --git a/tests/zfs-tests/tests/functional/replacement/replace_import.ksh b/tests/zfs-tests/tests/functional/replacement/replace_import.ksh
new file mode 100755
index 000000000..35d51d939
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/replacement/replace_import.ksh
@@ -0,0 +1,67 @@
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+# Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# Description:
+# Verify that on import an in progress replace operation is resumed.
+#
+# Strategy:
+# 1. For both healing and sequential resilvering replace:
+#    a. Create a pool
+#    b. Repalce a vdev with 'zpool replace' to resilver (-s) it.
+#    c. Export the pool
+#    d. Import the pool
+#    e. Verify the 'zpool replace' resumed resilvering.
+#    f. Destroy the pool
+#
+
+function cleanup
+{
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS \
+	    $ORIG_SCAN_SUSPEND_PROGRESS
+	destroy_pool $TESTPOOL1
+	rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+}
+
+log_assert "Verify replace is resumed on import"
+
+ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS)
+
+log_onexit cleanup
+
+log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+
+# Verify healing and sequential resilver resume on import.
+for arg in "" "-s"; do
+	log_must zpool create -f $TESTPOOL1 ${VDEV_FILES[@]}
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1
+	log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[0]} $SPARE_VDEV_FILE
+	log_must is_pool_resilvering $TESTPOOL1
+	log_must zpool export $TESTPOOL1
+	log_must zpool import -d $TEST_BASE_DIR $TESTPOOL1
+	log_must is_pool_resilvering $TESTPOOL1
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS
+	log_must zpool wait -t resilver $TESTPOOL1
+	log_must is_pool_resilvered $TESTPOOL1
+	destroy_pool $TESTPOOL1
+done
+
+log_pass "Verify replace is resumed on import"
diff --git a/tests/zfs-tests/tests/functional/replacement/replace_rebuild.ksh b/tests/zfs-tests/tests/functional/replacement/replace_rebuild.ksh
new file mode 100755
index 000000000..599735228
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/replacement/replace_rebuild.ksh
@@ -0,0 +1,158 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# DESCRIPTION:
+# 	Replacing disks during I/O should pass for supported pools.
+#
+# STRATEGY:
+#	1. Create multidisk pools (stripe/mirror) and
+#	   start some random I/O
+#	2. Replace a disk in the pool with another disk.
+#	3. Verify the integrity of the file system and the rebuilding.
+#
+# NOTE: Raidz does not support the sequential resilver (-s) option.
+#
+
+verify_runnable "global"
+
+function cleanup
+{
+	if [[ -n "$child_pids" ]]; then
+		for wait_pid in $child_pids
+		do
+			kill $wait_pid
+		done
+	fi
+
+	if poolexists $TESTPOOL1; then
+		destroy_pool $TESTPOOL1
+	fi
+
+	[[ -e $TESTDIR ]] && log_must rm -rf $TESTDIR/*
+}
+
+log_assert "Replacing a disk with -r during I/O completes."
+
+options=""
+options_display="default options"
+
+log_onexit cleanup
+
+[[ -n "$HOLES_FILESIZE" ]] && options=" $options -f $HOLES_FILESIZE "
+
+[[ -n "$HOLES_BLKSIZE" ]] && options="$options -b $HOLES_BLKSIZE "
+
+[[ -n "$HOLES_COUNT" ]] && options="$options -c $HOLES_COUNT "
+
+[[ -n "$HOLES_SEED" ]] && options="$options -s $HOLES_SEED "
+
+[[ -n "$HOLES_FILEOFFSET" ]] && options="$options -o $HOLES_FILEOFFSET "
+
+options="$options -r "
+
+[[ -n "$options" ]] && options_display=$options
+
+child_pids=""
+
+function replace_test
+{
+	typeset -i iters=2
+	typeset -i index=0
+	typeset opt=$1
+	typeset disk1=$2
+	typeset disk2=$3
+
+	typeset i=0
+	while [[ $i -lt $iters ]]; do
+		log_note "Invoking file_trunc with: $options_display"
+		file_trunc $options $TESTDIR/$TESTFILE.$i &
+		typeset pid=$!
+
+		sleep 1
+
+		child_pids="$child_pids $pid"
+		((i = i + 1))
+	done
+
+	log_must zpool replace -sw $opt $TESTPOOL1 $disk1 $disk2
+
+	for wait_pid in $child_pids
+	do
+		kill $wait_pid
+	done
+	child_pids=""
+
+	log_must zpool export $TESTPOOL1
+	log_must zpool import -d $TESTDIR $TESTPOOL1
+	log_must zfs umount $TESTPOOL1/$TESTFS1
+	log_must zdb -cdui $TESTPOOL1/$TESTFS1
+	log_must zfs mount $TESTPOOL1/$TESTFS1
+	verify_pool $TESTPOOL1
+}
+
+specials_list=""
+i=0
+while [[ $i != 3 ]]; do
+	log_must truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i
+	specials_list="$specials_list $TESTDIR/$TESTFILE1.$i"
+
+	((i = i + 1))
+done
+
+#
+# Create a replacement disk special file.
+#
+log_must truncate -s $MINVDEVSIZE $TESTDIR/$REPLACEFILE
+
+for type in "" "mirror"; do
+	for op in "" "-f"; do
+		create_pool $TESTPOOL1 $type $specials_list
+		log_must zfs create $TESTPOOL1/$TESTFS1
+		log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1
+
+		replace_test "$opt" $TESTDIR/$TESTFILE1.1 $TESTDIR/$REPLACEFILE
+
+		zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE"
+		if [[ $? -ne 0 ]]; then
+			log_fail "$REPLACEFILE is not present."
+		fi
+
+		destroy_pool $TESTPOOL1
+		log_must rm -rf /$TESTPOOL1
+	done
+done
+
+log_pass
diff --git a/tests/zfs-tests/tests/functional/replacement/replacement_001_pos.ksh b/tests/zfs-tests/tests/functional/replacement/replace_resilver.ksh
index 8f40436ff..253cf65e4 100755
--- a/tests/zfs-tests/tests/functional/replacement/replacement_001_pos.ksh
+++ b/tests/zfs-tests/tests/functional/replacement/replace_resilver.ksh
@@ -104,9 +104,7 @@ function replace_test
 		((i = i + 1))
 	done
 
-	log_must zpool replace $opt $TESTPOOL1 $disk1 $disk2
-
-	sleep 10
+	log_must zpool replace -w $opt $TESTPOOL1 $disk1 $disk2
 
 	for wait_pid in $child_pids
 	do
@@ -119,11 +117,12 @@ function replace_test
 	log_must zfs umount $TESTPOOL1/$TESTFS1
 	log_must zdb -cdui $TESTPOOL1/$TESTFS1
 	log_must zfs mount $TESTPOOL1/$TESTFS1
+	verify_pool $TESTPOOL1
 }
 
 specials_list=""
 i=0
-while [[ $i != 2 ]]; do
+while [[ $i != 3 ]]; do
 	log_must truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i
 	specials_list="$specials_list $TESTDIR/$TESTFILE1.$i"
 
@@ -143,7 +142,7 @@ for type in "" "raidz" "mirror"; do
 
 		replace_test "$opt" $TESTDIR/$TESTFILE1.1 $TESTDIR/$REPLACEFILE
 
-		zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$REPLACEFILE"
+		zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE"
 		if [[ $? -ne 0 ]]; then
 			log_fail "$REPLACEFILE is not present."
 		fi
diff --git a/tests/zfs-tests/tests/functional/replacement/replacement.cfg b/tests/zfs-tests/tests/functional/replacement/replacement.cfg
index b2ba1b885..271317b1c 100644
--- a/tests/zfs-tests/tests/functional/replacement/replacement.cfg
+++ b/tests/zfs-tests/tests/functional/replacement/replacement.cfg
@@ -36,3 +36,8 @@ export HOLES_SEED=${HOLES_SEED-""}
 export HOLES_FILEOFFSET=${HOLES_FILEOFFSET-""}
 export HOLES_COUNT=${HOLES_COUNT-"16384"}	   # FILESIZE/BLKSIZE/8
 export REPLACEFILE="sparedisk"
+
+set -A VDEV_FILES $TEST_BASE_DIR/file-{1..4}
+export VDEV_FILE_SIZE=$(( $SPA_MINDEVSIZE * 2 ))
+export SPARE_VDEV_FILE=$TEST_BASE_DIR/spare-1
+export SPARE_VDEV_FILE2=$TEST_BASE_DIR/spare-2
diff --git a/tests/zfs-tests/tests/functional/resilver/resilver_restart_001.ksh b/tests/zfs-tests/tests/functional/replacement/resilver_restart_001.ksh
index 9af1c972f..7896b2dbe 100755
--- a/tests/zfs-tests/tests/functional/resilver/resilver_restart_001.ksh
+++ b/tests/zfs-tests/tests/functional/replacement/resilver_restart_001.ksh
@@ -20,7 +20,7 @@
 #
 
 . $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/resilver/resilver.cfg
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
 
 #
 # DESCRIPTION:
@@ -50,7 +50,7 @@ function cleanup
 	    $ORIG_SCAN_SUSPEND_PROGRESS
 	log_must set_tunable32 ZEVENT_LEN_MAX $ORIG_ZFS_ZEVENT_LEN_MAX
 	log_must zinject -c all
-	destroy_pool $TESTPOOL
+	destroy_pool $TESTPOOL1
 	rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE
 }
 
@@ -70,7 +70,7 @@ function verify_restarts # <msg> <cnt> <defer>
 	[[ -z "$defer" ]] && return
 
 	# use zdb to find which vdevs have the resilver defer flag
-	VDEV_DEFERS=$(zdb -C $TESTPOOL | awk '
+	VDEV_DEFERS=$(zdb -C $TESTPOOL1 | awk '
 	    /children/ { gsub(/[^0-9]/, ""); child = $0 }
 	    /com\.datto:resilver_defer$/ { print child }
 	')
@@ -106,17 +106,17 @@ log_must set_tunable32 ZEVENT_LEN_MAX 512
 
 log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE
 
-log_must zpool create -f -o feature@resilver_defer=disabled $TESTPOOL \
+log_must zpool create -f -o feature@resilver_defer=disabled $TESTPOOL1 \
     raidz ${VDEV_FILES[@]}
 
 # create 4 filesystems
 for fs in fs{0..3}
 do
-	log_must zfs create -o primarycache=none -o recordsize=1k $TESTPOOL/$fs
+	log_must zfs create -o primarycache=none -o recordsize=1k $TESTPOOL1/$fs
 done
 
 # simultaneously write 16M to each of them
-set -A DATAPATHS /$TESTPOOL/fs{0..3}/dat.0
+set -A DATAPATHS /$TESTPOOL1/fs{0..3}/dat.0
 log_note "Writing data files"
 for path in ${DATAPATHS[@]}
 do
@@ -131,7 +131,7 @@ do
 
 	if [[ $test == "with" ]]
 	then
-		log_must zpool set feature@resilver_defer=enabled $TESTPOOL
+		log_must zpool set feature@resilver_defer=enabled $TESTPOOL1
 		RESTARTS=( "${DEFER_RESTARTS[@]}" )
 		VDEVS=( "${DEFER_VDEVS[@]}" )
 		VDEV_REPLACE="$SPARE_VDEV_FILE ${VDEV_FILES[1]}"
@@ -144,7 +144,7 @@ do
 	log_must set_tunable32 RESILVER_MIN_TIME_MS 50
 
 	# initiate a resilver and suspend the scan as soon as possible
-	log_must zpool replace $TESTPOOL $VDEV_REPLACE
+	log_must zpool replace $TESTPOOL1 $VDEV_REPLACE
 	log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1
 
 	# there should only be 1 resilver start
@@ -152,16 +152,16 @@ do
 
 	# offline then online a vdev to introduce a new DTL range after current
 	# scan, which should restart (or defer) the resilver
-	log_must zpool offline $TESTPOOL ${VDEV_FILES[2]}
-	log_must zpool sync $TESTPOOL
-	log_must zpool online $TESTPOOL ${VDEV_FILES[2]}
-	log_must zpool sync $TESTPOOL
+	log_must zpool offline $TESTPOOL1 ${VDEV_FILES[2]}
+	log_must zpool sync $TESTPOOL1
+	log_must zpool online $TESTPOOL1 ${VDEV_FILES[2]}
+	log_must zpool sync $TESTPOOL1
 
 	# there should now be 2 resilver starts w/o defer, 1 with defer
 	verify_restarts ' after offline/online' "${RESTARTS[1]}" "${VDEVS[1]}"
 
 	# inject read io errors on vdev and verify resilver does not restart
-	log_must zinject -a -d ${VDEV_FILES[2]} -e io -T read -f 0.25 $TESTPOOL
+	log_must zinject -a -d ${VDEV_FILES[2]} -e io -T read -f 0.25 $TESTPOOL1
 	log_must cat ${DATAPATHS[1]} > /dev/null
 	log_must zinject -c all
 
@@ -173,17 +173,12 @@ do
 	log_must set_tunable32 RESILVER_MIN_TIME_MS 3000
 
 	# wait for resilver to finish
-	for iter in {0..59}
-	do
-		is_pool_resilvered $TESTPOOL && break
-		sleep 1
-	done
-	is_pool_resilvered $TESTPOOL ||
-	    log_fail "resilver timed out"
+	log_must zpool wait -t resilver $TESTPOOL1
+	log_must is_pool_resilvered $TESTPOOL1
 
 	# wait for a few txg's to see if a resilver happens
-	log_must zpool sync $TESTPOOL
-	log_must zpool sync $TESTPOOL
+	log_must zpool sync $TESTPOOL1
+	log_must zpool sync $TESTPOOL1
 
 	# there should now be 2 resilver starts
 	verify_restarts ' after resilver' "${RESTARTS[3]}" "${VDEVS[3]}"
diff --git a/tests/zfs-tests/tests/functional/resilver/resilver_restart_002.ksh b/tests/zfs-tests/tests/functional/replacement/resilver_restart_002.ksh
index ebe5e693b..48763f9b2 100755
--- a/tests/zfs-tests/tests/functional/resilver/resilver_restart_002.ksh
+++ b/tests/zfs-tests/tests/functional/replacement/resilver_restart_002.ksh
@@ -20,7 +20,7 @@
 #
 
 . $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/resilver/resilver.cfg
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
 
 #
 # DESCRIPTION:
@@ -40,7 +40,7 @@
 function cleanup
 {
 	log_must zinject -c all
-	destroy_pool $TESTPOOL
+	destroy_pool $TESTPOOL1
 	rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE
 	log_must set_tunable32 SCAN_LEGACY $ORIG_SCAN_LEGACY
 }
@@ -56,25 +56,25 @@ log_must set_tunable32 SCAN_LEGACY 1
 
  # create the pool and a 32M file (32k blocks)
 log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[0]} $SPARE_VDEV_FILE
-log_must zpool create -f -O recordsize=1k $TESTPOOL ${VDEV_FILES[0]}
-log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=32 > /dev/null 2>&1
+log_must zpool create -f -O recordsize=1k $TESTPOOL1 ${VDEV_FILES[0]}
+log_must dd if=/dev/urandom of=/$TESTPOOL1/file bs=1M count=32 > /dev/null 2>&1
 
 # determine objset/object
-objset=$(zdb -d $TESTPOOL/ | sed -ne 's/.*ID \([0-9]*\).*/\1/p')
-object=$(ls -i /$TESTPOOL/file | awk '{print $1}')
+objset=$(zdb -d $TESTPOOL1/ | sed -ne 's/.*ID \([0-9]*\).*/\1/p')
+object=$(ls -i /$TESTPOOL1/file | awk '{print $1}')
 
 # inject event to cause error during resilver
-log_must zinject -b `printf "%x:%x:0:3fff" $objset $object` $TESTPOOL
+log_must zinject -b `printf "%x:%x:0:3fff" $objset $object` $TESTPOOL1
 
 # clear events and start resilver
 log_must zpool events -c
-log_must zpool attach $TESTPOOL ${VDEV_FILES[0]} $SPARE_VDEV_FILE
+log_must zpool attach $TESTPOOL1 ${VDEV_FILES[0]} $SPARE_VDEV_FILE
 
 log_note "waiting for read errors to start showing up"
 for iter in {0..59}
 do
-	zpool sync $TESTPOOL
-	err=$(zpool status $TESTPOOL | grep ${VDEV_FILES[0]} | awk '{print $3}')
+	zpool sync $TESTPOOL1
+	err=$(zpool status $TESTPOOL1 | grep ${VDEV_FILES[0]} | awk '{print $3}')
 	(( $err > 0 )) && break
 	sleep 1
 done
@@ -92,8 +92,8 @@ done
 (( $finish == 0 )) && log_fail "resilver took too long to finish"
 
 # wait a few syncs to ensure that zfs does not restart the resilver
-log_must zpool sync $TESTPOOL
-log_must zpool sync $TESTPOOL
+log_must zpool sync $TESTPOOL1
+log_must zpool sync $TESTPOOL1
 
 # check if resilver was restarted
 start=$(zpool events | grep "sysevent.fs.zfs.resilver_start" | wc -l)
diff --git a/tests/zfs-tests/tests/functional/replacement/scrub_cancel.ksh b/tests/zfs-tests/tests/functional/replacement/scrub_cancel.ksh
new file mode 100755
index 000000000..da8a0a26e
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/replacement/scrub_cancel.ksh
@@ -0,0 +1,112 @@
+#!/bin/ksh -p
+
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2019, Datto Inc. All rights reserved.
+# Copyright (c) 2020 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/replacement/replacement.cfg
+
+#
+# DESCRIPTION:
+# Verify scrub behaves as intended when contending with a healing or
+# sequential resilver.
+#
+# STRATEGY:
+# 1. Create a pool
+# 2. Add a modest amount of data to the pool.
+# 3. For healing and sequential resilver:
+#    a. Start scrubbing.
+#    b. Verify a resilver can be started and it cancels the scrub.
+#    c. Verify a scrub cannot be started when resilvering
+#
+
+function cleanup
+{
+	log_must set_tunable32 RESILVER_MIN_TIME_MS $ORIG_RESILVER_MIN_TIME
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS \
+	    $ORIG_SCAN_SUSPEND_PROGRESS
+	destroy_pool $TESTPOOL1
+	rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+}
+
+log_assert "Scrub was cancelled by resilver"
+
+ORIG_RESILVER_MIN_TIME=$(get_tunable RESILVER_MIN_TIME_MS)
+ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS)
+
+log_onexit cleanup
+
+log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE
+
+log_must zpool create -f $TESTPOOL1 ${VDEV_FILES[@]}
+log_must zfs create $TESTPOOL1/$TESTFS
+
+mntpnt=$(get_prop mountpoint $TESTPOOL1/$TESTFS)
+log_must dd if=/dev/urandom of=$mntpnt/file bs=1M count=64
+log_must zpool sync $TESTPOOL1
+
+# Request a healing or sequential resilver
+for replace_mode in "healing" "sequential"; do
+
+	#
+	# Healing resilvers abort the dsl_scan and reconfigure it for
+	# resilvering.  Sequential resilvers cancel the dsl_scan and start
+	# the vdev_rebuild thread.
+	#
+	if [[ "$replace_mode" = "healing" ]]; then
+		history_msg="scan aborted, restarting"
+		flags=""
+	else
+		history_msg="scan cancelled"
+		flags="-s"
+	fi
+
+	# Limit scanning time and suspend the scan as soon as possible.
+	log_must set_tunable32 RESILVER_MIN_TIME_MS 50
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1
+
+	# Initiate a scrub.
+	log_must zpool scrub $TESTPOOL1
+
+	# Initiate a resilver to cancel the scrub.
+	log_must zpool replace $flags $TESTPOOL1 ${VDEV_FILES[1]} \
+	    $SPARE_VDEV_FILE
+
+	# Verify the scrub was canceled, it may take a few seconds to exit.
+	while is_pool_scrubbing $TESTPOOL1; do
+		sleep 1
+	done
+	log_mustnot is_pool_scrubbing $TESTPOOL1
+
+	# Verify a scrub cannot be started while resilvering.
+	log_must is_pool_resilvering $TESTPOOL1
+	log_mustnot zpool scrub $TESTPOOL1
+
+	# Unsuspend resilver.
+	log_must set_tunable32 SCAN_SUSPEND_PROGRESS 0
+	log_must set_tunable32 RESILVER_MIN_TIME_MS 3000
+
+	# Wait for resilver to finish then put the original back.
+	log_must zpool wait $TESTPOOL1
+	log_must zpool replace $flags -w $TESTPOOL1 $SPARE_VDEV_FILE \
+	    ${VDEV_FILES[1]}
+done
+log_pass "Scrub was cancelled by resilver"
+
diff --git a/tests/zfs-tests/tests/functional/resilver/Makefile.am b/tests/zfs-tests/tests/functional/resilver/Makefile.am
deleted file mode 100644
index 38136a843..000000000
--- a/tests/zfs-tests/tests/functional/resilver/Makefile.am
+++ /dev/null
@@ -1,9 +0,0 @@
-pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/resilver
-dist_pkgdata_SCRIPTS = \
-	setup.ksh \
-	cleanup.ksh \
-	resilver_restart_001.ksh \
-	resilver_restart_002.ksh
-
-dist_pkgdata_DATA = \
-	resilver.cfg
diff --git a/tests/zfs-tests/tests/functional/resilver/cleanup.ksh b/tests/zfs-tests/tests/functional/resilver/cleanup.ksh
deleted file mode 100755
index 4dfa81424..000000000
--- a/tests/zfs-tests/tests/functional/resilver/cleanup.ksh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/ksh -p
-#
-# CDDL HEADER START
-#
-# The contents of this file are subject to the terms of the
-# Common Development and Distribution License (the "License").
-# You may not use this file except in compliance with the License.
-#
-# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
-# or http://www.opensolaris.org/os/licensing.
-# See the License for the specific language governing permissions
-# and limitations under the License.
-#
-# When distributing Covered Code, include this CDDL HEADER in each
-# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
-# If applicable, add the following below this CDDL HEADER, with the
-# fields enclosed by brackets "[]" replaced with your own identifying
-# information: Portions Copyright [yyyy] [name of copyright owner]
-#
-# CDDL HEADER END
-
-#
-# Copyright (c) 2019, Datto Inc. All rights reserved.
-#
-
-. $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/resilver/resilver.cfg
-
-verify_runnable "global"
-
-log_pass
diff --git a/tests/zfs-tests/tests/functional/resilver/resilver.cfg b/tests/zfs-tests/tests/functional/resilver/resilver.cfg
deleted file mode 100644
index 88dfd24ae..000000000
--- a/tests/zfs-tests/tests/functional/resilver/resilver.cfg
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# CDDL HEADER START
-#
-# The contents of this file are subject to the terms of the
-# Common Development and Distribution License (the "License").
-# You may not use this file except in compliance with the License.
-#
-# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
-# or http://www.opensolaris.org/os/licensing.
-# See the License for the specific language governing permissions
-# and limitations under the License.
-#
-# When distributing Covered Code, include this CDDL HEADER in each
-# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
-# If applicable, add the following below this CDDL HEADER, with the
-# fields enclosed by brackets "[]" replaced with your own identifying
-# information: Portions Copyright [yyyy] [name of copyright owner]
-#
-# CDDL HEADER END
-
-#
-# Copyright (c) 2019, Datto Inc. All rights reserved.
-#
-
-. $STF_SUITE/include/libtest.shlib
-
-verify_runnable "global"
-
-set -A VDEV_FILES $TEST_BASE_DIR/file-{1..4}
-SPARE_VDEV_FILE=$TEST_BASE_DIR/spare-1
-
-VDEV_FILE_SIZE=$(( $SPA_MINDEVSIZE * 2 ))
diff --git a/tests/zfs-tests/tests/functional/resilver/setup.ksh b/tests/zfs-tests/tests/functional/resilver/setup.ksh
deleted file mode 100755
index 4dfa81424..000000000
--- a/tests/zfs-tests/tests/functional/resilver/setup.ksh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/ksh -p
-#
-# CDDL HEADER START
-#
-# The contents of this file are subject to the terms of the
-# Common Development and Distribution License (the "License").
-# You may not use this file except in compliance with the License.
-#
-# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
-# or http://www.opensolaris.org/os/licensing.
-# See the License for the specific language governing permissions
-# and limitations under the License.
-#
-# When distributing Covered Code, include this CDDL HEADER in each
-# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
-# If applicable, add the following below this CDDL HEADER, with the
-# fields enclosed by brackets "[]" replaced with your own identifying
-# information: Portions Copyright [yyyy] [name of copyright owner]
-#
-# CDDL HEADER END
-
-#
-# Copyright (c) 2019, Datto Inc. All rights reserved.
-#
-
-. $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/resilver/resilver.cfg
-
-verify_runnable "global"
-
-log_pass