6 files changed, 241 insertions, 5 deletions
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index a4c6769db..3dd9656f0 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -728,9 +728,9 @@ tests = ['removal_all_vdev', 'removal_check_space',
     'removal_remap', 'removal_remap_deadlists',
     'removal_resume_export', 'removal_sanity', 'removal_with_add',
     'removal_with_create_fs', 'removal_with_dedup',
-    'removal_with_export', 'removal_with_ganging',
-    'removal_with_remap', 'removal_with_remove',
-    'removal_with_scrub', 'removal_with_send',
+    'removal_with_errors', 'removal_with_export',
+    'removal_with_ganging', 'removal_with_faulted', 'removal_with_remap',
+    'removal_with_remove', 'removal_with_scrub', 'removal_with_send',
     'removal_with_send_recv', 'removal_with_snapshot',
     'removal_with_write', 'removal_with_zdb', 'remove_expanded',
     'remove_mirror', 'remove_mirror_sanity', 'remove_raidz']
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index e66b2cbaf..2effa4207 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -1932,6 +1932,23 @@ function verify_filesys # pool filesystem dir
 }
 
 #
+# Given a pool issue a scrub and verify that no checksum errors are reported.
+#
+function verify_pool
+{
+	typeset pool=${1:-$TESTPOOL}
+
+	log_must zpool scrub $pool
+	log_must wait_scrubbed $pool
+
+	cksum=$(zpool status $pool | awk 'L{print $NF;L=0} /CKSUM$/{L=1}')
+	if [[ $cksum != 0 ]]; then
+		log_must zpool status -v
+	        log_fail "Unexpected CKSUM errors found on $pool ($cksum)"
+	fi
+}
+
+#
 # Given a pool, and this function list all disks in the pool
 #
 function get_disklist # pool
@@ -3025,8 +3042,11 @@ function vdevs_in_pool
 
 	shift
 
+	# We could use 'zpool list' to only get the vdevs of the pool but we
+	# can't reference a mirror/raidz vdev using its ID (i.e mirror-0),
+	# therefore we use the 'zpool status' output.
 	typeset tmpfile=$(mktemp)
-	zpool list -Hv "$pool" >$tmpfile
+	zpool status -v "$pool" | grep -A 1000 "config:" >$tmpfile
 	for vdev in $@; do
 		grep -w ${vdev##*/} $tmpfile >/dev/null 2>&1
 		[[ $? -ne 0 ]] && return 1
diff --git a/tests/zfs-tests/tests/functional/removal/Makefile.am b/tests/zfs-tests/tests/functional/removal/Makefile.am
index c2b333a00..c5d013e7c 100644
--- a/tests/zfs-tests/tests/functional/removal/Makefile.am
+++ b/tests/zfs-tests/tests/functional/removal/Makefile.am
@@ -21,7 +21,8 @@ dist_pkgdata_SCRIPTS = \
 	removal_remap_deadlists.ksh removal_remap.ksh \
 	removal_reservation.ksh removal_resume_export.ksh \
 	removal_sanity.ksh removal_with_add.ksh removal_with_create_fs.ksh \
-	removal_with_dedup.ksh removal_with_export.ksh \
+	removal_with_dedup.ksh removal_with_errors.ksh \
+	removal_with_export.ksh removal_with_faulted.ksh \
 	removal_with_ganging.ksh removal_with_remap.ksh \
 	removal_with_remove.ksh removal_with_scrub.ksh \
 	removal_with_send.ksh removal_with_send_recv.ksh \
diff --git a/tests/zfs-tests/tests/functional/removal/removal.kshlib b/tests/zfs-tests/tests/functional/removal/removal.kshlib
index c1ab044c7..fa0174db0 100644
--- a/tests/zfs-tests/tests/functional/removal/removal.kshlib
+++ b/tests/zfs-tests/tests/functional/removal/removal.kshlib
@@ -141,6 +141,8 @@ function test_removal_with_operation # callback [args]
 
 	kill $killpid
 	wait
+
+	verify_pool $TESTPOOL
 }
 
 #
diff --git a/tests/zfs-tests/tests/functional/removal/removal_with_errors.ksh b/tests/zfs-tests/tests/functional/removal/removal_with_errors.ksh
new file mode 100755
index 000000000..2ef56706a
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/removal/removal_with_errors.ksh
@@ -0,0 +1,109 @@
+#! /bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2014, 2017 by Delphix. All rights reserved.
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/removal/removal.kshlib
+
+#
+# DESCRIPTION:
+#
+# This test ensures the device removal is cancelled when hard IO
+# errors are encountered during the removal process.  This is done
+# to ensure that when removing a device all of the data is copied.
+#
+# STRATEGY:
+#
+# 1. We create a pool with enough redundancy such that IO errors
+#    will not result in the pool being suspended.
+# 2. We write some test data to the pool.
+# 3. We inject READ errors in to one half of the top-level mirror-0
+#    vdev which is being removed.  Then we start the removal process.
+# 4. Verify that the injected read errors cause the removal of
+#    mirror-0 to be cancelled and that mirror-0 has not been removed.
+# 5. Clear the read fault injection.
+# 6. Repeat steps 3-6 above except inject WRITE errors on one of
+#    child vdevs in the destination mirror-1.
+# 7. Lastly verify the pool data is still intact.
+#
+
+TMPDIR=${TMPDIR:-$TEST_BASE_DIR}
+DISK0=$TMPDIR/dsk0
+DISK1=$TMPDIR/dsk1
+DISK2=$TMPDIR/dsk2
+DISK3=$TMPDIR/dsk3
+
+log_must truncate -s $MINVDEVSIZE $DISK0 $DISK1
+log_must truncate -s $((MINVDEVSIZE * 4)) $DISK2 $DISK3
+
+function cleanup
+{
+	log_must zinject -c all
+	default_cleanup_noexit
+	log_must rm -f $DISK0 $DISK1 $DISK2 $DISK3
+}
+
+function wait_for_removing_cancel
+{
+	typeset pool=$1
+
+	while is_pool_removing $pool; do
+		sleep 1
+	done
+
+	#
+	# The pool state changes before the TXG finishes syncing; wait for
+	# the removal to be completed on disk.
+	#
+	sync_pool
+
+	log_mustnot is_pool_removed $pool
+	return 0
+}
+
+default_setup_noexit "mirror $DISK0 $DISK1 mirror $DISK2 $DISK3"
+log_onexit cleanup
+
+FILE_CONTENTS="Leeloo Dallas mul-ti-pass."
+
+echo $FILE_CONTENTS  >$TESTDIR/$TESTFILE0
+log_must [ "x$(<$TESTDIR/$TESTFILE0)" = "x$FILE_CONTENTS" ]
+log_must file_write -o create -f $TESTDIR/$TESTFILE1 -b $((2**20)) -c $((2**7))
+sync_pool $TESTPOOL
+
+# Verify that unexpected read errors automatically cancel the removal.
+log_must zinject -d $DISK0 -e io -T all -f 100 $TESTPOOL
+log_must zpool remove $TESTPOOL mirror-0
+log_must wait_for_removing_cancel $TESTPOOL
+log_must vdevs_in_pool $TESTPOOL mirror-0
+log_must zinject -c all
+
+# Verify that unexpected write errors automatically cancel the removal.
+log_must zinject -d $DISK3 -e io -T all -f 100 $TESTPOOL
+log_must zpool remove $TESTPOOL mirror-0
+log_must wait_for_removing_cancel $TESTPOOL
+log_must vdevs_in_pool $TESTPOOL mirror-0
+log_must zinject -c all
+
+log_must dd if=$TESTDIR/$TESTFILE0 of=/dev/null
+log_must [ "x$(<$TESTDIR/$TESTFILE0)" = "x$FILE_CONTENTS" ]
+log_must dd if=$TESTDIR/$TESTFILE1 of=/dev/null
+
+log_pass "Device not removed due to unexpected errors."
diff --git a/tests/zfs-tests/tests/functional/removal/removal_with_faulted.ksh b/tests/zfs-tests/tests/functional/removal/removal_with_faulted.ksh
new file mode 100755
index 000000000..44d222860
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/removal/removal_with_faulted.ksh
@@ -0,0 +1,104 @@
+#! /bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2014, 2017 by Delphix. All rights reserved.
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/removal/removal.kshlib
+
+#
+# DESCRIPTION:
+#
+# This test ensures that even when child vdevs are unavailable the
+# device removal process copies from readable source children to
+# writable destination children.  This may be different than the
+# default mapping which preferentially pairs up source and destination
+# child vdevs based on their child ids.
+#
+# Default Mapping:
+#   mirror-0                mirror-1
+#     DISK0 (child 0) ------> DISK2 (child 0)
+#     DISK1 (child 1) ------> DISK3 (child 1)
+#
+# We want to setup a scenario where the default mapping would make
+# it impossible to copy any data during the removal process.  This
+# is done by faulting both the mirror-0 (child 0) source vdev and
+# mirror-1 (child 1) destination vdev.  As shown below the default
+# mapping cannot be used due to the FAULTED vdevs.  Verify that an
+# alternate mapping is selected and all the readable data is copied.
+#
+# Default Mapping (BAD):
+#   mirror-0                mirror-1
+#     DISK0 (FAULTED) ------> DISK2
+#     DISK1 ----------------> DISK3 (FAULTED)
+#
+# Required Mapping (GOOD):
+#   mirror-0                mirror-1
+#     DISK0 (FAULTED)   +---> DISK2
+#     DISK1 ------------+     DISK3 (FAULTED)
+#
+# STRATEGY:
+#
+# 1. We create a pool with two top-level mirror vdevs.
+# 2. We write some test data to the pool.
+# 3. We fault two children to force the scenario described above.
+# 4. We remove the mirror-0 device.
+# 5. We verify that the device has been removed and that all of the
+#    data is still intact.
+#
+
+TMPDIR=${TMPDIR:-$TEST_BASE_DIR}
+DISK0=$TMPDIR/dsk0
+DISK1=$TMPDIR/dsk1
+DISK2=$TMPDIR/dsk2
+DISK3=$TMPDIR/dsk3
+
+log_must truncate -s $MINVDEVSIZE $DISK0 $DISK1
+log_must truncate -s $((MINVDEVSIZE * 4)) $DISK2 $DISK3
+
+function cleanup
+{
+	default_cleanup_noexit
+	log_must rm -f $DISK0 $DISK1 $DISK2 $DISK3
+}
+
+default_setup_noexit "mirror $DISK0 $DISK1 mirror $DISK2 $DISK3"
+log_onexit cleanup
+
+log_must zpool offline -f $TESTPOOL $DISK0
+log_must zpool offline -f $TESTPOOL $DISK3
+
+FILE_CONTENTS="Leeloo Dallas mul-ti-pass."
+
+echo $FILE_CONTENTS  >$TESTDIR/$TESTFILE0
+log_must [ "x$(<$TESTDIR/$TESTFILE0)" = "x$FILE_CONTENTS" ]
+log_must file_write -o create -f $TESTDIR/$TESTFILE1 -b $((2**20)) -c $((2**7))
+sync_pool $TESTPOOL
+
+log_must zpool remove $TESTPOOL mirror-0
+log_must wait_for_removal $TESTPOOL
+log_mustnot vdevs_in_pool $TESTPOOL mirror-0
+
+verify_pool $TESTPOOL
+
+log_must dd if=$TESTDIR/$TESTFILE0 of=/dev/null
+log_must [ "x$(<$TESTDIR/$TESTFILE0)" = "x$FILE_CONTENTS" ]
+log_must dd if=$TESTDIR/$TESTFILE1 of=/dev/null
+
+log_pass "Can remove with faulted vdevs"