aboutsummaryrefslogtreecommitdiffstats
path: root/tests
diff options
context:
space:
mode:
authorBrian Behlendorf <[email protected]>2018-12-04 09:37:37 -0800
committerGitHub <[email protected]>2018-12-04 09:37:37 -0800
commit7c9a42921e60dbad0e3003bd571591f073860233 (patch)
tree7dcdfdf535f286a9c3d4dc5f4996ed0e59c501c2 /tests
parentc40a1124e1d1010b665909ad31d2904630018f6f (diff)
Detect IO errors during device removal
* Detect IO errors during device removal While device removal cannot verify the checksums of individual blocks during device removal, it can reasonably detect hard IO errors from the leaf vdevs. Failure to perform this error checking can result in device removal completing successfully, but moving no data which will permanently corrupt the pool. Situation 1: faulted/degraded vdevs In the configuration shown below, the removal of mirror-0 will permanently corrupt the pool. Device removal will preferentially copy data from 'vdev1 -> vdev3' and from 'vdev2 -> vdev4'. Which in this case will result in nothing being copied since one vdev in each of those groups in unavailable. However, device removal will complete successfully since all IO errors are ignored. tank DEGRADED 0 0 0 mirror-0 DEGRADED 0 0 0 /var/tmp/vdev1 FAULTED 0 0 0 external fault /var/tmp/vdev2 ONLINE 0 0 0 mirror-1 DEGRADED 0 0 0 /var/tmp/vdev3 ONLINE 0 0 0 /var/tmp/vdev4 FAULTED 0 0 0 external fault This issue is resolved by updating the source child selection logic to exclude unreadable leaf vdevs. Additionally, unwritable destination child vdevs which can never succeed are skipped to prevent generating a large number of write IO errors. Situation 2: individual hard IO errors During removal if an unexpected hard IO error is encountered when either reading or writing the child vdev the entire removal operation is cancelled. While it may be possible to reconstruct the data after removal that cannot be guaranteed. The only strictly safe thing to do is to cancel the removal. As a future improvement we may want to instead suspend the removal process and allow the damaged region to be retried. But that work is left for another time, hard IO errors during the removal process are expected to be exceptionally rare. Reviewed-by: Serapheim Dimitropoulos <[email protected]> Reviewed-by: Tony Hutter <[email protected]> Reviewed-by: Tom Caputi <[email protected]> Signed-off-by: Brian Behlendorf <[email protected]> Issue #6900 Closes #8161
Diffstat (limited to 'tests')
-rw-r--r--tests/runfiles/linux.run6
-rw-r--r--tests/zfs-tests/include/libtest.shlib22
-rw-r--r--tests/zfs-tests/tests/functional/removal/Makefile.am3
-rw-r--r--tests/zfs-tests/tests/functional/removal/removal.kshlib2
-rwxr-xr-xtests/zfs-tests/tests/functional/removal/removal_with_errors.ksh109
-rwxr-xr-xtests/zfs-tests/tests/functional/removal/removal_with_faulted.ksh104
6 files changed, 241 insertions, 5 deletions
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index a4c6769db..3dd9656f0 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -728,9 +728,9 @@ tests = ['removal_all_vdev', 'removal_check_space',
'removal_remap', 'removal_remap_deadlists',
'removal_resume_export', 'removal_sanity', 'removal_with_add',
'removal_with_create_fs', 'removal_with_dedup',
- 'removal_with_export', 'removal_with_ganging',
- 'removal_with_remap', 'removal_with_remove',
- 'removal_with_scrub', 'removal_with_send',
+ 'removal_with_errors', 'removal_with_export',
+ 'removal_with_ganging', 'removal_with_faulted', 'removal_with_remap',
+ 'removal_with_remove', 'removal_with_scrub', 'removal_with_send',
'removal_with_send_recv', 'removal_with_snapshot',
'removal_with_write', 'removal_with_zdb', 'remove_expanded',
'remove_mirror', 'remove_mirror_sanity', 'remove_raidz']
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index e66b2cbaf..2effa4207 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -1932,6 +1932,23 @@ function verify_filesys # pool filesystem dir
}
#
+# Given a pool issue a scrub and verify that no checksum errors are reported.
+#
+function verify_pool
+{
+ typeset pool=${1:-$TESTPOOL}
+
+ log_must zpool scrub $pool
+ log_must wait_scrubbed $pool
+
+ cksum=$(zpool status $pool | awk 'L{print $NF;L=0} /CKSUM$/{L=1}')
+ if [[ $cksum != 0 ]]; then
+ log_must zpool status -v
+ log_fail "Unexpected CKSUM errors found on $pool ($cksum)"
+ fi
+}
+
+#
# Given a pool, and this function list all disks in the pool
#
function get_disklist # pool
@@ -3025,8 +3042,11 @@ function vdevs_in_pool
shift
+ # We could use 'zpool list' to only get the vdevs of the pool but we
+ # can't reference a mirror/raidz vdev using its ID (i.e mirror-0),
+ # therefore we use the 'zpool status' output.
typeset tmpfile=$(mktemp)
- zpool list -Hv "$pool" >$tmpfile
+ zpool status -v "$pool" | grep -A 1000 "config:" >$tmpfile
for vdev in $@; do
grep -w ${vdev##*/} $tmpfile >/dev/null 2>&1
[[ $? -ne 0 ]] && return 1
diff --git a/tests/zfs-tests/tests/functional/removal/Makefile.am b/tests/zfs-tests/tests/functional/removal/Makefile.am
index c2b333a00..c5d013e7c 100644
--- a/tests/zfs-tests/tests/functional/removal/Makefile.am
+++ b/tests/zfs-tests/tests/functional/removal/Makefile.am
@@ -21,7 +21,8 @@ dist_pkgdata_SCRIPTS = \
removal_remap_deadlists.ksh removal_remap.ksh \
removal_reservation.ksh removal_resume_export.ksh \
removal_sanity.ksh removal_with_add.ksh removal_with_create_fs.ksh \
- removal_with_dedup.ksh removal_with_export.ksh \
+ removal_with_dedup.ksh removal_with_errors.ksh \
+ removal_with_export.ksh removal_with_faulted.ksh \
removal_with_ganging.ksh removal_with_remap.ksh \
removal_with_remove.ksh removal_with_scrub.ksh \
removal_with_send.ksh removal_with_send_recv.ksh \
diff --git a/tests/zfs-tests/tests/functional/removal/removal.kshlib b/tests/zfs-tests/tests/functional/removal/removal.kshlib
index c1ab044c7..fa0174db0 100644
--- a/tests/zfs-tests/tests/functional/removal/removal.kshlib
+++ b/tests/zfs-tests/tests/functional/removal/removal.kshlib
@@ -141,6 +141,8 @@ function test_removal_with_operation # callback [args]
kill $killpid
wait
+
+ verify_pool $TESTPOOL
}
#
diff --git a/tests/zfs-tests/tests/functional/removal/removal_with_errors.ksh b/tests/zfs-tests/tests/functional/removal/removal_with_errors.ksh
new file mode 100755
index 000000000..2ef56706a
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/removal/removal_with_errors.ksh
@@ -0,0 +1,109 @@
+#! /bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2014, 2017 by Delphix. All rights reserved.
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/removal/removal.kshlib
+
+#
+# DESCRIPTION:
+#
+# This test ensures the device removal is cancelled when hard IO
+# errors are encountered during the removal process. This is done
+# to ensure that when removing a device all of the data is copied.
+#
+# STRATEGY:
+#
+# 1. We create a pool with enough redundancy such that IO errors
+# will not result in the pool being suspended.
+# 2. We write some test data to the pool.
+# 3. We inject READ errors in to one half of the top-level mirror-0
+# vdev which is being removed. Then we start the removal process.
+# 4. Verify that the injected read errors cause the removal of
+# mirror-0 to be cancelled and that mirror-0 has not been removed.
+# 5. Clear the read fault injection.
+# 6. Repeat steps 3-6 above except inject WRITE errors on one of
+# child vdevs in the destination mirror-1.
+# 7. Lastly verify the pool data is still intact.
+#
+
+TMPDIR=${TMPDIR:-$TEST_BASE_DIR}
+DISK0=$TMPDIR/dsk0
+DISK1=$TMPDIR/dsk1
+DISK2=$TMPDIR/dsk2
+DISK3=$TMPDIR/dsk3
+
+log_must truncate -s $MINVDEVSIZE $DISK0 $DISK1
+log_must truncate -s $((MINVDEVSIZE * 4)) $DISK2 $DISK3
+
+function cleanup
+{
+ log_must zinject -c all
+ default_cleanup_noexit
+ log_must rm -f $DISK0 $DISK1 $DISK2 $DISK3
+}
+
+function wait_for_removing_cancel
+{
+ typeset pool=$1
+
+ while is_pool_removing $pool; do
+ sleep 1
+ done
+
+ #
+ # The pool state changes before the TXG finishes syncing; wait for
+ # the removal to be completed on disk.
+ #
+ sync_pool
+
+ log_mustnot is_pool_removed $pool
+ return 0
+}
+
+default_setup_noexit "mirror $DISK0 $DISK1 mirror $DISK2 $DISK3"
+log_onexit cleanup
+
+FILE_CONTENTS="Leeloo Dallas mul-ti-pass."
+
+echo $FILE_CONTENTS >$TESTDIR/$TESTFILE0
+log_must [ "x$(<$TESTDIR/$TESTFILE0)" = "x$FILE_CONTENTS" ]
+log_must file_write -o create -f $TESTDIR/$TESTFILE1 -b $((2**20)) -c $((2**7))
+sync_pool $TESTPOOL
+
+# Verify that unexpected read errors automatically cancel the removal.
+log_must zinject -d $DISK0 -e io -T all -f 100 $TESTPOOL
+log_must zpool remove $TESTPOOL mirror-0
+log_must wait_for_removing_cancel $TESTPOOL
+log_must vdevs_in_pool $TESTPOOL mirror-0
+log_must zinject -c all
+
+# Verify that unexpected write errors automatically cancel the removal.
+log_must zinject -d $DISK3 -e io -T all -f 100 $TESTPOOL
+log_must zpool remove $TESTPOOL mirror-0
+log_must wait_for_removing_cancel $TESTPOOL
+log_must vdevs_in_pool $TESTPOOL mirror-0
+log_must zinject -c all
+
+log_must dd if=$TESTDIR/$TESTFILE0 of=/dev/null
+log_must [ "x$(<$TESTDIR/$TESTFILE0)" = "x$FILE_CONTENTS" ]
+log_must dd if=$TESTDIR/$TESTFILE1 of=/dev/null
+
+log_pass "Device not removed due to unexpected errors."
diff --git a/tests/zfs-tests/tests/functional/removal/removal_with_faulted.ksh b/tests/zfs-tests/tests/functional/removal/removal_with_faulted.ksh
new file mode 100755
index 000000000..44d222860
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/removal/removal_with_faulted.ksh
@@ -0,0 +1,104 @@
+#! /bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2014, 2017 by Delphix. All rights reserved.
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/removal/removal.kshlib
+
+#
+# DESCRIPTION:
+#
+# This test ensures that even when child vdevs are unavailable the
+# device removal process copies from readable source children to
+# writable destination children. This may be different than the
+# default mapping which preferentially pairs up source and destination
+# child vdevs based on their child ids.
+#
+# Default Mapping:
+# mirror-0 mirror-1
+# DISK0 (child 0) ------> DISK2 (child 0)
+# DISK1 (child 1) ------> DISK3 (child 1)
+#
+# We want to setup a scenario where the default mapping would make
+# it impossible to copy any data during the removal process. This
+# is done by faulting both the mirror-0 (child 0) source vdev and
+# mirror-1 (child 1) destination vdev. As shown below the default
+# mapping cannot be used due to the FAULTED vdevs. Verify that an
+# alternate mapping is selected and all the readable data is copied.
+#
+# Default Mapping (BAD):
+# mirror-0 mirror-1
+# DISK0 (FAULTED) ------> DISK2
+# DISK1 ----------------> DISK3 (FAULTED)
+#
+# Required Mapping (GOOD):
+# mirror-0 mirror-1
+# DISK0 (FAULTED) +---> DISK2
+# DISK1 ------------+ DISK3 (FAULTED)
+#
+# STRATEGY:
+#
+# 1. We create a pool with two top-level mirror vdevs.
+# 2. We write some test data to the pool.
+# 3. We fault two children to force the scenario described above.
+# 4. We remove the mirror-0 device.
+# 5. We verify that the device has been removed and that all of the
+# data is still intact.
+#
+
+TMPDIR=${TMPDIR:-$TEST_BASE_DIR}
+DISK0=$TMPDIR/dsk0
+DISK1=$TMPDIR/dsk1
+DISK2=$TMPDIR/dsk2
+DISK3=$TMPDIR/dsk3
+
+log_must truncate -s $MINVDEVSIZE $DISK0 $DISK1
+log_must truncate -s $((MINVDEVSIZE * 4)) $DISK2 $DISK3
+
+function cleanup
+{
+ default_cleanup_noexit
+ log_must rm -f $DISK0 $DISK1 $DISK2 $DISK3
+}
+
+default_setup_noexit "mirror $DISK0 $DISK1 mirror $DISK2 $DISK3"
+log_onexit cleanup
+
+log_must zpool offline -f $TESTPOOL $DISK0
+log_must zpool offline -f $TESTPOOL $DISK3
+
+FILE_CONTENTS="Leeloo Dallas mul-ti-pass."
+
+echo $FILE_CONTENTS >$TESTDIR/$TESTFILE0
+log_must [ "x$(<$TESTDIR/$TESTFILE0)" = "x$FILE_CONTENTS" ]
+log_must file_write -o create -f $TESTDIR/$TESTFILE1 -b $((2**20)) -c $((2**7))
+sync_pool $TESTPOOL
+
+log_must zpool remove $TESTPOOL mirror-0
+log_must wait_for_removal $TESTPOOL
+log_mustnot vdevs_in_pool $TESTPOOL mirror-0
+
+verify_pool $TESTPOOL
+
+log_must dd if=$TESTDIR/$TESTFILE0 of=/dev/null
+log_must [ "x$(<$TESTDIR/$TESTFILE0)" = "x$FILE_CONTENTS" ]
+log_must dd if=$TESTDIR/$TESTFILE1 of=/dev/null
+
+log_pass "Can remove with faulted vdevs"