diff options
Diffstat (limited to 'tests/zfs-tests')
25 files changed, 1105 insertions, 172 deletions
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index 9fbcc37c6..5e07cda4d 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -2222,26 +2222,27 @@ function check_pool_status # pool token keyword <verbose> if [[ $verbose == true ]]; then log_note $scan fi - echo $scan | grep -i "$keyword" > /dev/null 2>&1 + echo $scan | egrep -i "$keyword" > /dev/null 2>&1 return $? } # # The following functions are instance of check_pool_status() -# is_pool_resilvering - to check if the pool is resilver in progress -# is_pool_resilvered - to check if the pool is resilver completed -# is_pool_scrubbing - to check if the pool is scrub in progress -# is_pool_scrubbed - to check if the pool is scrub completed -# is_pool_scrub_stopped - to check if the pool is scrub stopped -# is_pool_scrub_paused - to check if the pool has scrub paused -# is_pool_removing - to check if the pool is removing a vdev -# is_pool_removed - to check if the pool is remove completed -# is_pool_discarding - to check if the pool has checkpoint being discarded +# is_pool_resilvering - to check if the pool resilver is in progress +# is_pool_resilvered - to check if the pool resilver is completed +# is_pool_scrubbing - to check if the pool scrub is in progress +# is_pool_scrubbed - to check if the pool scrub is completed +# is_pool_scrub_stopped - to check if the pool scrub is stopped +# is_pool_scrub_paused - to check if the pool scrub has paused +# is_pool_removing - to check if the pool removing is a vdev +# is_pool_removed - to check if the pool remove is completed +# is_pool_discarding - to check if the pool checkpoint is being discarded # function is_pool_resilvering #pool <verbose> { - check_pool_status "$1" "scan" "resilver in progress since " $2 + check_pool_status "$1" "scan" \ + "resilver[ ()0-9A-Za-z_-]* in progress since" $2 return $? } @@ -3487,7 +3488,7 @@ function wait_scrubbed typeset pool=${1:-$TESTPOOL} while true ; do is_pool_scrubbed $pool && break - log_must sleep 1 + sleep 1 done } diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am index 24f3e50bb..c56518c55 100644 --- a/tests/zfs-tests/tests/functional/Makefile.am +++ b/tests/zfs-tests/tests/functional/Makefile.am @@ -65,7 +65,6 @@ SUBDIRS = \ rename_dirs \ replacement \ reservation \ - resilver \ rootpool \ rsend \ scrub_mirror \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg index ee5b2b4e1..4991b76bf 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg @@ -79,6 +79,7 @@ typeset -a properties=( "feature@redacted_datasets" "feature@bookmark_written" "feature@log_spacemap" + "feature@device_rebuild" ) if is_linux || is_freebsd; then diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/Makefile.am index 6a21cac4f..451d83a79 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/Makefile.am +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/Makefile.am @@ -4,6 +4,7 @@ dist_pkgdata_SCRIPTS = \ cleanup.ksh \ zpool_wait_replace.ksh \ zpool_wait_replace_cancel.ksh \ + zpool_wait_rebuild.ksh \ zpool_wait_resilver.ksh \ zpool_wait_scrub_basic.ksh \ zpool_wait_scrub_cancel.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/zpool_wait_rebuild.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/zpool_wait_rebuild.ksh new file mode 100755 index 000000000..8cd586459 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_wait/scan/zpool_wait_rebuild.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh -p +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_wait/zpool_wait.kshlib + +# +# DESCRIPTION: +# 'zpool wait' works when waiting for sequential resilvering to complete. +# +# STRATEGY: +# 1. Attach a device to the pool so that sequential resilvering starts. +# 2. Start 'zpool wait'. +# 3. Monitor the waiting process to make sure it returns neither too soon nor +# too late. +# 4. Repeat 1-3, except using the '-w' flag with 'zpool attach' instead of using +# 'zpool wait'. +# + +function cleanup +{ + remove_io_delay + kill_if_running $pid + get_disklist $TESTPOOL | grep $DISK2 >/dev/null && \ + log_must zpool detach $TESTPOOL $DISK2 +} + +typeset -r IN_PROGRESS_CHECK="is_pool_resilvering $TESTPOOL" +typeset pid + +log_onexit cleanup + +add_io_delay $TESTPOOL + +# Test 'zpool wait -t resilver' +log_must zpool attach -s $TESTPOOL $DISK1 $DISK2 +log_bkgrnd zpool wait -t resilver $TESTPOOL +pid=$! +check_while_waiting $pid "$IN_PROGRESS_CHECK" + +log_must zpool detach $TESTPOOL $DISK2 + +# Test 'zpool attach -w' +log_bkgrnd zpool attach -sw $TESTPOOL $DISK1 $DISK2 +pid=$! +while ! is_pool_resilvering $TESTPOOL && proc_exists $pid; do + log_must sleep .5 +done +check_while_waiting $pid "$IN_PROGRESS_CHECK" + +log_pass "'zpool wait -t resilver' and 'zpool attach -w' work." diff --git a/tests/zfs-tests/tests/functional/replacement/Makefile.am b/tests/zfs-tests/tests/functional/replacement/Makefile.am index d47fcd5e1..fe6e49121 100644 --- a/tests/zfs-tests/tests/functional/replacement/Makefile.am +++ b/tests/zfs-tests/tests/functional/replacement/Makefile.am @@ -2,9 +2,20 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/replacement dist_pkgdata_SCRIPTS = \ setup.ksh \ cleanup.ksh \ - replacement_001_pos.ksh \ - replacement_002_pos.ksh \ - replacement_003_pos.ksh + attach_import.ksh \ + attach_multiple.ksh \ + attach_rebuild.ksh \ + attach_resilver.ksh \ + detach.ksh \ + rebuild_disabled_feature.ksh \ + rebuild_multiple.ksh \ + rebuild_raidz.ksh \ + replace_import.ksh \ + replace_rebuild.ksh \ + replace_resilver.ksh \ + resilver_restart_001.ksh \ + resilver_restart_002.ksh \ + scrub_cancel.ksh dist_pkgdata_DATA = \ replacement.cfg diff --git a/tests/zfs-tests/tests/functional/replacement/attach_import.ksh b/tests/zfs-tests/tests/functional/replacement/attach_import.ksh new file mode 100755 index 000000000..e2749b164 --- /dev/null +++ b/tests/zfs-tests/tests/functional/replacement/attach_import.ksh @@ -0,0 +1,67 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/replacement/replacement.cfg + +# +# Description: +# Verify that on import an in progress attach operation is resumed. +# +# Strategy: +# 1. For both healing and sequential resilvering. +# a. Create a pool +# b. Add a vdev with 'zpool attach' and resilver (-s) it. +# c. Export the pool +# d. Import the pool +# e. Verify the 'zpool attach' resumed resilvering +# f. Destroy the pool +# + +function cleanup +{ + log_must set_tunable32 SCAN_SUSPEND_PROGRESS \ + $ORIG_SCAN_SUSPEND_PROGRESS + destroy_pool $TESTPOOL1 + rm -f ${VDEV_FILES[@]} +} + +log_assert "Verify attach is resumed on import" + +ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS) + +log_onexit cleanup + +log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} + +# Verify healing and sequential resilver resume on import. +for arg in "" "-s"; do + log_must zpool create -f $TESTPOOL1 ${VDEV_FILES[0]} + log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 + log_must zpool attach $arg $TESTPOOL1 ${VDEV_FILES[0]} ${VDEV_FILES[1]} + log_must is_pool_resilvering $TESTPOOL1 + log_must zpool export $TESTPOOL1 + log_must zpool import -d $TEST_BASE_DIR $TESTPOOL1 + log_must is_pool_resilvering $TESTPOOL1 + log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS + log_must zpool wait -t resilver $TESTPOOL1 + log_must is_pool_resilvered $TESTPOOL1 + destroy_pool $TESTPOOL1 +done + +log_pass "Verify attach is resumed on import" diff --git a/tests/zfs-tests/tests/functional/replacement/attach_multiple.ksh b/tests/zfs-tests/tests/functional/replacement/attach_multiple.ksh new file mode 100755 index 000000000..b3192b2bf --- /dev/null +++ b/tests/zfs-tests/tests/functional/replacement/attach_multiple.ksh @@ -0,0 +1,111 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/replacement/replacement.cfg + +# +# Description: +# Verify that attach/detach work while resilvering and attaching +# multiple vdevs. +# +# Strategy: +# 1. Create a single vdev pool +# 2. While healing or sequential resilvering: +# a. Attach a vdev to convert the pool to a mirror. +# b. Attach a vdev to convert the pool to a 3-way mirror. +# c. Verify the original vdev cannot be removed (no redundant copies) +# d. Detach a vdev. Healing and sequential resilver remain running. +# e. Detach a vdev. Healing resilver remains running, sequential +# resilver is canceled. +# f. Wait for resilver to complete. +# + +function cleanup +{ + log_must set_tunable32 SCAN_SUSPEND_PROGRESS \ + $ORIG_SCAN_SUSPEND_PROGRESS + destroy_pool $TESTPOOL1 + rm -f ${VDEV_FILES[@]} +} + +log_assert "Verify attach/detech with multiple vdevs" + +ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS) + +log_onexit cleanup + +log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} + +# Verify resilver resumes on import. +log_must zpool create -f $TESTPOOL1 ${VDEV_FILES[0]} + +for replace_mode in "healing" "sequential"; do + # + # Resilvers abort the dsl_scan and reconfigure it for resilvering. + # Rebuilds cancel the dsl_scan and start the vdev_rebuild thread. + # + if [[ "$replace_mode" = "healing" ]]; then + flags="" + else + flags="-s" + fi + + log_mustnot is_pool_resilvering $TESTPOOL1 + log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 + + # Attach first vdev (stripe -> mirror) + log_must zpool attach $flags $TESTPOOL1 \ + ${VDEV_FILES[0]} ${VDEV_FILES[1]} + log_must is_pool_resilvering $TESTPOOL1 + + # Attach second vdev (2-way -> 3-way mirror) + log_must zpool attach $flags $TESTPOOL1 \ + ${VDEV_FILES[1]} ${VDEV_FILES[2]} + log_must is_pool_resilvering $TESTPOOL1 + + # Original vdev cannot be detached until there is sufficent redundancy. + log_mustnot zpool detach $TESTPOOL1 ${VDEV_FILES[0]} + + # Detach first vdev (resilver keeps running) + log_must zpool detach $TESTPOOL1 ${VDEV_FILES[1]} + log_must is_pool_resilvering $TESTPOOL1 + + # + # Detach second vdev. There's a difference in behavior between + # healing and sequential resilvers. A healing resilver will not be + # cancelled even though there's nothing on the original vdev which + # needs to be rebuilt. A sequential resilver on the otherhand is + # canceled when returning to a non-redundant striped layout. At + # some point the healing resilver behavior should be updated to match + # the sequential resilver behavior. + # + log_must zpool detach $TESTPOOL1 ${VDEV_FILES[2]} + + if [[ "$replace_mode" = "healing" ]]; then + log_must is_pool_resilvering $TESTPOOL1 + else + log_mustnot is_pool_resilvering $TESTPOOL1 + fi + + log_must set_tunable32 SCAN_SUSPEND_PROGRESS \ + $ORIG_SCAN_SUSPEND_PROGRESS + log_must zpool wait $TESTPOOL1 +done + +log_pass "Verify attach/detech with multiple vdevs" diff --git a/tests/zfs-tests/tests/functional/replacement/attach_rebuild.ksh b/tests/zfs-tests/tests/functional/replacement/attach_rebuild.ksh new file mode 100755 index 000000000..e9427c7ad --- /dev/null +++ b/tests/zfs-tests/tests/functional/replacement/attach_rebuild.ksh @@ -0,0 +1,173 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/replacement/replacement.cfg + +# +# DESCRIPTION: +# Attaching disks during I/O should pass for supported pools. +# +# STRATEGY: +# 1. Create multidisk pools (stripe/mirror/raidz) and +# start some random I/O +# 2. Attach a disk to the pool. +# 3. Verify the integrity of the file system and the resilvering. +# +# NOTE: Raidz does not support the sequential resilver (-s) option. +# + +verify_runnable "global" + +function cleanup +{ + if [[ -n "$child_pids" ]]; then + for wait_pid in $child_pids; do + kill $wait_pid + done + fi + + if poolexists $TESTPOOL1; then + destroy_pool $TESTPOOL1 + fi + + [[ -e $TESTDIR ]] && log_must rm -rf $TESTDIR/* +} + +log_assert "Replacing a disk during I/O completes." + +options="" +options_display="default options" + +log_onexit cleanup + +[[ -n "$HOLES_FILESIZE" ]] && options=" $options -f $HOLES_FILESIZE " + +[[ -n "$HOLES_BLKSIZE" ]] && options="$options -b $HOLES_BLKSIZE " + +[[ -n "$HOLES_COUNT" ]] && options="$options -c $HOLES_COUNT " + +[[ -n "$HOLES_SEED" ]] && options="$options -s $HOLES_SEED " + +[[ -n "$HOLES_FILEOFFSET" ]] && options="$options -o $HOLES_FILEOFFSET " + +options="$options -r " + +[[ -n "$options" ]] && options_display=$options + +child_pids="" + +function attach_test +{ + typeset -i iters=2 + typeset -i index=0 + typeset opt=$1 + typeset disk1=$2 + typeset disk2=$3 + + typeset i=0 + while [[ $i -lt $iters ]]; do + log_note "Invoking file_trunc with: $options_display" + file_trunc $options $TESTDIR/$TESTFILE.$i & + typeset pid=$! + + sleep 1 + + child_pids="$child_pids $pid" + ((i = i + 1)) + done + + log_must zpool attach -sw $opt $TESTPOOL1 $disk1 $disk2 + + for wait_pid in $child_pids; do + kill $wait_pid + done + child_pids="" + + log_must zpool export $TESTPOOL1 + log_must zpool import -d $TESTDIR $TESTPOOL1 + log_must zfs umount $TESTPOOL1/$TESTFS1 + log_must zdb -cdui $TESTPOOL1/$TESTFS1 + log_must zfs mount $TESTPOOL1/$TESTFS1 + verify_pool $TESTPOOL1 +} + +specials_list="" +i=0 +while [[ $i != 3 ]]; do + truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i + specials_list="$specials_list $TESTDIR/$TESTFILE1.$i" + + ((i = i + 1)) +done + +# +# Create a replacement disk special file. +# +truncate -s $MINVDEVSIZE $TESTDIR/$REPLACEFILE + +for op in "" "-f"; do + create_pool $TESTPOOL1 mirror $specials_list + log_must zfs create $TESTPOOL1/$TESTFS1 + log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1 + + attach_test "$opt" $TESTDIR/$TESTFILE1.1 $TESTDIR/$REPLACEFILE + + zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE" + if [[ $? -ne 0 ]]; then + log_fail "$REPLACEFILE is not present." + fi + + destroy_pool $TESTPOOL1 +done + +log_note "Verify 'zpool attach' fails with non-mirrors." + +for type in "" "raidz" "raidz1"; do + for op in "" "-f"; do + create_pool $TESTPOOL1 $type $specials_list + log_must zfs create $TESTPOOL1/$TESTFS1 + log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1 + + log_mustnot zpool attach -s "$opt" $TESTDIR/$TESTFILE1.1 \ + $TESTDIR/$REPLACEFILE + + zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE" + if [[ $? -eq 0 ]]; then + log_fail "$REPLACEFILE should not be present." + fi + + destroy_pool $TESTPOOL1 + done +done + +log_pass diff --git a/tests/zfs-tests/tests/functional/replacement/replacement_002_pos.ksh b/tests/zfs-tests/tests/functional/replacement/attach_resilver.ksh index 391aa5cf0..4261d4d67 100755 --- a/tests/zfs-tests/tests/functional/replacement/replacement_002_pos.ksh +++ b/tests/zfs-tests/tests/functional/replacement/attach_resilver.ksh @@ -104,9 +104,7 @@ function attach_test ((i = i + 1)) done - log_must zpool attach $opt $TESTPOOL1 $disk1 $disk2 - - sleep 10 + log_must zpool attach -w $opt $TESTPOOL1 $disk1 $disk2 for wait_pid in $child_pids do @@ -119,13 +117,13 @@ function attach_test log_must zfs umount $TESTPOOL1/$TESTFS1 log_must zdb -cdui $TESTPOOL1/$TESTFS1 log_must zfs mount $TESTPOOL1/$TESTFS1 - + verify_pool $TESTPOOL1 } specials_list="" i=0 -while [[ $i != 2 ]]; do - mkfile $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i +while [[ $i != 3 ]]; do + truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i specials_list="$specials_list $TESTDIR/$TESTFILE1.$i" ((i = i + 1)) @@ -134,7 +132,7 @@ done # # Create a replacement disk special file. # -mkfile $MINVDEVSIZE $TESTDIR/$REPLACEFILE +truncate -s $MINVDEVSIZE $TESTDIR/$REPLACEFILE for op in "" "-f"; do create_pool $TESTPOOL1 mirror $specials_list @@ -143,7 +141,7 @@ for op in "" "-f"; do attach_test "$opt" $TESTDIR/$TESTFILE1.1 $TESTDIR/$REPLACEFILE - zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$REPLACEFILE" + zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE" if [[ $? -ne 0 ]]; then log_fail "$REPLACEFILE is not present." fi @@ -162,7 +160,7 @@ for type in "" "raidz" "raidz1"; do log_mustnot zpool attach "$opt" $TESTDIR/$TESTFILE1.1 \ $TESTDIR/$REPLACEFILE - zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$REPLACEFILE" + zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE" if [[ $? -eq 0 ]]; then log_fail "$REPLACEFILE should not be present." fi diff --git a/tests/zfs-tests/tests/functional/replacement/replacement_003_pos.ksh b/tests/zfs-tests/tests/functional/replacement/detach.ksh index 71b9602ee..aa3ec4f7a 100755 --- a/tests/zfs-tests/tests/functional/replacement/replacement_003_pos.ksh +++ b/tests/zfs-tests/tests/functional/replacement/detach.ksh @@ -121,8 +121,8 @@ function detach_test specials_list="" i=0 -while [[ $i != 2 ]]; do - mkfile $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i +while [[ $i != 3 ]]; do + truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i specials_list="$specials_list $TESTDIR/$TESTFILE1.$i" ((i = i + 1)) @@ -134,7 +134,7 @@ log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1 detach_test $TESTDIR/$TESTFILE1.1 -zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$TESTFILE1.1" +zpool iostat -v $TESTPOOL1 | grep "$TESTFILE1.1" if [[ $? -eq 0 ]]; then log_fail "$TESTFILE1.1 should no longer be present." fi @@ -143,14 +143,14 @@ destroy_pool $TESTPOOL1 log_note "Verify 'zpool detach' fails with non-mirrors." -for type in "" "raidz" "raidz1" ; do +for type in "" "raidz" "raidz1"; do create_pool $TESTPOOL1 $type $specials_list log_must zfs create $TESTPOOL1/$TESTFS1 log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1 log_mustnot zpool detach $TESTDIR/$TESTFILE1.1 - zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$TESTFILE1.1" + zpool iostat -v $TESTPOOL1 | grep "$TESTFILE1.1" if [[ $? -ne 0 ]]; then log_fail "$TESTFILE1.1 is not present." fi diff --git a/tests/zfs-tests/tests/functional/replacement/rebuild_disabled_feature.ksh b/tests/zfs-tests/tests/functional/replacement/rebuild_disabled_feature.ksh new file mode 100755 index 000000000..d17d83b78 --- /dev/null +++ b/tests/zfs-tests/tests/functional/replacement/rebuild_disabled_feature.ksh @@ -0,0 +1,78 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/replacement/replacement.cfg + +# +# Description: +# Verify device_rebuild feature flags. +# +# Strategy: +# 1. Create a pool with all features disabled. +# 2. Verify 'zpool replace -s' fails and the feature is disabled. +# 3. Enable the device_rebuild feature. +# 4. Verify 'zpool replace -s' works and the feature is active. +# 5. Wait for the feature to return to enabled. +# + +function cleanup +{ + log_must set_tunable32 SCAN_SUSPEND_PROGRESS \ + $ORIG_SCAN_SUSPEND_PROGRESS + destroy_pool $TESTPOOL1 + rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE +} + +function check_feature_flag +{ + feature=$1 + pool=$2 + expected_value=$3 + + value="$(zpool get -H -o property,value all $pool | \ + egrep "$feature" | awk '{print $2}')" + if [ "$value" = "$expected_value" ]; then + log_note "$feature verified to be $value" + else + log_fail "$feature should be $expected_value but is $value" + fi +} + +log_assert "Verify device_rebuild feature flags." + +ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS) + +log_onexit cleanup + +log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE +log_must zpool create -d $TESTPOOL1 ${VDEV_FILES[@]} + +log_mustnot zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE +check_feature_flag "feature@device_rebuild" "$TESTPOOL1" "disabled" + +log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 +log_must zpool set feature@device_rebuild=enabled $TESTPOOL1 +log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE +check_feature_flag "feature@device_rebuild" "$TESTPOOL1" "active" + +log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS +log_must zpool wait -t resilver $TESTPOOL1 +check_feature_flag "feature@device_rebuild" "$TESTPOOL1" "enabled" + +log_pass "Verify device_rebuild feature flags." diff --git a/tests/zfs-tests/tests/functional/replacement/rebuild_multiple.ksh b/tests/zfs-tests/tests/functional/replacement/rebuild_multiple.ksh new file mode 100755 index 000000000..7775cbff4 --- /dev/null +++ b/tests/zfs-tests/tests/functional/replacement/rebuild_multiple.ksh @@ -0,0 +1,126 @@ +#!/bin/ksh -p + +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/replacement/replacement.cfg + +# +# DESCRIPTION: +# Sequential reconstruction (unlike healing reconstruction) operate on the +# top-level vdev. This means that a sequential resilver operation can be +# started/stopped on a different top-level vdev without impacting other +# sequential resilvers. +# +# STRATEGY: +# 1. Create a mirrored pool. +# + +function cleanup +{ + log_must set_tunable32 SCAN_SUSPEND_PROGRESS \ + $ORIG_SCAN_SUSPEND_PROGRESS + destroy_pool $TESTPOOL1 + rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE $SPARE_VDEV_FILE2 +} + +function check_history +{ + pool=$1 + msg=$2 + exp=$3 + + count=$(zpool history -i $pool | grep "rebuild" | grep -c "$msg") + if [[ "$count" -ne "$exp" ]]; then + log_fail "Expected $exp rebuild '$msg' messages, found $count" + else + log_note "Found $count/$exp rebuild '$msg' messages" + fi +} + +log_assert "Rebuilds operate on the top-level vdevs" + +ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS) + +log_onexit cleanup + +log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} \ + $SPARE_VDEV_FILE $SPARE_VDEV_FILE2 + +# Verify two sequential resilvers can run concurrently. +log_must zpool create -f $TESTPOOL1 \ + mirror ${VDEV_FILES[0]} ${VDEV_FILES[1]} \ + mirror ${VDEV_FILES[2]} ${VDEV_FILES[3]} +log_must zfs create $TESTPOOL1/$TESTFS + +mntpnt=$(get_prop mountpoint $TESTPOOL1/$TESTFS) +log_must dd if=/dev/urandom of=$mntpnt/file bs=1M count=32 +log_must zpool sync $TESTPOOL1 + +log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 + +log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE +log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[3]} $SPARE_VDEV_FILE2 + +check_history $TESTPOOL1 "started" 2 +check_history $TESTPOOL1 "reset" 0 +check_history $TESTPOOL1 "complete" 0 +check_history $TESTPOOL1 "canceled" 0 + +log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS +log_must zpool wait -t resilver $TESTPOOL1 + +check_history $TESTPOOL1 "complete" 2 +destroy_pool $TESTPOOL1 + +# Verify canceling one resilver (zpool detach) does not impact others. +log_must zpool create -f $TESTPOOL1 \ + mirror ${VDEV_FILES[0]} ${VDEV_FILES[1]} \ + mirror ${VDEV_FILES[2]} ${VDEV_FILES[3]} +log_must zfs create $TESTPOOL1/$TESTFS + +mntpnt=$(get_prop mountpoint $TESTPOOL1/$TESTFS) +log_must dd if=/dev/urandom of=$mntpnt/file bs=1M count=32 +log_must zpool sync $TESTPOOL1 + +log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 + +log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE +log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[3]} $SPARE_VDEV_FILE2 + +check_history $TESTPOOL1 "started" 2 +check_history $TESTPOOL1 "reset" 0 +check_history $TESTPOOL1 "complete" 0 +check_history $TESTPOOL1 "canceled" 0 + +log_must zpool detach $TESTPOOL1 $SPARE_VDEV_FILE2 + +check_history $TESTPOOL1 "complete" 0 +check_history $TESTPOOL1 "canceled" 1 + +log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS +log_must zpool wait -t resilver $TESTPOOL1 + +check_history $TESTPOOL1 "complete" 1 +check_history $TESTPOOL1 "canceled" 1 +destroy_pool $TESTPOOL1 + +log_pass "Rebuilds operate on the top-level vdevs" diff --git a/tests/zfs-tests/tests/functional/replacement/rebuild_raidz.ksh b/tests/zfs-tests/tests/functional/replacement/rebuild_raidz.ksh new file mode 100755 index 000000000..c919b44b2 --- /dev/null +++ b/tests/zfs-tests/tests/functional/replacement/rebuild_raidz.ksh @@ -0,0 +1,70 @@ +#!/bin/ksh -p + +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/replacement/replacement.cfg + +# +# DESCRIPTION: +# Executing 'zpool replace -s' for raidz vdevs failed. Sequential +# resilvers are only allowed for stripe/mirror pools. +# +# STRATEGY: +# 1. Create a raidz pool, verify 'zpool replace -s' fails +# 2. Create a stripe/mirror pool, verify 'zpool replace -s' passes +# + +function cleanup +{ + log_must set_tunable32 SCAN_SUSPEND_PROGRESS \ + $ORIG_SCAN_SUSPEND_PROGRESS + destroy_pool $TESTPOOL1 + rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE +} + +log_assert "Sequential resilver is not allowed for raidz vdevs" + +ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS) + +log_onexit cleanup + +log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE + +# raidz[1-3] +for vdev_type in "raidz" "raidz2" "raidz3"; do + log_must zpool create -f $TESTPOOL1 $vdev_type ${VDEV_FILES[@]} + log_mustnot zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} \ + $SPARE_VDEV_FILE + destroy_pool $TESTPOOL1 +done + +# stripe +log_must zpool create $TESTPOOL1 ${VDEV_FILES[@]} +log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE +destroy_pool $TESTPOOL1 + +# mirror +log_must zpool create $TESTPOOL1 mirror ${VDEV_FILES[0]} ${VDEV_FILES[1]} +log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[1]} $SPARE_VDEV_FILE +destroy_pool $TESTPOOL1 + +log_pass "Sequential resilver is not allowed for raidz vdevs" diff --git a/tests/zfs-tests/tests/functional/replacement/replace_import.ksh b/tests/zfs-tests/tests/functional/replacement/replace_import.ksh new file mode 100755 index 000000000..35d51d939 --- /dev/null +++ b/tests/zfs-tests/tests/functional/replacement/replace_import.ksh @@ -0,0 +1,67 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/replacement/replacement.cfg + +# +# Description: +# Verify that on import an in progress replace operation is resumed. +# +# Strategy: +# 1. For both healing and sequential resilvering replace: +# a. Create a pool +# b. Repalce a vdev with 'zpool replace' to resilver (-s) it. +# c. Export the pool +# d. Import the pool +# e. Verify the 'zpool replace' resumed resilvering. +# f. Destroy the pool +# + +function cleanup +{ + log_must set_tunable32 SCAN_SUSPEND_PROGRESS \ + $ORIG_SCAN_SUSPEND_PROGRESS + destroy_pool $TESTPOOL1 + rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE +} + +log_assert "Verify replace is resumed on import" + +ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS) + +log_onexit cleanup + +log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE + +# Verify healing and sequential resilver resume on import. +for arg in "" "-s"; do + log_must zpool create -f $TESTPOOL1 ${VDEV_FILES[@]} + log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 + log_must zpool replace -s $TESTPOOL1 ${VDEV_FILES[0]} $SPARE_VDEV_FILE + log_must is_pool_resilvering $TESTPOOL1 + log_must zpool export $TESTPOOL1 + log_must zpool import -d $TEST_BASE_DIR $TESTPOOL1 + log_must is_pool_resilvering $TESTPOOL1 + log_must set_tunable32 SCAN_SUSPEND_PROGRESS $ORIG_SCAN_SUSPEND_PROGRESS + log_must zpool wait -t resilver $TESTPOOL1 + log_must is_pool_resilvered $TESTPOOL1 + destroy_pool $TESTPOOL1 +done + +log_pass "Verify replace is resumed on import" diff --git a/tests/zfs-tests/tests/functional/replacement/replace_rebuild.ksh b/tests/zfs-tests/tests/functional/replacement/replace_rebuild.ksh new file mode 100755 index 000000000..599735228 --- /dev/null +++ b/tests/zfs-tests/tests/functional/replacement/replace_rebuild.ksh @@ -0,0 +1,158 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/replacement/replacement.cfg + +# +# DESCRIPTION: +# Replacing disks during I/O should pass for supported pools. +# +# STRATEGY: +# 1. Create multidisk pools (stripe/mirror) and +# start some random I/O +# 2. Replace a disk in the pool with another disk. +# 3. Verify the integrity of the file system and the rebuilding. +# +# NOTE: Raidz does not support the sequential resilver (-s) option. +# + +verify_runnable "global" + +function cleanup +{ + if [[ -n "$child_pids" ]]; then + for wait_pid in $child_pids + do + kill $wait_pid + done + fi + + if poolexists $TESTPOOL1; then + destroy_pool $TESTPOOL1 + fi + + [[ -e $TESTDIR ]] && log_must rm -rf $TESTDIR/* +} + +log_assert "Replacing a disk with -r during I/O completes." + +options="" +options_display="default options" + +log_onexit cleanup + +[[ -n "$HOLES_FILESIZE" ]] && options=" $options -f $HOLES_FILESIZE " + +[[ -n "$HOLES_BLKSIZE" ]] && options="$options -b $HOLES_BLKSIZE " + +[[ -n "$HOLES_COUNT" ]] && options="$options -c $HOLES_COUNT " + +[[ -n "$HOLES_SEED" ]] && options="$options -s $HOLES_SEED " + +[[ -n "$HOLES_FILEOFFSET" ]] && options="$options -o $HOLES_FILEOFFSET " + +options="$options -r " + +[[ -n "$options" ]] && options_display=$options + +child_pids="" + +function replace_test +{ + typeset -i iters=2 + typeset -i index=0 + typeset opt=$1 + typeset disk1=$2 + typeset disk2=$3 + + typeset i=0 + while [[ $i -lt $iters ]]; do + log_note "Invoking file_trunc with: $options_display" + file_trunc $options $TESTDIR/$TESTFILE.$i & + typeset pid=$! + + sleep 1 + + child_pids="$child_pids $pid" + ((i = i + 1)) + done + + log_must zpool replace -sw $opt $TESTPOOL1 $disk1 $disk2 + + for wait_pid in $child_pids + do + kill $wait_pid + done + child_pids="" + + log_must zpool export $TESTPOOL1 + log_must zpool import -d $TESTDIR $TESTPOOL1 + log_must zfs umount $TESTPOOL1/$TESTFS1 + log_must zdb -cdui $TESTPOOL1/$TESTFS1 + log_must zfs mount $TESTPOOL1/$TESTFS1 + verify_pool $TESTPOOL1 +} + +specials_list="" +i=0 +while [[ $i != 3 ]]; do + log_must truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i + specials_list="$specials_list $TESTDIR/$TESTFILE1.$i" + + ((i = i + 1)) +done + +# +# Create a replacement disk special file. +# +log_must truncate -s $MINVDEVSIZE $TESTDIR/$REPLACEFILE + +for type in "" "mirror"; do + for op in "" "-f"; do + create_pool $TESTPOOL1 $type $specials_list + log_must zfs create $TESTPOOL1/$TESTFS1 + log_must zfs set mountpoint=$TESTDIR1 $TESTPOOL1/$TESTFS1 + + replace_test "$opt" $TESTDIR/$TESTFILE1.1 $TESTDIR/$REPLACEFILE + + zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE" + if [[ $? -ne 0 ]]; then + log_fail "$REPLACEFILE is not present." + fi + + destroy_pool $TESTPOOL1 + log_must rm -rf /$TESTPOOL1 + done +done + +log_pass diff --git a/tests/zfs-tests/tests/functional/replacement/replacement_001_pos.ksh b/tests/zfs-tests/tests/functional/replacement/replace_resilver.ksh index 8f40436ff..253cf65e4 100755 --- a/tests/zfs-tests/tests/functional/replacement/replacement_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/replacement/replace_resilver.ksh @@ -104,9 +104,7 @@ function replace_test ((i = i + 1)) done - log_must zpool replace $opt $TESTPOOL1 $disk1 $disk2 - - sleep 10 + log_must zpool replace -w $opt $TESTPOOL1 $disk1 $disk2 for wait_pid in $child_pids do @@ -119,11 +117,12 @@ function replace_test log_must zfs umount $TESTPOOL1/$TESTFS1 log_must zdb -cdui $TESTPOOL1/$TESTFS1 log_must zfs mount $TESTPOOL1/$TESTFS1 + verify_pool $TESTPOOL1 } specials_list="" i=0 -while [[ $i != 2 ]]; do +while [[ $i != 3 ]]; do log_must truncate -s $MINVDEVSIZE $TESTDIR/$TESTFILE1.$i specials_list="$specials_list $TESTDIR/$TESTFILE1.$i" @@ -143,7 +142,7 @@ for type in "" "raidz" "mirror"; do replace_test "$opt" $TESTDIR/$TESTFILE1.1 $TESTDIR/$REPLACEFILE - zpool iostat -v $TESTPOOL1 | grep "$TESTDIR/$REPLACEFILE" + zpool iostat -v $TESTPOOL1 | grep "$REPLACEFILE" if [[ $? -ne 0 ]]; then log_fail "$REPLACEFILE is not present." fi diff --git a/tests/zfs-tests/tests/functional/replacement/replacement.cfg b/tests/zfs-tests/tests/functional/replacement/replacement.cfg index b2ba1b885..271317b1c 100644 --- a/tests/zfs-tests/tests/functional/replacement/replacement.cfg +++ b/tests/zfs-tests/tests/functional/replacement/replacement.cfg @@ -36,3 +36,8 @@ export HOLES_SEED=${HOLES_SEED-""} export HOLES_FILEOFFSET=${HOLES_FILEOFFSET-""} export HOLES_COUNT=${HOLES_COUNT-"16384"} # FILESIZE/BLKSIZE/8 export REPLACEFILE="sparedisk" + +set -A VDEV_FILES $TEST_BASE_DIR/file-{1..4} +export VDEV_FILE_SIZE=$(( $SPA_MINDEVSIZE * 2 )) +export SPARE_VDEV_FILE=$TEST_BASE_DIR/spare-1 +export SPARE_VDEV_FILE2=$TEST_BASE_DIR/spare-2 diff --git a/tests/zfs-tests/tests/functional/resilver/resilver_restart_001.ksh b/tests/zfs-tests/tests/functional/replacement/resilver_restart_001.ksh index 9af1c972f..7896b2dbe 100755 --- a/tests/zfs-tests/tests/functional/resilver/resilver_restart_001.ksh +++ b/tests/zfs-tests/tests/functional/replacement/resilver_restart_001.ksh @@ -20,7 +20,7 @@ # . $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/resilver/resilver.cfg +. $STF_SUITE/tests/functional/replacement/replacement.cfg # # DESCRIPTION: @@ -50,7 +50,7 @@ function cleanup $ORIG_SCAN_SUSPEND_PROGRESS log_must set_tunable32 ZEVENT_LEN_MAX $ORIG_ZFS_ZEVENT_LEN_MAX log_must zinject -c all - destroy_pool $TESTPOOL + destroy_pool $TESTPOOL1 rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE } @@ -70,7 +70,7 @@ function verify_restarts # <msg> <cnt> <defer> [[ -z "$defer" ]] && return # use zdb to find which vdevs have the resilver defer flag - VDEV_DEFERS=$(zdb -C $TESTPOOL | awk ' + VDEV_DEFERS=$(zdb -C $TESTPOOL1 | awk ' /children/ { gsub(/[^0-9]/, ""); child = $0 } /com\.datto:resilver_defer$/ { print child } ') @@ -106,17 +106,17 @@ log_must set_tunable32 ZEVENT_LEN_MAX 512 log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE -log_must zpool create -f -o feature@resilver_defer=disabled $TESTPOOL \ +log_must zpool create -f -o feature@resilver_defer=disabled $TESTPOOL1 \ raidz ${VDEV_FILES[@]} # create 4 filesystems for fs in fs{0..3} do - log_must zfs create -o primarycache=none -o recordsize=1k $TESTPOOL/$fs + log_must zfs create -o primarycache=none -o recordsize=1k $TESTPOOL1/$fs done # simultaneously write 16M to each of them -set -A DATAPATHS /$TESTPOOL/fs{0..3}/dat.0 +set -A DATAPATHS /$TESTPOOL1/fs{0..3}/dat.0 log_note "Writing data files" for path in ${DATAPATHS[@]} do @@ -131,7 +131,7 @@ do if [[ $test == "with" ]] then - log_must zpool set feature@resilver_defer=enabled $TESTPOOL + log_must zpool set feature@resilver_defer=enabled $TESTPOOL1 RESTARTS=( "${DEFER_RESTARTS[@]}" ) VDEVS=( "${DEFER_VDEVS[@]}" ) VDEV_REPLACE="$SPARE_VDEV_FILE ${VDEV_FILES[1]}" @@ -144,7 +144,7 @@ do log_must set_tunable32 RESILVER_MIN_TIME_MS 50 # initiate a resilver and suspend the scan as soon as possible - log_must zpool replace $TESTPOOL $VDEV_REPLACE + log_must zpool replace $TESTPOOL1 $VDEV_REPLACE log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 # there should only be 1 resilver start @@ -152,16 +152,16 @@ do # offline then online a vdev to introduce a new DTL range after current # scan, which should restart (or defer) the resilver - log_must zpool offline $TESTPOOL ${VDEV_FILES[2]} - log_must zpool sync $TESTPOOL - log_must zpool online $TESTPOOL ${VDEV_FILES[2]} - log_must zpool sync $TESTPOOL + log_must zpool offline $TESTPOOL1 ${VDEV_FILES[2]} + log_must zpool sync $TESTPOOL1 + log_must zpool online $TESTPOOL1 ${VDEV_FILES[2]} + log_must zpool sync $TESTPOOL1 # there should now be 2 resilver starts w/o defer, 1 with defer verify_restarts ' after offline/online' "${RESTARTS[1]}" "${VDEVS[1]}" # inject read io errors on vdev and verify resilver does not restart - log_must zinject -a -d ${VDEV_FILES[2]} -e io -T read -f 0.25 $TESTPOOL + log_must zinject -a -d ${VDEV_FILES[2]} -e io -T read -f 0.25 $TESTPOOL1 log_must cat ${DATAPATHS[1]} > /dev/null log_must zinject -c all @@ -173,17 +173,12 @@ do log_must set_tunable32 RESILVER_MIN_TIME_MS 3000 # wait for resilver to finish - for iter in {0..59} - do - is_pool_resilvered $TESTPOOL && break - sleep 1 - done - is_pool_resilvered $TESTPOOL || - log_fail "resilver timed out" + log_must zpool wait -t resilver $TESTPOOL1 + log_must is_pool_resilvered $TESTPOOL1 # wait for a few txg's to see if a resilver happens - log_must zpool sync $TESTPOOL - log_must zpool sync $TESTPOOL + log_must zpool sync $TESTPOOL1 + log_must zpool sync $TESTPOOL1 # there should now be 2 resilver starts verify_restarts ' after resilver' "${RESTARTS[3]}" "${VDEVS[3]}" diff --git a/tests/zfs-tests/tests/functional/resilver/resilver_restart_002.ksh b/tests/zfs-tests/tests/functional/replacement/resilver_restart_002.ksh index ebe5e693b..48763f9b2 100755 --- a/tests/zfs-tests/tests/functional/resilver/resilver_restart_002.ksh +++ b/tests/zfs-tests/tests/functional/replacement/resilver_restart_002.ksh @@ -20,7 +20,7 @@ # . $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/resilver/resilver.cfg +. $STF_SUITE/tests/functional/replacement/replacement.cfg # # DESCRIPTION: @@ -40,7 +40,7 @@ function cleanup { log_must zinject -c all - destroy_pool $TESTPOOL + destroy_pool $TESTPOOL1 rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE log_must set_tunable32 SCAN_LEGACY $ORIG_SCAN_LEGACY } @@ -56,25 +56,25 @@ log_must set_tunable32 SCAN_LEGACY 1 # create the pool and a 32M file (32k blocks) log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[0]} $SPARE_VDEV_FILE -log_must zpool create -f -O recordsize=1k $TESTPOOL ${VDEV_FILES[0]} -log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=32 > /dev/null 2>&1 +log_must zpool create -f -O recordsize=1k $TESTPOOL1 ${VDEV_FILES[0]} +log_must dd if=/dev/urandom of=/$TESTPOOL1/file bs=1M count=32 > /dev/null 2>&1 # determine objset/object -objset=$(zdb -d $TESTPOOL/ | sed -ne 's/.*ID \([0-9]*\).*/\1/p') -object=$(ls -i /$TESTPOOL/file | awk '{print $1}') +objset=$(zdb -d $TESTPOOL1/ | sed -ne 's/.*ID \([0-9]*\).*/\1/p') +object=$(ls -i /$TESTPOOL1/file | awk '{print $1}') # inject event to cause error during resilver -log_must zinject -b `printf "%x:%x:0:3fff" $objset $object` $TESTPOOL +log_must zinject -b `printf "%x:%x:0:3fff" $objset $object` $TESTPOOL1 # clear events and start resilver log_must zpool events -c -log_must zpool attach $TESTPOOL ${VDEV_FILES[0]} $SPARE_VDEV_FILE +log_must zpool attach $TESTPOOL1 ${VDEV_FILES[0]} $SPARE_VDEV_FILE log_note "waiting for read errors to start showing up" for iter in {0..59} do - zpool sync $TESTPOOL - err=$(zpool status $TESTPOOL | grep ${VDEV_FILES[0]} | awk '{print $3}') + zpool sync $TESTPOOL1 + err=$(zpool status $TESTPOOL1 | grep ${VDEV_FILES[0]} | awk '{print $3}') (( $err > 0 )) && break sleep 1 done @@ -92,8 +92,8 @@ done (( $finish == 0 )) && log_fail "resilver took too long to finish" # wait a few syncs to ensure that zfs does not restart the resilver -log_must zpool sync $TESTPOOL -log_must zpool sync $TESTPOOL +log_must zpool sync $TESTPOOL1 +log_must zpool sync $TESTPOOL1 # check if resilver was restarted start=$(zpool events | grep "sysevent.fs.zfs.resilver_start" | wc -l) diff --git a/tests/zfs-tests/tests/functional/replacement/scrub_cancel.ksh b/tests/zfs-tests/tests/functional/replacement/scrub_cancel.ksh new file mode 100755 index 000000000..da8a0a26e --- /dev/null +++ b/tests/zfs-tests/tests/functional/replacement/scrub_cancel.ksh @@ -0,0 +1,112 @@ +#!/bin/ksh -p + +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2019, Datto Inc. All rights reserved. +# Copyright (c) 2020 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/replacement/replacement.cfg + +# +# DESCRIPTION: +# Verify scrub behaves as intended when contending with a healing or +# sequential resilver. +# +# STRATEGY: +# 1. Create a pool +# 2. Add a modest amount of data to the pool. +# 3. For healing and sequential resilver: +# a. Start scrubbing. +# b. Verify a resilver can be started and it cancels the scrub. +# c. Verify a scrub cannot be started when resilvering +# + +function cleanup +{ + log_must set_tunable32 RESILVER_MIN_TIME_MS $ORIG_RESILVER_MIN_TIME + log_must set_tunable32 SCAN_SUSPEND_PROGRESS \ + $ORIG_SCAN_SUSPEND_PROGRESS + destroy_pool $TESTPOOL1 + rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE +} + +log_assert "Scrub was cancelled by resilver" + +ORIG_RESILVER_MIN_TIME=$(get_tunable RESILVER_MIN_TIME_MS) +ORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS) + +log_onexit cleanup + +log_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE + +log_must zpool create -f $TESTPOOL1 ${VDEV_FILES[@]} +log_must zfs create $TESTPOOL1/$TESTFS + +mntpnt=$(get_prop mountpoint $TESTPOOL1/$TESTFS) +log_must dd if=/dev/urandom of=$mntpnt/file bs=1M count=64 +log_must zpool sync $TESTPOOL1 + +# Request a healing or sequential resilver +for replace_mode in "healing" "sequential"; do + + # + # Healing resilvers abort the dsl_scan and reconfigure it for + # resilvering. Sequential resilvers cancel the dsl_scan and start + # the vdev_rebuild thread. + # + if [[ "$replace_mode" = "healing" ]]; then + history_msg="scan aborted, restarting" + flags="" + else + history_msg="scan cancelled" + flags="-s" + fi + + # Limit scanning time and suspend the scan as soon as possible. + log_must set_tunable32 RESILVER_MIN_TIME_MS 50 + log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 + + # Initiate a scrub. + log_must zpool scrub $TESTPOOL1 + + # Initiate a resilver to cancel the scrub. + log_must zpool replace $flags $TESTPOOL1 ${VDEV_FILES[1]} \ + $SPARE_VDEV_FILE + + # Verify the scrub was canceled, it may take a few seconds to exit. + while is_pool_scrubbing $TESTPOOL1; do + sleep 1 + done + log_mustnot is_pool_scrubbing $TESTPOOL1 + + # Verify a scrub cannot be started while resilvering. + log_must is_pool_resilvering $TESTPOOL1 + log_mustnot zpool scrub $TESTPOOL1 + + # Unsuspend resilver. + log_must set_tunable32 SCAN_SUSPEND_PROGRESS 0 + log_must set_tunable32 RESILVER_MIN_TIME_MS 3000 + + # Wait for resilver to finish then put the original back. + log_must zpool wait $TESTPOOL1 + log_must zpool replace $flags -w $TESTPOOL1 $SPARE_VDEV_FILE \ + ${VDEV_FILES[1]} +done +log_pass "Scrub was cancelled by resilver" + diff --git a/tests/zfs-tests/tests/functional/resilver/Makefile.am b/tests/zfs-tests/tests/functional/resilver/Makefile.am deleted file mode 100644 index 38136a843..000000000 --- a/tests/zfs-tests/tests/functional/resilver/Makefile.am +++ /dev/null @@ -1,9 +0,0 @@ -pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/resilver -dist_pkgdata_SCRIPTS = \ - setup.ksh \ - cleanup.ksh \ - resilver_restart_001.ksh \ - resilver_restart_002.ksh - -dist_pkgdata_DATA = \ - resilver.cfg diff --git a/tests/zfs-tests/tests/functional/resilver/cleanup.ksh b/tests/zfs-tests/tests/functional/resilver/cleanup.ksh deleted file mode 100755 index 4dfa81424..000000000 --- a/tests/zfs-tests/tests/functional/resilver/cleanup.ksh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/ksh -p -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END - -# -# Copyright (c) 2019, Datto Inc. All rights reserved. -# - -. $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/resilver/resilver.cfg - -verify_runnable "global" - -log_pass diff --git a/tests/zfs-tests/tests/functional/resilver/resilver.cfg b/tests/zfs-tests/tests/functional/resilver/resilver.cfg deleted file mode 100644 index 88dfd24ae..000000000 --- a/tests/zfs-tests/tests/functional/resilver/resilver.cfg +++ /dev/null @@ -1,32 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END - -# -# Copyright (c) 2019, Datto Inc. All rights reserved. -# - -. $STF_SUITE/include/libtest.shlib - -verify_runnable "global" - -set -A VDEV_FILES $TEST_BASE_DIR/file-{1..4} -SPARE_VDEV_FILE=$TEST_BASE_DIR/spare-1 - -VDEV_FILE_SIZE=$(( $SPA_MINDEVSIZE * 2 )) diff --git a/tests/zfs-tests/tests/functional/resilver/setup.ksh b/tests/zfs-tests/tests/functional/resilver/setup.ksh deleted file mode 100755 index 4dfa81424..000000000 --- a/tests/zfs-tests/tests/functional/resilver/setup.ksh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/ksh -p -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END - -# -# Copyright (c) 2019, Datto Inc. All rights reserved. -# - -. $STF_SUITE/include/libtest.shlib -. $STF_SUITE/tests/functional/resilver/resilver.cfg - -verify_runnable "global" - -log_pass |