From f55d6ee818763a78328345c5bee9f38d18b852a6 Mon Sep 17 00:00:00 2001 From: Alek P Date: Wed, 15 Mar 2023 10:34:10 -0700 Subject: Improve tests and update man page for healing recv Fix the manpage. The "SYNOPSIS" section is incorrectly formatted for receive -c. I also took this opportunity to reword some parts and fix a run-on sentence in the manpage. Add large block testing for corrective recv. This adds a new test that makes sure blocks generated using zfs send -L/--large-block large-block send flag are able to be used for healing. Since with unloaded key and errlog feature enabled corruption is not shown in zpool status #13675 is fixed the zfs_receive_corrective.ksh test no longer sets -o feature@head_errlog=disabled on pool creation so that it can also test for regressions related to head_errlog feature. Note that the zfs_receive_compressed_corrective.ksh and zfs_receive_large_block_corrective.ksh tests are still creating pools with -o feature@head_errlog=disabled. Reviewed-by: Brian Behlendorf Signed-off-by: Alek Pinchuk Closes #14615 --- tests/runfiles/common.run | 2 +- tests/zfs-tests/tests/Makefile.am | 1 + .../zfs_receive/zfs_receive_corrective.ksh | 2 +- .../zfs_receive_large_block_corrective.ksh | 194 +++++++++++++++++++++ 4 files changed, 197 insertions(+), 2 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh (limited to 'tests') diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index d30bcd804..e4824457e 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -242,7 +242,7 @@ tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos', 'zfs_receive_raw', 'zfs_receive_raw_incremental', 'zfs_receive_-e', 'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props', 'zfs_receive_-wR-encrypted-mix', 'zfs_receive_corrective', - 'zfs_receive_compressed_corrective'] + 'zfs_receive_compressed_corrective', 'zfs_receive_large_block_corrective'] tags = ['functional', 'cli_root', 'zfs_receive'] [tests/functional/cli_root/zfs_rename] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index 4b0a28a6e..9e738227f 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -774,6 +774,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh \ functional/cli_root/zfs_receive/zfs_receive_corrective.ksh \ functional/cli_root/zfs_receive/zfs_receive_compressed_corrective.ksh \ + functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh \ functional/cli_root/zfs_rename/cleanup.ksh \ functional/cli_root/zfs_rename/setup.ksh \ functional/cli_root/zfs_rename/zfs_rename_001_pos.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective.ksh index b2bbdf2a7..9ebde1cd9 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective.ksh @@ -85,7 +85,7 @@ typeset passphrase="password" typeset file="/$TESTPOOL/$TESTFS1/$TESTFILE0" log_must eval "poolexists $TESTPOOL && destroy_pool $TESTPOOL" -log_must zpool create -f -o feature@head_errlog=disabled $TESTPOOL $DISK +log_must zpool create -f $TESTPOOL $DISK log_must eval "echo $passphrase > /$TESTPOOL/pwd" diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh new file mode 100755 index 000000000..fbcd33f60 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh @@ -0,0 +1,194 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2019 Datto, Inc. All rights reserved. +# Copyright (c) 2022 Axcient. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# OpenZFS should be able to heal data using corrective recv when the send file +# was generated with the --large-block flag +# +# STRATEGY: +# 0. Create a file, checksum the file to be corrupted then compare it's checksum +# with the one obtained after healing under different testing scenarios: +# 1. Test healing (aka corrective) recv from a full send file +# 2. Test healing recv (aka heal recv) from an incremental send file +# 3. Test healing recv when compression on-disk is off but source was compressed +# 4. Test heal recv when compression on-disk is on but source was uncompressed +# 5. Test heal recv when compression doesn't match between send file and on-disk +# 6. Test healing recv of an encrypted dataset using an unencrypted send file +# 7. Test healing recv (on an encrypted dataset) using a raw send file +# 8. Test healing when specifying destination filesystem only (no snapshot) +# 9. Test incremental recv aftear healing recv +# + +verify_runnable "both" + +DISK=${DISKS%% *} + +backup=$TEST_BASE_DIR/backup +raw_backup=$TEST_BASE_DIR/raw_backup +ibackup=$TEST_BASE_DIR/ibackup +unc_backup=$TEST_BASE_DIR/unc_backup + +function cleanup +{ + log_must rm -f $backup $raw_backup $ibackup $unc_backup + + poolexists $TESTPOOL && destroy_pool $TESTPOOL + log_must zpool create -f $TESTPOOL $DISK +} + +function test_corrective_recv +{ + log_must zpool scrub -w $TESTPOOL + log_must zpool status -v $TESTPOOL + log_must eval "zpool status -v $TESTPOOL | \ + grep \"Permanent errors have been detected\"" + + # make sure we will read the corruption from disk by flushing the ARC + log_must zinject -a + + log_must eval "zfs recv -c $1 < $2" + + log_must zpool scrub -w $TESTPOOL + log_must zpool status -v $TESTPOOL + log_mustnot eval "zpool status -v $TESTPOOL | \ + grep \"Permanent errors have been detected\"" + typeset cksum=$(md5digest $file) + [[ "$cksum" == "$checksum" ]] || \ + log_fail "Checksums differ ($cksum != $checksum)" +} + +log_onexit cleanup + +log_assert "ZFS corrective receive should be able to heal data corruption" + +typeset passphrase="password" +typeset file="/$TESTPOOL/$TESTFS1/$TESTFILE0" + +log_must eval "poolexists $TESTPOOL && destroy_pool $TESTPOOL" +log_must zpool create -f -o feature@large_blocks=enabled \ + -o feature@head_errlog=disabled $TESTPOOL $DISK + +log_must eval "echo $passphrase > /$TESTPOOL/pwd" + +log_must zfs create -o recordsize=1m -o primarycache=none \ + -o atime=off -o compression=lz4 $TESTPOOL/$TESTFS1 + +log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync +log_must eval "echo 'aaaaaaaa' >> "$file +typeset checksum=$(md5digest $file) + +log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1 + +# create full send file +log_must eval "zfs send -L $TESTPOOL/$TESTFS1@snap1 > $backup" + +log_must dd if=/dev/urandom of=$file"1" bs=1024 count=1024 oflag=sync +log_must eval "echo 'bbbbbbbb' >> "$file"1" +log_must zfs snapshot $TESTPOOL/$TESTFS1@snap2 +# create incremental send file +log_must eval "zfs send -Li $TESTPOOL/$TESTFS1@snap1 \ + $TESTPOOL/$TESTFS1@snap2 > $ibackup" + +corrupt_blocks_at_level $file 0 +# test healing recv from a full send file +test_corrective_recv $TESTPOOL/$TESTFS1@snap1 $backup + +corrupt_blocks_at_level $file"1" 0 +# test healing recv from an incremental send file +test_corrective_recv $TESTPOOL/$TESTFS1@snap2 $ibackup + +# create new uncompressed dataset using our send file +log_must eval "zfs recv -o compression=off -o primarycache=none \ + $TESTPOOL/$TESTFS2 < $backup" +typeset compr=$(get_prop compression $TESTPOOL/$TESTFS2) +[[ "$compr" == "off" ]] || \ + log_fail "Unexpected compression $compr in recved dataset" +corrupt_blocks_at_level "/$TESTPOOL/$TESTFS2/$TESTFILE0" 0 +# test healing recv when compression on-disk is off but source was compressed +test_corrective_recv "$TESTPOOL/$TESTFS2@snap1" $backup + +# create a full sendfile from an uncompressed source +log_must eval "zfs send -L $TESTPOOL/$TESTFS2@snap1 > $unc_backup" +log_must eval "zfs recv -o compression=gzip -o primarycache=none \ + -o recordsize=1m $TESTPOOL/testfs3 < $unc_backup" +typeset compr=$(get_prop compression $TESTPOOL/testfs3) +[[ "$compr" == "gzip" ]] || \ + log_fail "Unexpected compression $compr in recved dataset" +corrupt_blocks_at_level "/$TESTPOOL/testfs3/$TESTFILE0" 0 +# test healing recv when compression on-disk is on but source was uncompressed +test_corrective_recv "$TESTPOOL/testfs3@snap1" $unc_backup + +# create new compressed dataset using our send file +log_must eval "zfs recv -o compression=gzip -o primarycache=none \ + -o recordsize=1m $TESTPOOL/testfs4 < $backup" +typeset compr=$(get_prop compression $TESTPOOL/testfs4) +[[ "$compr" == "gzip" ]] || \ + log_fail "Unexpected compression $compr in recved dataset" +corrupt_blocks_at_level "/$TESTPOOL/testfs4/$TESTFILE0" 0 +# test healing recv when compression doesn't match between send file and on-disk +test_corrective_recv "$TESTPOOL/testfs4@snap1" $backup + +# create new encrypted (and compressed) dataset using our send file +log_must eval "zfs recv -o encryption=aes-256-ccm -o keyformat=passphrase \ + -o recordsize=1m -o keylocation=file:///$TESTPOOL/pwd -o primarycache=none \ + $TESTPOOL/testfs5 < $backup" +typeset encr=$(get_prop encryption $TESTPOOL/testfs5) +[[ "$encr" == "aes-256-ccm" ]] || \ + log_fail "Unexpected encryption $encr in recved dataset" +log_must eval "zfs send -L --raw $TESTPOOL/testfs5@snap1 > $raw_backup" +log_must eval "zfs send -L $TESTPOOL/testfs5@snap1 > $backup" +corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0 +# test healing recv of an encrypted dataset using an unencrypted send file +test_corrective_recv "$TESTPOOL/testfs5@snap1" $backup +corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0 +log_must zfs unmount $TESTPOOL/testfs5 +log_must zfs unload-key $TESTPOOL/testfs5 +# test healing recv (on an encrypted dataset) using a raw send file +test_corrective_recv "$TESTPOOL/testfs5@snap1" $raw_backup +# non raw send file healing an encrypted dataset with an unloaded key will fail +log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap1 < $backup" + +log_must zfs rollback -r $TESTPOOL/$TESTFS1@snap1 +corrupt_blocks_at_level $file 0 +# test healing when specifying destination filesystem only (no snapshot) +test_corrective_recv $TESTPOOL/$TESTFS1 $backup +# test incremental recv aftear healing recv +log_must eval "zfs recv -o recordsize=1m $TESTPOOL/$TESTFS1 < $ibackup" + +# test that healing recv can not be combined with incompatible recv options +log_mustnot eval "zfs recv -h -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -F -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -s -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -u -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -d -c $TESTPOOL/$TESTFS1@snap1 < $backup" +log_mustnot eval "zfs recv -e -c $TESTPOOL/$TESTFS1@snap1 < $backup" + +# ensure healing recv doesn't work when snap GUIDS don't match +log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap2 < $backup" +log_mustnot eval "zfs recv -c $TESTPOOL/testfs5 < $backup" + +# test that healing recv doesn't work on non-existing snapshots +log_mustnot eval "zfs recv -c $TESTPOOL/$TESTFS1@missing < $backup" + +log_pass "OpenZFS corrective recv works for data healing" -- cgit v1.2.3