diff options
| -rw-r--r-- | module/zfs/dmu_recv.c | 20 | ||||
| -rw-r--r-- | tests/runfiles/linux.run | 7 | ||||
| -rw-r--r-- | tests/zfs-tests/tests/functional/rsend/Makefile.am | 2 | ||||
| -rw-r--r-- | tests/zfs-tests/tests/functional/rsend/rsend.kshlib | 17 | ||||
| -rwxr-xr-x | tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh | 43 | ||||
| -rwxr-xr-x | tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh | 118 | ||||
| -rwxr-xr-x | tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh | 112 |
7 files changed, 271 insertions, 48 deletions
diff --git a/module/zfs/dmu_recv.c b/module/zfs/dmu_recv.c index 0db307097..fc5d47f5f 100644 --- a/module/zfs/dmu_recv.c +++ b/module/zfs/dmu_recv.c @@ -1176,6 +1176,7 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, 1ULL << drro->drr_indblkshift : 0; int nblkptr = deduce_nblkptr(drro->drr_bonustype, drro->drr_bonuslen); + boolean_t did_free = B_FALSE; object = drro->drr_object; @@ -1205,6 +1206,8 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, drro->drr_object, 0, DMU_OBJECT_END); if (err != 0) return (SET_ERROR(EINVAL)); + else + did_free = B_TRUE; } /* @@ -1235,11 +1238,15 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, * processed. However, for raw receives we manually set the * maxblkid from the drr_maxblkid and so we must first free * everything above that blkid to ensure the DMU is always - * consistent with itself. + * consistent with itself. We will never free the first block + * of the object here because a maxblkid of 0 could indicate + * an object with a single block or one with no blocks. This + * free may be skipped when dmu_free_long_range() was called + * above since it covers the entire object's contents. */ - if (rwa->raw) { + if (rwa->raw && object != DMU_NEW_OBJECT && !did_free) { err = dmu_free_long_range(rwa->os, drro->drr_object, - (drro->drr_maxblkid + 1) * drro->drr_blksz, + (drro->drr_maxblkid + 1) * doi.doi_data_block_size, DMU_OBJECT_END); if (err != 0) return (SET_ERROR(EINVAL)); @@ -1380,11 +1387,8 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, drro->drr_nlevels, tx)); /* - * Set the maxblkid. We will never free the first block of - * an object here because a maxblkid of 0 could indicate - * an object with a single block or one with no blocks. - * This will always succeed because we freed all blocks - * beyond the new maxblkid above. + * Set the maxblkid. This will always succeed because + * we freed all blocks beyond the new maxblkid above. */ VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object, drro->drr_maxblkid, tx)); diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index f01a36da8..836be089b 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -804,9 +804,10 @@ tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD', 'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize', 'send-c_recv_dedup', 'send_encrypted_files', 'send_encrypted_hierarchy', - 'send_encrypted_props', 'send_freeobjects', 'send_realloc_dnode_size', - 'send_realloc_files', 'send_holds', 'send_hole_birth', 'send_mixed_raw', - 'send-wDR_encrypted_zvol'] + 'send_encrypted_props', 'send_encrypted_truncated_files', + 'send_freeobjects', 'send_realloc_dnode_size', 'send_realloc_files', + 'send_realloc_encrypted_files', 'send_holds', 'send_hole_birth', + 'send_mixed_raw', 'send-wDR_encrypted_zvol'] tags = ['functional', 'rsend'] [tests/functional/scrub_mirror] diff --git a/tests/zfs-tests/tests/functional/rsend/Makefile.am b/tests/zfs-tests/tests/functional/rsend/Makefile.am index 24d4d61cd..8669a51fb 100644 --- a/tests/zfs-tests/tests/functional/rsend/Makefile.am +++ b/tests/zfs-tests/tests/functional/rsend/Makefile.am @@ -24,6 +24,7 @@ dist_pkgdata_SCRIPTS = \ send_encrypted_files.ksh \ send_encrypted_hierarchy.ksh \ send_encrypted_props.ksh \ + send_encrypted_truncated_files.ksh \ send-cD.ksh \ send-c_embedded_blocks.ksh \ send-c_incremental.ksh \ @@ -42,6 +43,7 @@ dist_pkgdata_SCRIPTS = \ send_freeobjects.ksh \ send_realloc_dnode_size.ksh \ send_realloc_files.ksh \ + send_realloc_encrypted_files.ksh \ send_holds.ksh \ send_hole_birth.ksh \ send_mixed_raw.ksh \ diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib index 8d5fc216d..2ef6775e6 100644 --- a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib +++ b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib @@ -464,12 +464,14 @@ function rm_files # $1 Number of files to modify # $2 Maximum file size # $3 File system to modify the file on +# $4 Enabled xattrs (optional) # function churn_files { nfiles=$1 maxsize=$2 fs=$3 + xattrs=${4:-1} # # Remove roughly half of the files in order to make it more @@ -514,7 +516,7 @@ function churn_files # if [[ -e $file_name ]]; then value=$((RANDOM % 5)) - if [ $value -eq 0 ]; then + if [ $value -eq 0 -a $xattrs -ne 0 ]; then attrname="testattr$((RANDOM % 3))" attr -qr $attrname $file_name || \ log_fail "Failed to remove $attrname" @@ -543,11 +545,14 @@ function churn_files bs=$file_size count=1 >/dev/null 2>&1 || \ log_fail "Failed to create $file_name" - for j in {0..2}; do - attrname="testattr$j" - attr -qs $attrname -V TestValue $file_name || \ - log_fail "Failed to set $attrname" - done + if [ $xattrs -ne 0 ]; then + for j in {0..2}; do + attrname="testattr$j" + attr -qs $attrname -V TestValue \ + $file_name || log_fail \ + "Failed to set $attrname" + done + fi fi done diff --git a/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh b/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh index be9d33be9..6288178f8 100755 --- a/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh @@ -22,7 +22,8 @@ # # DESCRIPTION: -# +# Verify that a raw zfs send and receive can deal with several different +# types of file layouts. # # STRATEGY: # 1. Create a new encrypted filesystem @@ -30,18 +31,14 @@ # 3. Add a small 512 byte file to the filesystem # 4. Add a larger 32M file to the filesystem # 5. Add a large sparse file to the filesystem -# 6. Add a 3 files that are to be truncated later -# 7. Add 1000 empty files to the filesystem -# 8. Add a file with a large xattr value -# 9. Use xattrtest to create files with random xattrs (with and without xattrs=on) -# 10. Take a snapshot of the filesystem -# 11. Truncate one of the files from 32M to 128k -# 12. Truncate one of the files from 512k to 384k -# 13. Truncate one of the files from 512k to 0 to 384k -# 14. Remove the 1000 empty files to the filesystem -# 15. Take another snapshot of the filesystem -# 16. Send and receive both snapshots -# 17. Mount the filesystem and check the contents +# 6. Add 1000 empty files to the filesystem +# 7. Add a file with a large xattr value +# 8. Use xattrtest to create files with random xattrs (with and without xattrs=on) +# 9. Take a snapshot of the filesystem +# 10. Remove the 1000 empty files to the filesystem +# 11. Take another snapshot of the filesystem +# 12. Send and receive both snapshots +# 13. Mount the filesystem and check the contents # verify_runnable "both" @@ -68,15 +65,12 @@ log_must eval "echo 'password' > $keyfile" log_must zfs create -o encryption=on -o keyformat=passphrase \ -o keylocation=file://$keyfile $TESTPOOL/$TESTFS2 -# Create files with vaired layouts on disk +# Create files with varied layouts on disk log_must touch /$TESTPOOL/$TESTFS2/empty log_must mkfile 512 /$TESTPOOL/$TESTFS2/small log_must mkfile 32M /$TESTPOOL/$TESTFS2/full log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/sparse \ bs=512 count=1 seek=1048576 >/dev/null 2>&1 -log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated -log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2 -log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3 log_must mkdir -p /$TESTPOOL/$TESTFS2/dir for i in {1..1000}; do @@ -95,23 +89,10 @@ log_must zfs set compression=on xattr=sa $TESTPOOL/$TESTFS2 log_must touch /$TESTPOOL/$TESTFS2/attrs log_must eval "python -c 'print \"a\" * 4096' | \ attr -s bigval /$TESTPOOL/$TESTFS2/attrs" +log_must zfs set compression=off xattr=on $TESTPOOL/$TESTFS2 log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1 -# -# Truncate files created in the first snapshot. The first tests -# truncating a large file to a single block. The second tests -# truncating one block off the end of a file without changing -# the required nlevels to hold it. The last tests handling -# of a maxblkid that is dropped and then raised again. -# -log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated -log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2 -log_must truncate -s 0 /$TESTPOOL/$TESTFS2/truncated3 -log_must zpool sync $TESTPOOL -log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \ - bs=128k count=3 iflag=fullblock - # Remove the empty files created in the first snapshot for i in {1..1000}; do log_must rm /$TESTPOOL/$TESTFS2/dir/file-$i diff --git a/tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh b/tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh new file mode 100755 index 000000000..d701bcecb --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send_encrypted_truncated_files.ksh @@ -0,0 +1,118 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Datto Inc. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# DESCRIPTION: +# +# +# STRATEGY: +# 1. Create a new encrypted filesystem +# 2. Add a 4 files that are to be truncated later +# 3. Take a snapshot of the filesystem +# 4. Truncate one of the files from 32M to 128k +# 5. Truncate one of the files from 512k to 384k +# 6. Truncate one of the files from 512k to 0 to 384k via reallocation +# 7. Truncate one of the files from 1k to 0 to 512b via reallocation +# 8. Take another snapshot of the filesystem +# 9. Send and receive both snapshots +# 10. Mount the filesystem and check the contents +# + +verify_runnable "both" + +function cleanup +{ + datasetexists $TESTPOOL/$TESTFS2 && \ + log_must zfs destroy -r $TESTPOOL/$TESTFS2 + datasetexists $TESTPOOL/recv && \ + log_must zfs destroy -r $TESTPOOL/recv + [[ -f $keyfile ]] && log_must rm $keyfile + [[ -f $sendfile ]] && log_must rm $sendfile +} +log_onexit cleanup + +function recursive_cksum +{ + find $1 -type f -exec sha256sum {} \; | \ + sort -k 2 | awk '{ print $1 }' | sha256sum +} + +log_assert "Verify 'zfs send -w' works with many different file layouts" + +typeset keyfile=/$TESTPOOL/pkey +typeset sendfile=/$TESTPOOL/sendfile +typeset sendfile2=/$TESTPOOL/sendfile2 + +# Create an encrypted dataset +log_must eval "echo 'password' > $keyfile" +log_must zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=file://$keyfile $TESTPOOL/$TESTFS2 + +# Explicitly set the recordsize since the truncation sizes below depend on +# this value being 128k. This is currently same as the default recordsize. +log_must zfs set recordsize=128k $TESTPOOL/$TESTFS2 + +# Create files with varied layouts on disk +log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated +log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated2 +log_must mkfile 524288 /$TESTPOOL/$TESTFS2/truncated3 +log_must mkfile 1024 /$TESTPOOL/$TESTFS2/truncated4 + +log_must zfs snapshot $TESTPOOL/$TESTFS2@snap1 + +# +# Truncate files created in the first snapshot. The first tests +# truncating a large file to a single block. The second tests +# truncating one block off the end of a file without changing +# the required nlevels to hold it. The third tests handling +# of a maxblkid that is dropped and then raised again. The +# fourth tests an object that is truncated from a single block +# to a smaller single block. +# +log_must truncate -s 131072 /$TESTPOOL/$TESTFS2/truncated +log_must truncate -s 393216 /$TESTPOOL/$TESTFS2/truncated2 +log_must rm -f /$TESTPOOL/$TESTFS2/truncated3 +log_must rm -f /$TESTPOOL/$TESTFS2/truncated4 +log_must zpool sync $TESTPOOL +log_must zfs umount $TESTPOOL/$TESTFS2 +log_must zfs mount $TESTPOOL/$TESTFS2 +log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated3 \ + bs=128k count=3 iflag=fullblock +log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/truncated4 \ + bs=512 count=1 iflag=fullblock + +log_must zfs snapshot $TESTPOOL/$TESTFS2@snap2 +expected_cksum=$(recursive_cksum /$TESTPOOL/$TESTFS2) + +log_must eval "zfs send -wp $TESTPOOL/$TESTFS2@snap1 > $sendfile" +log_must eval "zfs send -wp -i @snap1 $TESTPOOL/$TESTFS2@snap2 > $sendfile2" + +log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile" +log_must eval "zfs recv -F $TESTPOOL/recv < $sendfile2" +log_must zfs load-key $TESTPOOL/recv + +log_must zfs mount -a +actual_cksum=$(recursive_cksum /$TESTPOOL/recv) +[[ "$expected_cksum" != "$actual_cksum" ]] && \ + log_fail "Recursive checksums differ ($expected_cksum != $actual_cksum)" + +log_pass "Verified 'zfs send -w' works with many different file layouts" diff --git a/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh b/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh new file mode 100755 index 000000000..0649beaa3 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh @@ -0,0 +1,112 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2019 by Lawrence Livermore National Security, LLC. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify encrypted raw incremental receives handle dnode reallocation. + +# Strategy: +# 1. Create a pool containing an encrypted filesystem. +# 2. Use 'zfs send -wp' to perform a raw send of the initial filesystem. +# 3. Repeat the followings steps N times to verify raw incremental receives. +# a) Randomly change several key dataset properties. +# b) Modify the contents of the filesystem such that dnode reallocation +# is likely during the 'zfs receive', and receive_object() exercises +# as much of its functionality as possible. +# c) Create a new snapshot and generate an raw incremental stream. +# d) Receive the raw incremental stream and verify the received contents. +# e) Destroy the incremental stream and old snapshot. +# + +verify_runnable "both" + +log_assert "Verify encrypted raw incremental receive handles reallocation" + +function cleanup +{ + rm -f $BACKDIR/fs@* + rm -f $keyfile + destroy_dataset $POOL/fs "-rR" + destroy_dataset $POOL/newfs "-rR" +} + +log_onexit cleanup + +typeset keyfile=/$TESTPOOL/pkey + +# Create an encrypted dataset +log_must eval "echo 'password' > $keyfile" +log_must zfs create -o encryption=on -o keyformat=passphrase \ + -o keylocation=file://$keyfile $POOL/fs + +last_snap=1 +log_must zfs snapshot $POOL/fs@snap${last_snap} +log_must eval "zfs send -wp $POOL/fs@snap${last_snap} \ + >$BACKDIR/fs@snap${last_snap}" +log_must eval "zfs recv $POOL/newfs < $BACKDIR/fs@snap${last_snap}" + +# Set atime=off to prevent the recursive_cksum from modifying newfs. +log_must zfs set atime=off $POOL/newfs + +for i in {1..5}; do + # Randomly modify several dataset properties in order to generate + # more interesting incremental send streams. + rand_set_prop $POOL/fs checksum "off" "fletcher4" "sha256" + rand_set_prop $POOL/fs compression "off" "lzjb" "gzip" "lz4" + rand_set_prop $POOL/fs recordsize "32K" "128K" + rand_set_prop $POOL/fs dnodesize "legacy" "auto" "4k" + rand_set_prop $POOL/fs xattr "on" "sa" + + # Churn the filesystem in such a way that we're likely to be both + # allocating and reallocating objects in the incremental stream. + # + # Disable xattrs until the following spill block issue is resolved: + # https://github.com/openzfs/openzfs/pull/705 + # + log_must churn_files 1000 524288 $POOL/fs 0 + expected_cksum=$(recursive_cksum /$fs) + + # Create a snapshot and use it to send an incremental stream. + this_snap=$((last_snap + 1)) + log_must zfs snapshot $POOL/fs@snap${this_snap} + log_must eval "zfs send -wp -i $POOL/fs@snap${last_snap} \ + $POOL/fs@snap${this_snap} > $BACKDIR/fs@snap${this_snap}" + + # Receive the incremental stream and verify the received contents. + log_must eval "zfs recv -Fu $POOL/newfs < $BACKDIR/fs@snap${this_snap}" + + log_must zfs load-key $POOL/newfs + log_must zfs mount $POOL/newfs + actual_cksum=$(recursive_cksum /$POOL/newfs) + log_must zfs umount $POOL/newfs + log_must zfs unload-key $POOL/newfs + + if [[ "$expected_cksum" != "$actual_cksum" ]]; then + log_fail "Checksums differ ($expected_cksum != $actual_cksum)" + fi + + # Destroy the incremental stream and old snapshot. + rm -f $BACKDIR/fs@snap${last_snap} + log_must zfs destroy $POOL/fs@snap${last_snap} + log_must zfs destroy $POOL/newfs@snap${last_snap} + last_snap=$this_snap +done + +log_pass "Verify encrypted raw incremental receive handles reallocation" |