diff options
author | Paul Dagnelie <pcd@delphix.com> | 2019-06-19 09:48:13 -0700 |
---|---|---|
committer | Brian Behlendorf <behlendorf1@llnl.gov> | 2019-06-19 09:48:12 -0700 |
commit | 30af21b02569ac192f52ce6e6511015f8a8d5729 (patch) | |
tree | e5f1091c2d3a6e511bbd2414782e490c18e0f59c /tests | |
parent | c1b5801bb5af0055e5f3d263beaa07026103e212 (diff) |
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
Diffstat (limited to 'tests')
39 files changed, 2559 insertions, 7 deletions
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 22fc26212..e45868765 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -735,6 +735,15 @@ tests = ['quota_001_pos', 'quota_002_pos', 'quota_003_pos', 'quota_004_pos', 'quota_005_pos', 'quota_006_neg'] tags = ['functional', 'quota'] +[tests/functional/redacted_send] +tests = ['redacted_compressed', 'redacted_contents', 'redacted_deleted', + 'redacted_disabled_feature', 'redacted_embedded', 'redacted_holes', + 'redacted_incrementals', 'redacted_largeblocks', 'redacted_many_clones', + 'redacted_mixed_recsize', 'redacted_mounts', 'redacted_negative', + 'redacted_origin', 'redacted_props', 'redacted_resume', 'redacted_size', + 'redacted_volume'] +tags = ['functional', 'redacted_send'] + [tests/functional/raidz] tests = ['raidz_001_neg', 'raidz_002_pos'] tags = ['functional', 'raidz'] diff --git a/tests/zfs-tests/cmd/Makefile.am b/tests/zfs-tests/cmd/Makefile.am index 39a538d2d..09c59f591 100644 --- a/tests/zfs-tests/cmd/Makefile.am +++ b/tests/zfs-tests/cmd/Makefile.am @@ -8,6 +8,7 @@ SUBDIRS = \ file_check \ file_trunc \ file_write \ + get_diff \ largest_file \ libzfs_input_check \ mkbusy \ @@ -24,4 +25,5 @@ SUBDIRS = \ rename_dir \ rm_lnkcnt_zero_file \ threadsappend \ - xattrtest + xattrtest \ + stride_dd diff --git a/tests/zfs-tests/cmd/get_diff/Makefile.am b/tests/zfs-tests/cmd/get_diff/Makefile.am new file mode 100644 index 000000000..06c39ddd8 --- /dev/null +++ b/tests/zfs-tests/cmd/get_diff/Makefile.am @@ -0,0 +1,6 @@ +include $(top_srcdir)/config/Rules.am + +pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/bin + +pkgexec_PROGRAMS = get_diff +get_diff_SOURCES = get_diff.c diff --git a/tests/zfs-tests/cmd/get_diff/get_diff.c b/tests/zfs-tests/cmd/get_diff/get_diff.c new file mode 100644 index 000000000..2799f46b0 --- /dev/null +++ b/tests/zfs-tests/cmd/get_diff/get_diff.c @@ -0,0 +1,109 @@ +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2018 by Delphix. All rights reserved. + */ + +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <sys/param.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> + +static void +usage(char *msg, int exit_value) +{ + (void) fprintf(stderr, "get_diff file redacted_file\n"); + (void) fprintf(stderr, "%s\n", msg); + exit(exit_value); +} + +/* + * This utility compares two files, an original and its redacted counterpart + * (in that order). It compares the files 512 bytes at a time, printing out + * any ranges (as offset and length) where the redacted file does not match + * the original. This output is used to verify that the expected ranges of + * a redacted file do not contain the original data. + */ +int +main(int argc, char *argv[]) +{ + off_t diff_off = 0, diff_len = 0, off = 0; + int fd1, fd2; + char *fname1, *fname2; + char buf1[DEV_BSIZE], buf2[DEV_BSIZE]; + ssize_t bytes; + + if (argc != 3) + usage("Incorrect number of arguments.", 1); + + if ((fname1 = argv[1]) == NULL) + usage("Filename missing.", 1); + if ((fd1 = open(fname1, O_LARGEFILE | O_RDONLY)) < 0) { + perror("open1 failed"); + exit(1); + } + + if ((fname2 = argv[2]) == NULL) + usage("Redacted filename missing.", 1); + if ((fd2 = open(fname2, O_LARGEFILE | O_RDONLY)) < 0) { + perror("open2 failed"); + exit(1); + } + + while ((bytes = pread(fd1, buf1, DEV_BSIZE, off)) > 0) { + if (pread(fd2, buf2, DEV_BSIZE, off) < 0) { + if (errno == EIO) { + /* + * A read in a redacted section of a file will + * fail with EIO. If we get EIO, continue on + * but ensure that a comparison of buf1 and + * buf2 will fail, indicating a redacted block. + */ + buf2[0] = ~buf1[0]; + } else { + perror("pread failed"); + exit(1); + } + } + if (memcmp(buf1, buf2, bytes) == 0) { + if (diff_len != 0) { + (void) fprintf(stdout, "%lld,%lld\n", + (long long)diff_off, (long long)diff_len); + assert(off == diff_off + diff_len); + diff_len = 0; + } + diff_off = 0; + } else { + if (diff_len == 0) + diff_off = off; + assert(off == diff_off + diff_len); + diff_len += bytes; + } + off += bytes; + } + + if (diff_len != 0 && diff_len != 0) { + (void) fprintf(stdout, "%lld,%lld\n", (long long)diff_off, + (long long)diff_len); + } + + (void) close(fd1); + (void) close(fd2); + + return (0); +} diff --git a/tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c b/tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c index 977b9e2f3..2de1ba20c 100644 --- a/tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c +++ b/tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c @@ -691,6 +691,34 @@ zfs_destroy(const char *dataset) } static void +test_redact(const char *snapshot1, const char *snapshot2) +{ + nvlist_t *required = fnvlist_alloc(); + nvlist_t *snapnv = fnvlist_alloc(); + char bookmark[MAXNAMELEN + 32]; + + fnvlist_add_string(required, "bookname", "testbookmark"); + fnvlist_add_boolean(snapnv, snapshot2); + fnvlist_add_nvlist(required, "snapnv", snapnv); + + IOC_INPUT_TEST(ZFS_IOC_REDACT, snapshot1, required, NULL, 0); + + nvlist_free(snapnv); + nvlist_free(required); + + strncpy(bookmark, snapshot1, sizeof (bookmark) - 1); + *strchr(bookmark, '@') = '\0'; + strncat(bookmark, "#testbookmark", sizeof (bookmark)); + zfs_destroy(bookmark); +} + +static void +test_get_bookmark_props(const char *bookmark) +{ + IOC_INPUT_TEST(ZFS_IOC_GET_BOOKMARK_PROPS, bookmark, NULL, NULL, 0); +} + +static void zfs_ioc_input_tests(const char *pool) { char filepath[] = "/tmp/ioc_test_file_XXXXXX"; @@ -700,6 +728,7 @@ zfs_ioc_input_tests(const char *pool) char bookmark[ZFS_MAX_DATASET_NAME_LEN + 32]; char backup[ZFS_MAX_DATASET_NAME_LEN]; char clone[ZFS_MAX_DATASET_NAME_LEN]; + char clonesnap[ZFS_MAX_DATASET_NAME_LEN + 32]; int tmpfd, err; /* @@ -710,6 +739,7 @@ zfs_ioc_input_tests(const char *pool) (void) snprintf(snapshot, sizeof (snapshot), "%s@snapshot", dataset); (void) snprintf(bookmark, sizeof (bookmark), "%s#bookmark", dataset); (void) snprintf(clone, sizeof (clone), "%s/test-fs-clone", pool); + (void) snprintf(clonesnap, sizeof (clonesnap), "%s@snap", clone); (void) snprintf(backup, sizeof (backup), "%s/backup", pool); err = lzc_create(dataset, DMU_OST_ZFS, NULL, NULL, 0); @@ -747,6 +777,7 @@ zfs_ioc_input_tests(const char *pool) test_bookmark(pool, snapshot, bookmark); test_get_bookmarks(dataset); + test_get_bookmark_props(bookmark); test_destroy_bookmarks(pool, bookmark); test_hold(pool, snapshot); @@ -754,6 +785,9 @@ zfs_ioc_input_tests(const char *pool) test_release(pool, snapshot); test_clone(snapshot, clone); + test_snapshot(pool, clonesnap); + test_redact(snapshot, clonesnap); + zfs_destroy(clonesnap); zfs_destroy(clone); test_rollback(dataset, snapshot); @@ -909,6 +943,8 @@ validate_ioc_values(void) ZFS_IOC_BASE + 78 == ZFS_IOC_POOL_DISCARD_CHECKPOINT && ZFS_IOC_BASE + 79 == ZFS_IOC_POOL_INITIALIZE && ZFS_IOC_BASE + 80 == ZFS_IOC_POOL_TRIM && + ZFS_IOC_BASE + 81 == ZFS_IOC_REDACT && + ZFS_IOC_BASE + 82 == ZFS_IOC_GET_BOOKMARK_PROPS && LINUX_IOC_BASE + 1 == ZFS_IOC_EVENTS_NEXT && LINUX_IOC_BASE + 2 == ZFS_IOC_EVENTS_CLEAR && LINUX_IOC_BASE + 3 == ZFS_IOC_EVENTS_SEEK); diff --git a/tests/zfs-tests/cmd/stride_dd/.gitignore b/tests/zfs-tests/cmd/stride_dd/.gitignore new file mode 100644 index 000000000..7c072ee0d --- /dev/null +++ b/tests/zfs-tests/cmd/stride_dd/.gitignore @@ -0,0 +1 @@ +/stride_dd diff --git a/tests/zfs-tests/cmd/stride_dd/Makefile.am b/tests/zfs-tests/cmd/stride_dd/Makefile.am new file mode 100644 index 000000000..d6f1adbac --- /dev/null +++ b/tests/zfs-tests/cmd/stride_dd/Makefile.am @@ -0,0 +1,7 @@ +include $(top_srcdir)/config/Rules.am + +pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/bin + +pkgexec_PROGRAMS = stride_dd +stride_dd_SOURCES = stride_dd.c +stride_dd_LDADD = -lrt diff --git a/tests/zfs-tests/cmd/stride_dd/stride_dd.c b/tests/zfs-tests/cmd/stride_dd/stride_dd.c new file mode 100644 index 000000000..88bd53292 --- /dev/null +++ b/tests/zfs-tests/cmd/stride_dd/stride_dd.c @@ -0,0 +1,214 @@ +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2018 by Delphix. All rights reserved. + */ + +#include <sys/types.h> +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> + +static int bsize = 0; +static int count = 0; +static char *ifile = NULL; +static char *ofile = NULL; +static int stride = 0; +static int seek = 0; +static char *execname = "stride_dd"; + +static void usage(void); +static void parse_options(int argc, char *argv[]); + +static void +usage(void) +{ + (void) fprintf(stderr, + "usage: %s -i inputfile -o outputfile -b blocksize -c count \n" + " -s stride [ -k seekblocks]\n" + "\n" + "Simplified version of dd that supports the stride option.\n" + "A stride of n means that for each block written, n - 1 blocks\n" + "are skipped in both the input and output file. A stride of 1\n" + "means that blocks are read and written consecutively.\n" + "All numeric parameters must be integers.\n" + "\n" + " inputfile: File to read from\n" + " outputfile: File to write to\n" + " blocksize: Size of each block to read/write\n" + " count: Number of blocks to read/write\n" + " stride: Read/write a block then skip (stride - 1) blocks\n" + " seekblocks: Number of blocks to skip at start of output\n", + execname); + (void) exit(1); +} + +static void +parse_options(int argc, char *argv[]) +{ + int c; + int errflag = 0; + + execname = argv[0]; + + extern char *optarg; + extern int optind, optopt; + + while ((c = getopt(argc, argv, ":b:c:i:o:s:k:")) != -1) { + switch (c) { + case 'b': + bsize = atoi(optarg); + break; + + case 'c': + count = atoi(optarg); + break; + + case 'i': + ifile = optarg; + break; + + case 'o': + ofile = optarg; + break; + + case 's': + stride = atoi(optarg); + break; + + case 'k': + seek = atoi(optarg); + break; + + case ':': + (void) fprintf(stderr, + "Option -%c requires an operand\n", optopt); + errflag++; + break; + + case '?': + default: + (void) fprintf(stderr, + "Unrecognized option: -%c\n", optopt); + errflag++; + break; + } + + if (errflag) { + (void) usage(); + } + } + + if (bsize <= 0 || count <= 0 || stride <= 0 || ifile == NULL || + ofile == NULL || seek < 0) { + (void) fprintf(stderr, + "Required parameter(s) missing or invalid.\n"); + (void) usage(); + } +} + +int +main(int argc, char *argv[]) +{ + int i; + int ifd; + int ofd; + void *buf; + int c; + + parse_options(argc, argv); + + ifd = open(ifile, O_RDONLY); + if (ifd == -1) { + (void) fprintf(stderr, "%s: %s: ", execname, ifile); + perror("open"); + exit(2); + } + + ofd = open(ofile, O_WRONLY | O_CREAT, 0666); + if (ofd == -1) { + (void) fprintf(stderr, "%s: %s: ", execname, ofile); + perror("open"); + exit(2); + } + + /* + * We use valloc because some character block devices expect a + * page-aligned buffer. + */ + int err = posix_memalign(&buf, 4096, bsize); + if (err != 0) { + (void) fprintf(stderr, + "%s: %s\n", execname, strerror(err)); + exit(2); + } + + if (seek > 0) { + if (lseek(ofd, seek * bsize, SEEK_CUR) == -1) { + perror("output lseek"); + exit(2); + } + } + + for (i = 0; i < count; i++) { + c = read(ifd, buf, bsize); + if (c != bsize) { + + perror("read"); + exit(2); + } + if (c != bsize) { + if (c < 0) { + perror("read"); + } else { + (void) fprintf(stderr, + "%s: unexpected short read, read %d " + "bytes, expected %d\n", execname, + c, bsize); + } + exit(2); + } + + c = write(ofd, buf, bsize); + if (c != bsize) { + if (c < 0) { + perror("write"); + } else { + (void) fprintf(stderr, + "%s: unexpected short write, wrote %d " + "bytes, expected %d\n", execname, + c, bsize); + } + exit(2); + } + + if (stride > 1) { + if (lseek(ifd, (stride - 1) * bsize, SEEK_CUR) == -1) { + perror("input lseek"); + exit(2); + } + if (lseek(ofd, (stride - 1) * bsize, SEEK_CUR) == -1) { + perror("output lseek"); + exit(2); + } + } + } + free(buf); + + (void) close(ofd); + (void) close(ifd); + + return (0); +} diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 127a1477d..f8ad02246 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -1,4 +1,5 @@ # +# Copyright (c) 2016, 2018 by Delphix. All rights reserved. # These variables are used by zfs-tests.sh to constrain which utilities # may be used by the suite. The suite will create a directory which is # the only element of $PATH and create symlinks from that dir to the @@ -163,6 +164,7 @@ export ZFSTEST_FILES='chg_usr_exec file_check file_trunc file_write + get_diff largest_file libzfs_input_check mkbusy @@ -180,4 +182,5 @@ export ZFSTEST_FILES='chg_usr_exec rm_lnkcnt_zero_file threadsappend user_ns_exec - xattrtest' + xattrtest + stride_dd' diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index 1b841d7ba..9c3c20fd9 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -405,7 +405,8 @@ function create_recv_clone log_must eval "zfs send $snap | zfs recv -u $recvfs" log_must mkfile 1m "$mountpoint/data" log_must zfs snapshot $incr - log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 > $sendfile" + log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 \ + iflag=fullblock > $sendfile" log_mustnot eval "zfs recv -su $recvfs < $sendfile" destroy_dataset "$sendfs" "-r" log_must rm -f "$sendfile" diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am index da27673ec..2d23eb296 100644 --- a/tests/zfs-tests/tests/functional/Makefile.am +++ b/tests/zfs-tests/tests/functional/Makefile.am @@ -52,6 +52,7 @@ SUBDIRS = \ projectquota \ quota \ raidz \ + redacted_send \ redundancy \ refquota \ refreserv \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh new file mode 100755 index 000000000..447fbb36b --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# When a snapshot is destroyed, we used to recurse all clones +# that are downstream of the destroyed snapshot (e.g. to remove +# its key and merge its deadlist entries to the previous one). +# This recursion would break the stack on deeply nested clone +# hierarchies. To avoid this problem today, we keep heap-allocated +# records of all the clones as we traverse their hierarchy. +# +# This test ensures and showcases that our new method works with +# deeply nested clone hierarchies. +# +# STRATEGY: +# 1. Create an fs and take a snapshot of it (snapshot foo) +# 2. Take a second snapshot of the same fs (snapshot bar) on +# top of snapshot foo +# 3. Create a clone of snapshot bar and then take a snapshot +# of it. +# 4. Create a clone of the newly-created snapshot and then +# take a snapshot of it. +# 5. Repeat step [4] many times to create a deeply nested hierarchy. +# 6. Destroy snapshot foo. +# + +verify_runnable "both" + +typeset FS0=$TESTPOOL/0 +typeset FOO=foo +typeset BAR=BAR + +typeset FS0SNAPFOO=$FS0@$FOO +typeset FS0SNAPBAR=$FS0@$BAR + +typeset -i numds=300 + +log_must zfs create $FS0 + +function test_cleanup +{ + log_must zfs destroy -Rf $FS0 + + return 0 +} + +log_must zfs snapshot $FS0SNAPFOO +log_must zfs snapshot $FS0SNAPBAR + +log_onexit test_cleanup + +for (( i=1; i<numds; i++ )); do + log_must zfs clone $TESTPOOL/$((i-1))@$BAR $TESTPOOL/$i + log_must zfs snapshot $TESTPOOL/$i@$BAR +done + +log_must zfs destroy $FS0SNAPFOO + +log_pass "Snapshot deletion doesn't break the stack in deeply nested " \ + "clone hierarchies." diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send-b.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send-b.ksh index cd879846c..ab2a9508e 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send-b.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send-b.ksh @@ -65,7 +65,6 @@ done for opt in ${opts[@]}; do log_mustnot eval "zfs send -b$opt $SENDFS > /dev/null" log_mustnot eval "zfs send -b$opt $SENDFS#bm > /dev/null" - log_mustnot eval "zfs send -b$opt -i $SENDFS#bm $SENDFS@s2 > /dev/null" done # Do 3..6 in a loop to verify various combination of "zfs send" options diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh index 7192551b6..5d3d7cf0d 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh @@ -15,7 +15,7 @@ # # -# Copyright (c) 2012, 2016 by Delphix. All rights reserved. +# Copyright (c) 2012, 2018 by Delphix. All rights reserved. # . $STF_SUITE/include/libtest.shlib @@ -36,6 +36,7 @@ verify_runnable "both" function cleanup { + log_must set_tunable32 zfs_override_estimate_recordsize 8192 for ds in $datasets; do destroy_dataset $ds "-rf" done @@ -90,6 +91,7 @@ function verify_size_estimates log_assert "Verify 'zfs send -nvP' generates valid stream estimates" log_onexit cleanup +log_must set_tunable32 zfs_override_estimate_recordsize 0 typeset -l block_count=0 typeset -l block_size typeset -i PERCENT=1 diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg index fdcce8b56..9e2f00f6a 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg @@ -77,6 +77,9 @@ typeset -a properties=( "feature@obsolete_counts" "feature@zpool_checkpoint" "feature@spacemap_v2" + "feature@redaction_bookmarks" + "feature@redacted_datasets" + "feature@bookmark_written" ) # Additional properties added for Linux. diff --git a/tests/zfs-tests/tests/functional/redacted_send/Makefile.am b/tests/zfs-tests/tests/functional/redacted_send/Makefile.am new file mode 100644 index 000000000..dd6b4eb67 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/Makefile.am @@ -0,0 +1,25 @@ +pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/redacted_send +dist_pkgdata_SCRIPTS = \ + setup.ksh \ + cleanup.ksh \ + redacted_compressed.ksh \ + redacted_contents.ksh \ + redacted_deleted.ksh \ + redacted_disabled_feature.ksh \ + redacted_embedded.ksh \ + redacted_holes.ksh \ + redacted_incrementals.ksh \ + redacted_largeblocks.ksh \ + redacted_many_clones.ksh \ + redacted_mixed_recsize.ksh \ + redacted_mounts.ksh \ + redacted_negative.ksh \ + redacted_origin.ksh \ + redacted_props.ksh \ + redacted_resume.ksh \ + redacted_size.ksh \ + redacted_volume.ksh + +dist_pkgdata_DATA = \ + redacted.cfg \ + redacted.kshlib diff --git a/tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh b/tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh new file mode 100755 index 000000000..596c661ed --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh @@ -0,0 +1,33 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +destroy_pool $POOL +destroy_pool $POOL2 +log_must set_tunable32 zfs_allow_redacted_dataset_mount 0 + +log_pass diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted.cfg b/tests/zfs-tests/tests/functional/redacted_send/redacted.cfg new file mode 100644 index 000000000..f964b37ba --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted.cfg @@ -0,0 +1,86 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +export DISK1=$(echo $DISKS | awk '{print $1}') +export DISK2=$(echo $DISKS | awk '{print $2}') + +export POOL=$TESTPOOL +export POOL2=$TESTPOOL2 +export FS=$TESTFS +export FS2=$TESTFS2 + +# +# These are the byte ranges that differ between files and their redacted +# counterparts. See compare_files() for more detail. +# +typeset RANGE0="0,2097152" +typeset RANGE1="0,131072" +typeset RANGE2="1048576,2097152" +typeset RANGE3="0,131072 +1966080,131072 +3932160,131072" +typeset RANGE4="0,131072 +262144,131072 +524288,131072 +786432,131072" +typeset RANGE5="0,1048576 +7340032,1048576" +typeset RANGE6="393216,131072 +655360,131072 +917504,131072 +1179648,131072 +1441792,393216 +1966080,393216 +2621440,262144 +3145728,262144 +3670016,262144 +4194304,262144 +4718592,262144 +5242880,262144" +typeset RANGE7="1048576,6291456" +typeset RANGE8="4063232,131072" +typeset RANGE9="0,131072 +262144,131072 +524288,131072 +786432,131072 +1048576,131072 +1310720,131072 +1572864,131072 +1835008,131072 +2097152,131072 +2359296,131072 +2621440,131072 +2883584,131072 +3145728,131072 +3407872,131072 +3670016,131072 +3932160,131072" +typeset RANGE10="0,393216" +typeset RANGE11="0,1048576" +typeset RANGE12="0,2097152" +typeset RANGE13="0,16384" +typeset RANGE14="" +typeset RANGE15="0,4194304" +typeset RANGE16="0,6291456"
\ No newline at end of file diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib b/tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib new file mode 100644 index 000000000..d3d219bb9 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib @@ -0,0 +1,270 @@ +#!/bin/ksh + +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2016, 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/tests/functional/redacted_send/redacted.cfg + +function setup_dataset +{ + typeset ds_name=$1 + typeset opts=$2 + typeset file_create_func=$3 + typeset sendfs="$POOL/$ds_name" + [[ -n $file_create_func ]] || file_create_func=setup_common + + log_must zfs create $opts $sendfs + + $file_create_func $sendfs + + log_must zfs snapshot $sendfs@snap + log_must zfs clone $opts $sendfs@snap $POOL/${ds_name}_clone + log_must zfs snapshot $POOL/${ds_name}_clone@snap +} + +function setup_common +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + typeset bs=$(get_prop recsize $sendfs) + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=$bs count=16 + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=$bs count=32 +} + +function setup_embedded +{ + typeset sendfs=$1 + + typeset recsize + typeset mntpnt=$(get_prop mountpoint $sendfs) + for recsize in 512 1024 2048 4096 8192 16384; do + if is_linux; then + log_must dd if=/dev/urandom of=$mntpnt/$recsize bs=8 \ + count=1 seek=$(((recsize / 8) - 1)) + else + log_must mkholes -d $((recsize - 8)):8 $mntpnt/$recsize + fi + done +} + +function setup_holes +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + typeset M=$((1024 * 1024)) + + if is_linux; then + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=8M count=1 + + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1M count=1 + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1M count=1 seek=7 \ + conv=notrunc + + log_must dd if=/dev/urandom of=$mntpnt/f3 bs=1M count=6 seek=1 + log_must truncate $mntpnt/f3 --size=$((8 * M)) + + log_must truncate $mntpnt/f4 --size=$((8 * M)) + else + log_must mkholes -d 0:$((8 * M)) $mntpnt/f1 + log_must mkholes -d 0:$M -d $((7 * M)):$M $mntpnt/f2 + log_must mkholes -d $M:$((6 * M)) -h $((7 * M)):$M $mntpnt/f3 + log_must mkholes -h 0:$((8 * M)) $mntpnt/f4 + fi + + log_must zfs create $sendfs/manyrm + for i in {1..256}; do + log_must stride_dd -i /dev/urandom -o $mntpnt/manyrm/f$i -b 512 \ + -c $(random 100) -s $(random 4) + done + + log_must zfs snapshot $sendfs/manyrm@snap + log_must zfs clone $sendfs/manyrm@snap $sendfs/manyrm_clone + log_must zfs snapshot $sendfs/manyrm_clone@snap +} + +function setup_incrementals +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + typeset bs=$(get_prop recsize $sendfs) + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=$bs count=16 + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=$bs count=32 + log_must mkdir $mntpnt/d1 + log_must eval "cat $mntpnt/f1 $mntpnt/f2 >$mntpnt/d1/f1" + log_must zfs snapshot $sendfs@snap0 + + log_must zfs clone $sendfs@snap0 $POOL/hole + mntpnt=$(get_prop mountpoint $POOL/hole) + log_must dd if=/dev/zero of=$mntpnt/f2 bs=$bs count=16 conv=notrunc + log_must zfs snapshot $POOL/hole@snap + + log_must zfs clone $sendfs@snap0 $POOL/stride3 + mntpnt=$(get_prop mountpoint $POOL/stride3) + log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $bs -c 11 -s 3 + log_must zfs snapshot $POOL/stride3@snap + + log_must zfs clone $sendfs@snap0 $POOL/stride5 + mntpnt=$(get_prop mountpoint $POOL/stride5) + log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $bs -c 7 -s 5 + log_must zfs snapshot $POOL/stride5@snap + + log_must zfs clone $sendfs@snap0 $POOL/int + log_must zfs snapshot $POOL/int@snap + + log_must zfs clone $POOL/int@snap $POOL/rm + mntpnt=$(get_prop mountpoint $POOL/rm) + log_must rm -rf $mntpnt/[df][12] + log_must zfs snapshot $POOL/rm@snap + + log_must zfs clone $POOL/int@snap $POOL/write + mntpnt=$(get_prop mountpoint $POOL/write) + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=16 conv=notrunc + log_must dd if=/dev/urandom of=$mntpnt/d1/f1 bs=512 count=16 seek=16 \ + conv=notrunc + log_must zfs snapshot $POOL/write@snap +} + +function setup_mounts +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + log_must touch $mntpnt/empty + log_must dd if=/dev/urandom of=$mntpnt/contents1 bs=512 count=2 + log_must dd if=/dev/urandom of=$mntpnt/contents2 bs=512 count=2 + log_must mkdir $mntpnt/dir1 + log_must touch $mntpnt/dir1/empty + log_must dd if=/dev/urandom of=$mntpnt/dir1/contents1 bs=512 count=2 + log_must dd if=/dev/urandom of=$mntpnt/dir1/contents2 bs=512 count=2 + log_must mkdir $mntpnt/dir1/dir2 + log_must touch $mntpnt/dir1/dir2/empty + log_must dd if=/dev/urandom of=$mntpnt/dir1/dir2/file bs=512 count=2 + + log_must zfs create -s -V 16p $sendfs/vol + log_must zfs snapshot $sendfs/vol@snap + log_must zfs clone $sendfs/vol@snap $sendfs/vol_clone + log_must zfs snapshot $sendfs/vol_clone@snap +} + +function mount_redacted +{ + typeset flag='' + while getopts "f" opt; do + case $opt in + f) + flag='-f' + ;; + esac + done + shift $(($OPTIND - 1)) + + typeset ds=$1 + log_must set_tunable32 zfs_allow_redacted_dataset_mount 1 + zfs mount $flag -oro $ds || return 1 + log_must set_tunable32 zfs_allow_redacted_dataset_mount 0 + return 0 +} + +function unmount_redacted +{ + typeset ds=$1 + + zfs unmount $ds +} + +# +# This function calls a utility that prints out the ranges where a file +# and its redacted counterpart differ, each range on a new line like this: +# +# 0,131072 +# 1966080,131072 +# 3932160,131072 +# +# The output is then checked against a variable containing the expected +# output to verify the redacted ranges are the ones expected. +# +function compare_files +{ + typeset sendfs=$1 + typeset recvfs=$2 + typeset file=$3 + typeset expected="$4" + typeset tmpfile="$tmpdir/get_file.out" + + log_must mount_redacted -f $recvfs + + typeset file1="$(get_prop mountpoint $sendfs)/$file" + typeset file2="$(get_prop mountpoint $recvfs)/$file" + log_note "Comparing $file1 and $file2" + [[ -f $file1 ]] || log_fail "File $file1 does not exist." + [[ -f $file2 ]] || log_fail "File $file2 does not exist." + + log_must eval "get_diff $file1 $file2 >$tmpfile" + typeset range="$(cat $tmpfile)" + log_must unmount_redacted $recvfs + [[ "$expected" = "$range" ]] || log_fail "Unexpected range: $range" +} + +function redacted_cleanup +{ + typeset ds_list=$@ + typeset ds + + # Verify the receiving pool can still be exported and imported. + log_must zpool export $POOL2 + log_must zpool import $POOL2 + + for ds in $ds_list; do + datasetexists $ds && log_must zfs destroy -R $ds + done + + log_must set_tunable32 zfs_allow_redacted_dataset_mount 0 + rm -f $(get_prop mountpoint $POOL)/tmp/* +} + +# Retrieve the redaction list of a bookmark or snapshot, using +# the property or zdb output, as requested. +function get_guid_list +{ + typeset filename=$1 + typeset dataset=$2 + typeset use_zdb=${3:-false} + + if $use_zdb; then + guid_list=$(zdb -vvvv $dataset | sed -e 's/,//g' \ + -ne 's/^.*Snapshots: \[\(.*\)\]/\1/p') + else + guid_list=$(get_prop redact_snaps $dataset) + fi + + for guid in $(echo $guid_list | tr ',' ' '); do + echo $guid + done | sort >$filename +} diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh new file mode 100755 index 000000000..0a8bf3903 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh @@ -0,0 +1,71 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that compressed send streams are redacted correctly. +# +# Strategy: +# 1. Receive a redacted compressed send stream, verifying compression and +# redaction. +# 2. Receive an incremental on the full receive, verifying compression and +# redaction. +# + +typeset ds_name="compressed" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name "-o compress=lz4" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset clone_mnt="$(get_prop mountpoint $clone)" + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must stride_dd -i /dev/urandom -o $clone_mnt/f1 -b $((128 * 1024)) -c 4 -s 2 +log_must zfs snapshot $clone@snap1 +log_must rm $clone_mnt/f2 +log_must zfs snapshot $clone@snap2 + +log_must zfs redact $sendfs@snap book1 $clone@snap1 $clone@snap2 +log_must eval "zfs send -c --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must stream_has_features $stream compressed lz4 redacted +compare_files $sendfs $recvfs "f1" "$RANGE4" +verify_stream_size $stream $sendfs +log_must mount_redacted -f $recvfs +verify_stream_size $stream $recvfs +log_must unmount_redacted $recvfs + +log_must eval "zfs send -c -i $sendfs@snap $clone@snap1 >$stream" +log_must eval "zfs recv $POOL2/inc1 <$stream" +log_must stream_has_features $stream compressed lz4 +typeset mntpnt=$(get_prop mountpoint $POOL2) +log_must diff $clone_mnt/f1 $mntpnt/inc1/f1 +log_must diff $send_mnt/f2 $mntpnt/inc1/f2 + +log_must eval "zfs send -c -i $sendfs@snap $clone@snap2 >$stream" +log_must eval "zfs recv $POOL2/inc2 <$stream" +log_must stream_has_features $stream compressed lz4 +log_must diff $clone_mnt/f1 $mntpnt/inc1/f1 +[[ -f $mntpnt/inc2/f2 ]] && log_fail "File f2 should not exist." + +log_pass "Compressed send streams are redacted correctly." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh new file mode 100755 index 000000000..58dbde7f1 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh @@ -0,0 +1,162 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redaction works as expected for various scenarios. +# +# Strategy: +# 1. An unmodified file does not get redacted at all. +# 2. Empty redaction list redacts everything. +# 3. A file removed in the clone redacts the whole file. +# 4. A file moved in the clone does not redact the file. +# 5. A copied, then removed file in the clone redacts the whole file. +# 6. Overwriting a file with identical contents redacts the file. +# 7. A paritally modified block redacts the entire block. +# 8. Only overlapping areas of modified ranges are redacted. +# 9. Send from the root dataset of a pool work correctly. +# + +typeset ds_name="contents" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +# An unmodified file does not get redacted at all. +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must diff $send_mnt/f2 $recv_mnt/f2 +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Removing a file in the clone redacts the entire file. +log_must rm "$clone_mnt/f1" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book3 $clone@snap1 +log_must eval "zfs send --redact book3 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE0" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Moving a file in the clone does not redact the file. +log_must mv "$clone_mnt/f1" "$clone_mnt/f1.moved" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book4 $clone@snap1 +log_must eval "zfs send --redact book4 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +[[ -f $recv_mnt/f1.moved ]] && log_fail "Found moved file in redacted receive." +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Copying, then removing a file in the clone does redact the file. +log_must cp "$clone_mnt/f1" "$clone_mnt/f1.copied" +log_must rm "$clone_mnt/f1" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book5 $clone@snap1 +log_must eval "zfs send --redact book5 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE0" +log_must mount_redacted -f $recvfs +[[ -f $recv_mnt/f1.copied ]] && log_fail "Found moved file in redacted receive." +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Overwriting the contents of a block with identical contents redacts the file. +log_must cp "$clone_mnt/f1" "$clone_mnt/f1.copied" +log_must cp "$clone_mnt/f1.copied" "$clone_mnt/f1" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book6 $clone@snap1 +log_must eval "zfs send --redact book6 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE0" +log_must mount_redacted -f $recvfs +[[ -f $recv_mnt/f1.copied ]] && log_fail "Found moved file in redacted receive." +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Modifying some of a block redacts the whole block. +log_must dd if=/dev/urandom of=$clone_mnt/f1 conv=notrunc seek=2 count=1 bs=32k +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book7 $clone@snap1 +log_must eval "zfs send --redact book7 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE1" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Only overlapping areas of modified ranges are redacted. +log_must dd if=/dev/urandom of=$clone_mnt/f2 bs=1024k count=3 conv=notrunc +log_must zfs snapshot $clone@snap1 +log_must zfs clone $sendfs@snap $clone/new +typeset mntpnt="$(get_prop mountpoint $clone/new)" +log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1024k seek=1 count=3 \ + conv=notrunc +log_must zfs snapshot $clone/new@snap +log_must zfs redact $sendfs@snap book8 $clone@snap1 $clone/new@snap +log_must eval "zfs send --redact book8 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE2" +log_must zfs destroy -R $clone/new +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# FizzBuzz version +log_must zfs clone $sendfs@snap $POOL/stride3 +mntpnt="$(get_prop mountpoint $POOL/stride3)" +log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $((128 * 1024)) -c 11 -s 3 +log_must zfs snapshot $POOL/stride3@snap +log_must zfs clone $sendfs@snap $POOL/stride5 +mntpnt="$(get_prop mountpoint $POOL/stride5)" +log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $((128 * 1024)) -c 7 -s 5 +log_must zfs snapshot $POOL/stride5@snap +log_must zfs redact $sendfs@snap book8a $POOL/stride3@snap $POOL/stride5@snap +log_must eval "zfs send --redact book8a $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE3" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Send from the root dataset of a pool work correctly. +log_must dd if=/dev/urandom of=/$POOL/f1 bs=128k count=4 +log_must zfs snapshot $POOL@snap +log_must zfs clone $POOL@snap $POOL/clone +log_must dd if=/dev/urandom of=/$POOL/clone/f1 bs=128k count=1 conv=notrunc +log_must zfs snapshot $POOL/clone@snap +log_must zfs redact $POOL@snap book9 $POOL/clone@snap +log_must eval "zfs send --redact book9 $POOL@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $POOL $recvfs "f1" "$RANGE1" +log_must zfs destroy -R $POOL@snap + +log_pass "Redaction works as expected for various scenarios." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh new file mode 100755 index 000000000..e25b51cb9 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh @@ -0,0 +1,103 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017, 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redaction works as expected with respect to deleted files +# +# Strategy: +# 1. A file on the delete queue counts as deleted when using it to calculate +# redaction. +# 2. A file that is removed in the tosnap of an incremental, where the fromsnap +# is a redaction bookmark that contains references to that file, does not +# result in records for that file. +# + +typeset ds_name="deleted" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset clone2="$POOL/${ds_name}_clone2" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +# +# A file on the delete queue counts as deleted when using it to calculate +# redaction. +# + +# +# Open file descriptor 5 for appending to $clone_mnt/f1 so that it will go on +# the delete queue when we rm it. +# +exec 5>>$clone_mnt/f1 +log_must dd if=/dev/urandom of=$clone_mnt/f1 bs=512 count=1 conv=notrunc +log_must rm $clone_mnt/f1 +log_must zfs snapshot $clone@snap1 +# Close file descriptor 5 +exec 5>&- +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +# +# We have temporarily disabled redaction blkptrs, so this will not +# fail as was originally intended. We should uncomment this line +# when we reenable redaction blkptrs. +# +#log_mustnot dd if=$recv_mnt/f1 of=/dev/null bs=512 count=1 +log_must diff $send_mnt/f2 $recv_mnt/f2 +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# +# A file that is removed in the tosnap of an incremental, where the fromsnap +# is a redaction bookmark that contains references to that file, does not +# result in records for that file. +# +log_must zfs clone $sendfs@snap $clone2 +typeset clone2_mnt="$(get_prop mountpoint $clone2)" +log_must rm -rf $clone2_mnt/* +log_must zfs snapshot $clone2@snap +log_must zfs redact $sendfs@snap book2 $clone2@snap +log_must zfs destroy -R $clone2 +log_must eval "zfs send --redact book2 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must rm $send_mnt/f1 +log_must zfs snapshot $sendfs@snap2 +log_must zfs clone $sendfs@snap2 $clone2 +typeset clone2_mnt="$(get_prop mountpoint $clone2)" +log_must rm $clone2_mnt/* +log_must zfs snapshot $clone2@snap +log_must zfs redact $sendfs@snap2 book3 $clone2@snap +log_must zfs destroy -R $clone2 +log_must eval "zfs send -i $sendfs#book2 --redact book3 $sendfs@snap2 >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +log_must diff <(ls $send_mnt) <(ls $recv_mnt) +log_must zfs destroy -R $recvfs +log_must zfs rollback -R $sendfs@snap + +log_pass "Verify Redaction works as expected with respect to deleted files." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh new file mode 100755 index 000000000..24478f1bc --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh @@ -0,0 +1,71 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify the functionality of the redaction_bookmarks and redacted_datasets +# features. +# +# Strategy: +# 1. Create a pool with all features disabled. +# 2. Verify redacted send fails. +# 3. Enable redaction_bookmarks and verify redacted sends works. +# 4. Verify recepit of a redacted stream fails. +# 5. Enable recacted_datasets and verify zfs receive works. +# + +typeset ds_name="disabled" +typeset sendfs="$POOL/$ds_name" +typeset sendfs1="$POOL2/${ds_name}1" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset clone1="$POOL2/${ds_name}_clone1" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' + +function cleanup +{ + destroy_pool $POOL2 + create_pool $POOL2 $DISK2 + log_must zfs snapshot $POOL2@init + redacted_cleanup $sendfs $recvfs +} + +log_onexit cleanup + +destroy_pool $POOL2 +log_must zpool create -d $POOL2 $DISK2 + +log_must zfs create $sendfs1 +log_must zfs snapshot $sendfs1@snap +log_must zfs clone $sendfs1@snap $clone1 +log_must zfs snapshot $clone1@snap + +log_mustnot zfs redact $sendfs1@snap book1 $clone1@snap +log_must zpool set feature@redaction_bookmarks=enabled $POOL2 +log_must zfs redact $sendfs1@snap book1 $clone1@snap + +log_must zfs redact $sendfs@snap book1 $clone@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_mustnot eval "zfs recv $recvfs <$stream" +log_must zpool set feature@redacted_datasets=enabled $POOL2 +log_must eval "zfs recv $recvfs <$stream" + +log_pass "The redacted send/recv features work correctly." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh new file mode 100755 index 000000000..94937a2f7 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh @@ -0,0 +1,103 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify embedded blocks and redacted send work correctly together. +# +# Strategy: +# 1. Create recsize sized files with embedded blocks from size 512b to 16k. +# 2. Receive a redacted send stream with nothing redacted. +# 3. Verify the received files match the source, contain embedded blocks, and +# that the stream has the redacted and embedded data features. +# 4. Receive a redacted send stream with files 512, 2048 and 8192 redacted. +# 5. Verify that the redacted files no longer match, but the others still +# contain embedded blocks and the stream has the redacted and embedded +# data features. +# + +typeset ds_name="embedded" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '-o compress=lz4' setup_embedded +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset recsize send_obj recv_obj + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must zfs redact $sendfs@snap book1 $clone@snap +log_must eval "zfs send -e --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must stream_has_features $stream redacted embed_data + +log_must mount_redacted -f $recvfs +for recsize in 512 1024 2048 4096 8192 16384; do + send_obj=$(get_objnum $send_mnt/$recsize) + recv_obj=$(get_objnum $recv_mnt/$recsize) + + log_must diff $send_mnt/$recsize $recv_mnt/$recsize + log_must eval "zdb -ddddd $sendfs $send_obj >$tmpdir/send.zdb" + log_must eval "zdb -ddddd $recvfs $recv_obj >$tmpdir/recv.zdb" + + grep -q "EMBEDDED" $tmpdir/send.zdb || \ + log_fail "Obj $send_obj not embedded in $sendfs" + grep -q "EMBEDDED" $tmpdir/recv.zdb || \ + log_fail "Obj $recv_obj not embedded in $recvfs" + + cat $stream | zstreamdump -v | log_must grep -q \ + "WRITE_EMBEDDED object = $send_obj offset = 0" +done + +log_must zfs destroy -R $recvfs +for recsize in 512 2048 8192; do + log_must dd if=/dev/urandom of=$clone_mnt/$recsize bs=$recsize count=1 +done +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book2 $clone@snap1 +log_must eval "zfs send -e --redact book2 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must stream_has_features $stream redacted embed_data + +log_must mount_redacted -f $recvfs +for recsize in 512 2048 8192; do + log_mustnot diff $send_mnt/$recsize $recv_mnt/$recsize +done +for recsize in 1024 4096 16384; do + send_obj=$(get_objnum $send_mnt/$recsize) + recv_obj=$(get_objnum $recv_mnt/$recsize) + + log_must diff $send_mnt/$recsize $recv_mnt/$recsize + log_must eval "zdb -ddddd $sendfs $send_obj >$tmpdir/send.zdb" + log_must eval "zdb -ddddd $recvfs $recv_obj >$tmpdir/recv.zdb" + + grep -q "EMBEDDED" $tmpdir/send.zdb || \ + log_fail "Obj $send_obj not embedded in $sendfs" + grep -q "EMBEDDED" $tmpdir/recv.zdb || \ + log_fail "Obj $recv_obj not embedded in $recvfs" + + cat $stream | zstreamdump -v | log_must grep -q \ + "WRITE_EMBEDDED object = $send_obj offset = 0" +done + +log_pass "Embedded blocks and redacted send work correctly together." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh new file mode 100755 index 000000000..47063848f --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh @@ -0,0 +1,120 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redacted send streams reliably handle holes. +# +# Strategy: +# 1. Holes written at the beginning and end of a non-sparse file in the +# redacted list are correctly redacted. +# 2. Holes written throughout a non-sparse file in the redacted list are +# correctly redacted. +# 3. Data written into a hole in a sparse file in the redacted list are +# correctly redacted. +# 4. Holes in metadata blocks. +# + +typeset ds_name="holes" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_holes +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset M=$((1024 * 1024)) + +log_onexit redacted_cleanup $sendfs $recvfs + +# Write holes at the start and end of a non-sparse file. +if is_linux; then + log_must dd if=/dev/zero of=$clone_mnt/f1 bs=1M count=1 conv=notrunc + log_must dd if=/dev/zero of=$clone_mnt/f1 bs=1M count=1 conv=notrunc seek=7 +else + log_must mkholes -h 0:$M -h $((7 * M)):$M $clone_mnt/f1 +fi +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE5" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Write two overlapping sets of holes into the same non-sparse file. +log_must stride_dd -i /dev/zero -o $clone_mnt/f1 -b $((128 * 1024)) -c 8 -s 2 -k 3 +log_must stride_dd -i /dev/zero -o $clone_mnt/f1 -b $((256 * 1024)) -c 8 -s 2 -k 6 +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book2 $clone@snap1 +log_must eval "zfs send --redact book2 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE6" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Write data into the middle of a hole. +if is_linux; then + log_must dd if=/dev/urandom of=$clone_mnt/f2 bs=1M count=2 seek=3 \ + conv=notrunc +else + log_must mkholes -d $((3 * M)):$((2 * M)) $clone_mnt/f2 +fi +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book3 $clone@snap1 +log_must eval "zfs send --redact book3 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE14" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Remove a file with holes. +log_must rm $clone_mnt/f3 +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book4 $clone@snap1 +log_must eval "zfs send --redact book4 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f3" "$RANGE7" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Create a hole in a L0 metadata block by removing files. +log_must rm $send_mnt/manyrm_clone/f{32..96} +log_must zfs snapshot $sendfs/manyrm_clone@snap1 + +log_must zfs redact $sendfs/manyrm@snap book6 $sendfs/manyrm_clone@snap1 +log_must eval "zfs send --redact book6 $sendfs/manyrm@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +for i in {1..31} {97..256}; do + diff $send_mnt/manyrm/f$i $recv_mnt/f$i || log_fail \ + "File f$i did not match in the send and recv datasets." +done +for i in {32..96}; do + file_size=$(stat -c %s $send_mnt/manyrm/f$i) + redacted_size=$(stat -c %s $recv_mnt/f$i) + [[ $file_size -eq $redacted_size ]] || log_fail \ + "File f$i has size $file_size and redacted size $redacted_size" +done +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +log_pass "Redacted send streams reliably handle holes." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh new file mode 100755 index 000000000..1d2ed3a68 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh @@ -0,0 +1,152 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that incrementals (redacted and normal) work with redacted datasets. +# +# Strategy: +# 1. Test normal incrementals from the original snap to a subset of the +# redaction list. +# 2. Test receipt of intermediate clones, and their children. +# 3. Test receipt with origin snap specified by '-o origin='. +# 4. Test incrementals from redaction bookmarks. +# + +typeset ds_name="incrementals" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_incrementals +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs $POOL2/rfs + +# Setup a redacted send using a redaction list at varying depth. +log_must zfs redact $sendfs@snap0 book1 $POOL/rm@snap $POOL/stride3@snap \ + $POOL/stride5@snap +log_must eval "zfs send --redact book1 $sendfs@snap0 >$stream" +log_must eval "zfs receive $POOL2/rfs <$stream" + +# Verify receipt of normal incrementals to redaction list members. +log_must eval "zfs send -i $sendfs@snap0 $POOL/stride3@snap >$stream" +log_must eval "zfs recv $POOL2/rstride3 <$stream" +log_must diff -r /$POOL/stride3 /$POOL2/rstride3 +log_must eval "zfs send -i $sendfs@snap0 $POOL/stride5@snap >$stream" +log_must eval "zfs recv $POOL2/rstride5 <$stream" +log_must diff -r /$POOL/stride5 /$POOL2/rstride5 + +# But not a normal child that we weren't redacted with respect to. +log_must eval "zfs send -i $sendfs@snap0 $POOL/hole@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rhole@snap <$stream" + +# Verify we can receive an intermediate clone redacted with respect to a +# subset of the original redaction list. +log_must zfs redact $POOL/int@snap book2 $POOL/rm@snap +log_must eval "zfs send -i $sendfs@snap0 --redact book2 $POOL/int@snap >$stream" +log_must eval "zfs recv $POOL2/rint <$stream" +compare_files $POOL/int $POOL2/rint "f1" "$RANGE0" +compare_files $POOL/int $POOL2/rint "f2" "$RANGE15" +compare_files $POOL/int $POOL2/rint "d1/f1" "$RANGE16" +log_must mount_redacted -f $POOL2/rint + +# Verify we can receive grandchildren on the child. +log_must eval "zfs send -i $POOL/int@snap $POOL/rm@snap >$stream" +log_must eval "zfs receive $POOL2/rrm <$stream" +log_must diff -r /$POOL/rm /$POOL2/rrm + +# But not a grandchild that the received child wasn't redacted with respect to. +log_must eval "zfs send -i $POOL/int@snap $POOL/write@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rwrite<$stream" + +# Verify we cannot receive an intermediate clone that isn't redacted with +# respect to a subset of the original redaction list. +log_must zfs redact $POOL/int@snap book4 $POOL/rm@snap $POOL/write@snap +log_must eval "zfs send -i $sendfs@snap0 --redact book4 $POOL/int@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rint <$stream" +log_must zfs redact $POOL/int@snap book5 $POOL/write@snap +log_must eval "zfs send -i $sendfs@snap0 --redact book5 $POOL/int@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rint <$stream" +log_mustnot zfs redact $POOL/int@snap book6 $POOL/hole@snap + +# Verify we can receive a full clone of the grandchild on the child. +log_must eval "zfs send $POOL/write@snap >$stream" +log_must eval "zfs recv -o origin=$POOL2/rint@snap $POOL2/rwrite <$stream" +log_must diff -r /$POOL/write /$POOL2/rwrite + +# Along with other origins. +log_must eval "zfs recv -o origin=$POOL2/rfs@snap0 $POOL2/rwrite1 <$stream" +log_must diff -r /$POOL/write /$POOL2/rwrite1 +log_must eval "zfs recv -o origin=$POOL2@init $POOL2/rwrite2 <$stream" +log_must diff -r /$POOL/write /$POOL2/rwrite2 +log_must zfs destroy -R $POOL2/rwrite2 + +log_must zfs destroy -R $POOL2/rfs + +# Write some data for tests of incremental sends from bookmarks +log_must zfs snapshot $sendfs@snap1 +log_must zfs clone $sendfs@snap1 $POOL/hole1 +typeset mntpnt=$(get_prop mountpoint $POOL/hole1) +log_must dd if=/dev/zero of=$mntpnt/f2 bs=128k count=16 conv=notrunc +log_must zfs snapshot $POOL/hole1@snap +log_must zfs clone $sendfs@snap1 $POOL/write1 +mntpnt=$(get_prop mountpoint $POOL/write1) +log_must dd if=/dev/urandom of=$mntpnt/f2 bs=128k count=16 conv=notrunc +log_must zfs snapshot $POOL/write1@snap +log_must zfs clone $POOL/int@snap $POOL/write2 +mntpnt=$(get_prop mountpoint $POOL/write2) +log_must dd if=/dev/urandom of=$mntpnt/f2 bs=128k count=16 conv=notrunc +log_must zfs snapshot $POOL/write2@snap + +# Setup a redacted send using a redaction list at varying depth. +log_must zfs redact $sendfs@snap0 book7 $POOL/rm@snap $POOL/stride3@snap \ + $POOL/stride5@snap +log_must eval "zfs send --redact book7 $sendfs@snap0 >$stream" +log_must eval "zfs receive $POOL2/rfs <$stream" + +# Verify we can receive a redacted incremental sending from the bookmark. +log_must zfs redact $sendfs@snap1 book8 $POOL/write1@snap +log_must eval "zfs send -i $sendfs#book7 --redact book8 $sendfs@snap1 >$stream" +log_must eval "zfs receive $POOL2/rfs <$stream" +# The stride3 and stride5 snaps redact 3 128k blocks at block offsets 0 15 and +# 30 of f2. The write1 snap only covers the first two of those three blocks. +compare_files $sendfs $POOL2/rfs "f2" "$RANGE12" +log_must mount_redacted -f $POOL2/rfs +log_must diff $send_mnt/f1 /$POOL2/rfs/f1 +log_must diff $send_mnt/d1/f1 /$POOL2/rfs/d1/f1 +unmount_redacted $POOL2/rfs + +# Verify we can receive a normal child we weren't redacted with respect to by +# sending from the bookmark. +log_must eval "zfs send -i $sendfs#book7 $POOL/hole1@snap >$stream" +log_must eval "zfs recv $POOL2/rhole1 <$stream" +log_must diff -r /$POOL/hole1 /$POOL2/rhole1 + +# Verify we can receive an intermediate clone redacted with respect to a +# non-subset if we send from the bookmark. +log_must zfs redact $POOL/int@snap book9 $POOL/write2@snap +log_must eval "zfs send -i $sendfs#book7 --redact book9 $POOL/int@snap >$stream" +log_must eval "zfs receive $POOL2/rint <$stream" +compare_files $sendfs $POOL2/rint "f2" "$RANGE12" + +log_pass "Incrementals (redacted and normal) work with redacted datasets." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh new file mode 100755 index 000000000..caccdd360 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh @@ -0,0 +1,63 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify large blocks and redacted send work correctly together. +# +# Strategy: +# 1. Create a dataset and clone with a 1m recordsize, modifying a few k +# within the first 1m of a 16m file. +# 2. Verify that the whole first 1m of the file is redacted. +# 3. Receive an incremental stream from the original snap to the snap it +# was redacted with respect to. +# 4. Verify that the received dataset matches the clone +# + +typeset ds_name="largeblocks" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '-o recsize=1m' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must dd if=/dev/urandom of=$clone_mnt/f1 bs=32k count=3 seek=8 conv=notrunc +log_must zfs snapshot $clone@snap1 + +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send -L --redact book1 $sendfs@snap >$stream" +log_must stream_has_features $stream redacted large_blocks +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE11" +log_must mount_redacted -f $recvfs +log_must diff $send_mnt/f2 $recv_mnt/f2 +unmount_redacted $recvfs + +log_must eval "zfs send -L -i $sendfs@snap $clone@snap1 >$stream" +log_must stream_has_features $stream large_blocks +log_must eval "zfs recv $recvfs/new <$stream" +log_must diff -r $clone_mnt $recv_mnt/new + +log_pass "Large blocks and redacted send work correctly together." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh new file mode 100755 index 000000000..3386643b2 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh @@ -0,0 +1,68 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redacted send can deal with a large redaction list. +# +# Strategy: +# 1. Create 64 clones of sendfs each of which modifies two blocks in a file. +# The first modification is at an offset unique to each clone, and the +# second (the last block in the file) is common to them all. +# 2. Verify a redacted stream with a reasonable redaction list length can +# be correctly processed. +# 3. Verify that if the list is too long, the send fails gracefully. +# + +typeset ds_name="many_clones" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset redaction_list='' +typeset mntpnt + +log_onexit redacted_cleanup $sendfs $recvfs + +# Fill in both the last block, and a different block in every clone. +for i in {1..64}; do + log_must zfs clone $sendfs@snap ${clone}$i + mntpnt=$(get_prop mountpoint ${clone}$i) + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=$i \ + conv=notrunc + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=63 \ + conv=notrunc + log_must zfs snapshot ${clone}$i@snap +done + +# The limit isn't necessarily 32 snapshots. The maximum number of snapshots in +# the redacted list is determined in dsl_bookmark_create_redacted_check(). +log_must zfs redact $sendfs@snap book1 $clone{1..32}@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE8" + +log_mustnot zfs redact $sendfs@snap book2 $clone{1..64}@snap + +log_pass "Redacted send can deal with a large redaction list." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh new file mode 100755 index 000000000..e1cd09e17 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redacted send works with datasets of different sizes. +# +# Strategy: +# 1. Create two dataset one with recsize 512, and one 1m and create a 2m file. +# 2. For each dataset, create clones of both 512 and 1m recsize and modify +# the first 16k of the file. +# 3. Send each original dataset, redacted with respect to each of the clones +# into both a dataset inheriting a 512 recsize and a 1m one. +# 4. Verify that the smallest unit of redaction is that of the origin fs. +# + +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +typeset mntpnt + +log_onexit redacted_cleanup $POOL/512 $POOL/1m $POOL2/512 $POOL2/1m + +# Set up the datasets we'll send and redact from. +log_must zfs create -o recsize=512 $POOL/512 +mntpnt=$(get_prop mountpoint $POOL/512) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=1024k count=2 +log_must zfs snapshot $POOL/512@snap +log_must zfs clone -o recsize=1m $POOL/512@snap $POOL/1mclone +mntpnt=$(get_prop mountpoint $POOL/1mclone) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=32 conv=notrunc +log_must zfs snapshot $POOL/1mclone@snap + +log_must zfs create -o recsize=1m $POOL/1m +mntpnt=$(get_prop mountpoint $POOL/1m) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=1024k count=2 +log_must zfs snapshot $POOL/1m@snap +log_must zfs clone -o recsize=512 $POOL/1m@snap $POOL/512clone +mntpnt=$(get_prop mountpoint $POOL/512clone) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=32 conv=notrunc +log_must zfs snapshot $POOL/512clone@snap + +# Create datasets that allow received datasets to inherit recordsize. +log_must zfs create -o recsize=512 $POOL2/512 +log_must zfs create -o recsize=1m $POOL2/1m + +# Do the sends and verify the contents. +log_must zfs redact $POOL/512@snap book1 $POOL/1mclone@snap +log_must eval "zfs send --redact book1 $POOL/512@snap>$stream" +log_must eval "zfs recv $POOL2/512/recva <$stream" +compare_files $POOL/512 $POOL2/512/recva "f1" "$RANGE13" +log_must eval "zfs recv $POOL2/1m/recvb <$stream" +compare_files $POOL/512 $POOL2/1m/recvb "f1" "$RANGE13" + +log_must zfs redact $POOL/1m@snap book2 $POOL/512clone@snap +log_must eval "zfs send --redact book2 $POOL/1m@snap >$stream" +log_must eval "zfs recv $POOL2/512/recvc <$stream" +compare_files $POOL/1m $POOL2/512/recvc "f1" "$RANGE11" +log_must eval "zfs recv $POOL2/1m/recvd <$stream" +compare_files $POOL/1m $POOL2/1m/recvd "f1" "$RANGE11" + +log_pass "Redaction works correctly with different recordsizes." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh new file mode 100755 index 000000000..b24f5ccad --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh @@ -0,0 +1,109 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that received redacted datasets are not mounted by default, but +# can still be mounted after setting zfs_allow_redacted_dataset_mount. +# +# Strategy: +# 1. Verify a received redacted stream isn't mounted by default. +# 2. Set zfs_allow_redacted_dataset_mount and verify it can't be mounted +# without the -f flag, but can with -f. +# 3. Receive a redacted volume. +# 4. Verify the device file isn't present until the kernel variable is set. +# 5. Verify the files in the send fs are also present in the recv fs. +# + +typeset ds_name="mounts" +typeset sendfs="$POOL/$ds_name" +typeset sendvol="$sendfs/vol" +typeset recvfs="$POOL2/$ds_name" +typeset recvvol="$POOL2/vol" +typeset clone="$POOL/${ds_name}_clone" +typeset clonevol="${sendvol}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_mounts +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset recv_vol_file="/dev/zvol/$recvvol" + +log_onexit redacted_cleanup $sendfs $recvfs $recvvol + +log_must rm $clone_mnt/empty $clone_mnt/contents1 +log_must dd if=/dev/urandom of=$clone_mnt/contents2 bs=512 count=1 conv=notrunc +log_must rm $clone_mnt/dir1/contents1 +log_must rm -rf $clone_mnt/dir1/dir2 +log_must dd if=/dev/urandom of=$clone_mnt/dir1/contents2 bs=512 count=1 \ + conv=notrunc +log_must dd if=/dev/urandom of=$clone_mnt/dir1/empty bs=512 count=1 +log_must zfs snapshot $clone@snap1 + +log_must zfs redact $sendfs@snap book1 $clone@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs receive $recvfs <$stream" +log_mustnot ismounted $recvfs +log_mustnot mount_redacted $recvfs +log_mustnot ismounted $recvfs +log_must mount_redacted -f $recvfs +log_must ismounted $recvfs + +# Verify that the send and recv fs both have the same files under their +# mountpoints by comparing find output with the name of the mountpoint +# deleted. +contents=$(log_must find $recv_mnt) +contents_orig=$(log_must find $send_mnt) +log_must diff <(echo ${contents//$recv_mnt/}) \ + <(echo ${contents_orig//$send_mnt/}) +log_must zfs redact $sendvol@snap book2 $clonevol@snap +log_must eval "zfs send --redact book2 $sendvol@snap >$stream" +log_must eval "zfs receive $recvvol <$stream" +[[ -b $recv_vol_file ]] && log_fail "Volume device file should not exist." +log_must set_tunable32 zfs_allow_redacted_dataset_mount 1 +log_must zpool export $POOL2 +log_must zpool import $POOL2 +udevadm settle + +# The device file isn't guaranteed to show up right away. +if [[ ! -b $recv_vol_file ]]; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + [[ -b $recv_vol_file ]] && break + done +fi +[[ -b $recv_vol_file ]] || log_fail "Volume device file should exist." + +log_must dd if=/dev/urandom of=$send_mnt/dir1/contents1 bs=512 count=2 +log_must rm $send_mnt/dir1/dir2/empty +log_must zfs snapshot $sendfs@snap2 +log_must eval "zfs send -i $sendfs#book1 $sendfs@snap2 >$stream" +log_must eval "zfs receive $recvfs <$stream" +log_must mount_redacted -f $recvfs +log_must ismounted $recvfs +contents=$(log_must find $recv_mnt) +contents_orig=$(log_must find $send_mnt) +log_must diff <(echo ${contents//$recv_mnt/}) \ + <(echo ${contents_orig//$send_mnt/}) + +log_pass "Received redacted streams can be mounted." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh new file mode 100755 index 000000000..e27eb601e --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh @@ -0,0 +1,80 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Test that redacted send correctly detects invalid arguments. +# + +typeset sendfs="$POOL2/sendfs" +typeset recvfs="$POOL2/recvfs" +typeset clone1="$POOL2/clone1" +typeset clone2="$POOL2/clone2" +typeset clone3="$POOL2/clone3" +typeset clone3="$POOL2/clone4" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) + +log_onexit redacted_cleanup $sendfs $recvfs $clone3 + +log_must zfs create $sendfs +log_must zfs snapshot $sendfs@snap1 +log_must zfs snapshot $sendfs@snap2 +log_must zfs snapshot $sendfs@snap3 +log_must zfs clone $sendfs@snap2 $clone1 +log_must zfs snapshot $clone1@snap +log_must zfs bookmark $clone1@snap $clone1#book +log_must zfs clone $sendfs@snap2 $clone2 +log_must zfs snapshot $clone2@snap + +# Incompatible flags +log_must zfs redact $sendfs@snap2 book $clone1@snap +log_mustnot eval "zfs send -R --redact book $sendfs@snap2 >/dev/null" + +typeset arg +for arg in "$sendfs" "$clone1#book"; do + log_mustnot eval "zfs send --redact book $arg >/dev/null" +done + +# Bad redaction list arguments +log_mustnot zfs redact $sendfs@snap1 +log_mustnot zfs redact $sendfs@snap1 book +log_mustnot zfs redact $sendfs#book1 book4 $clone1 +log_mustnot eval "zfs send --redact $sendfs#book $sendfs@snap >/dev/null" + +# Redaction snapshots not a descendant of tosnap +log_mustnot zfs redact $sendfs@snap2 book $sendfs@snap2 +log_must zfs redact $sendfs@snap2 book2 $clone1@snap $clone2@snap +log_must eval "zfs send --redact book2 $sendfs@snap2 >$stream" +log_must zfs redact $sendfs@snap2 book3 $clone1@snap $clone2@snap +log_must eval "zfs send -i $sendfs@snap1 --redact book3 $sendfs@snap2 \ + >/dev/null" +log_mustnot zfs redact $sendfs@snap3 $sendfs@snap3 $clone1@snap + +# Full redacted sends of redacted datasets are not allowed. +log_must eval "zfs recv $recvfs <$stream" +log_must zfs snapshot $recvfs@snap +log_must zfs clone $recvfs@snap $clone3 +log_must zfs snapshot $clone3@snap +log_mustnot zfs redact $recvfs@snap book5 $clone3@snap + +# Nor may a redacted dataset appear in the redaction list. +log_mustnot zfs redact testpool2/recvfs@snap2 book7 testpool2/recvfs@snap + +log_pass "Verify that redacted send correctly detects invalid arguments." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh new file mode 100755 index 000000000..74e5914f2 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh @@ -0,0 +1,87 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Test that receiving sends from redaction bookmarks and redacted datasets +# works correctly in certain edge cases. +# 1. Send A(B,C,D) to pool2. +# 2. Verify send from A(B, C, D) can be received onto it. +# 3. Verify send from A(B, C) can be received onto it. +# 4. Verify send from A() can be received onto it. +# 5. Verify send from A(E) cannot be received onto it. +# 6. Verify send from redaction bookmark for A(B, C) can be received onto it. +# 7. Verify send from redaction bookmark for A() can be received onto it. +# 8. Verify send from redaction bookmark for A(E) cannot be received onto it. +# + +typeset ds_name="origin" +typeset sendfs="$POOL/$ds_name" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_incrementals +typeset dsA=$sendfs@snap0 +typeset dsB=$POOL/hole@snap +typeset dsC=$POOL/rm@snap +typeset dsD=$POOL/write@snap +typeset dsE=$POOL/stride3@snap +typeset dsF=$POOL/stride5@snap +typeset targ=$POOL2/targfs@snap + +log_onexit redacted_cleanup $sendfs $POOL2/rBCD $POOL2/targfs \ + $POOL2/rBC $POOL2/rE + +# Set up all the filesystems and clones. +log_must zfs redact $dsA BCD $dsB $dsC $dsD +log_must eval "zfs send --redact BCD $dsA >$stream" +log_must eval "zfs receive $POOL2/rBCD <$stream" +log_must eval "zfs receive $targ <$stream" + +log_must zfs redact $dsA BC $dsB $dsC +log_must eval "zfs send --redact BC $dsA >$stream" +log_must eval "zfs receive $POOL2/rBC <$stream" + +log_must zfs redact $dsA E $dsE +log_must eval "zfs send --redact E $dsA >$stream" +log_must eval "zfs receive $POOL2/rE <$stream" + +log_must eval "zfs send $dsF >$stream" +log_must eval "zfs receive -o origin=$POOL2/rBCD@snap0 $POOL2/BCDrF <$stream" +log_must eval "zfs receive -o origin=$POOL2/rBC@snap0 $POOL2/BCrF <$stream" +log_must eval "zfs receive -o origin=$POOL2/rE@snap0 $POOL2/ErF <$stream" + +# Run tests from redacted datasets. +log_must eval "zfs send -i $POOL2/rBCD@snap0 $POOL2/BCDrF@snap >$stream" +log_must eval "zfs receive -o origin=$targ $POOL2/tdBCD <$stream" + +log_must eval "zfs send -i $POOL2/rBC@snap0 $POOL2/BCrF@snap >$stream" +log_must eval "zfs receive -o origin=$targ $POOL2/tdBC <$stream" + +log_must eval "zfs send -i $POOL2/rE@snap0 $POOL2/ErF@snap >$stream" +log_mustnot eval "zfs receive -o origin=$targ $POOL2/tdE <$stream" + +# Run tests from redaction bookmarks. +log_must eval "zfs send -i $sendfs#BC $dsF >$stream" +log_must eval "zfs receive -o origin=$targ $POOL2/tbBC <$stream" + +log_must eval "zfs send -i $sendfs#E $dsF >$stream" +log_mustnot eval "zfs receive -o origin=$targ $POOL2/tbE <$stream" + +log_pass "Verify sends from redacted datasets and bookmarks work correctly." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh new file mode 100755 index 000000000..e4163c4ef --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify the list of redacted snapshot guids as properties. +# +# Strategy: +# 1. Create a redacted dataset and receive it into another pool. +# 2. Verify that the redaction list in the book mark (according to zdb) +# matches the list shown in the redact_snaps property. +# 3. Verify that the received snapshot has a matching redaction list. +# + +typeset ds_name="props" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset mntpnt + +log_onexit redacted_cleanup $sendfs $recvfs + +# Verify a plain dataset, snapshot or bookmark has an empty list. +log_must zfs snapshot $sendfs@empty_snapshot +log_must zfs bookmark $sendfs@empty_snapshot $sendfs#empty_bookmark +found_list=$(get_prop redact_snaps $sendfs) +[[ $found_list = "-" ]] || log_fail "Unexpected dataset list: $found_list" +found_list=$(get_prop redact_snaps $sendfs@empty_snapshot) +[[ $found_list = "-" ]] || log_fail "Unexpected snapshot list: $found_list" +found_list=$(get_prop redact_snaps $sendfs#empty_bookmark) +[[ $found_list = "-" ]] || log_fail "Unexpected bookmark list: $found_list" + +# Fill in a different block in every clone. +for i in {1..16}; do + log_must zfs clone $sendfs@snap ${clone}$i + mntpnt=$(get_prop mountpoint ${clone}$i) + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=$i \ + conv=notrunc + log_must zfs snapshot ${clone}$i@snap +done + +log_must zfs redact $sendfs@snap book1 $clone{1..16}@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" + +get_guid_list $tmpdir/prop_list $sendfs#book1 +get_guid_list $tmpdir/zdb_list $sendfs#book1 true +get_guid_list $tmpdir/recvd_prop_list $recvfs@snap + +count=$(wc -l $tmpdir/prop_list | awk '{print $1}') +[[ $count -eq 16 ]] || log_fail "Found incorrect number of redaction snapshots." + +diff $tmpdir/prop_list $tmpdir/zdb_list || \ + log_fail "Property list differed from zdb output" +diff $tmpdir/prop_list $tmpdir/recvd_prop_list || \ + log_fail "Received property list differed from sent" + +log_pass "The redaction list is consistent between sent and received datasets." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh new file mode 100755 index 000000000..9a3c5329d --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh @@ -0,0 +1,87 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that resumable send works correctly with redacted streams. +# +# Strategy: +# 1. Do a full redacted resumable send. +# 2. Verify the received contents are correct. +# 3. Do an incremental redacted resumable send. +# 4. Verify the received contents are correct. +# 5. Verify that recv -A removes a partially received dataset. +# + +typeset ds_name="resume" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset clone1="$POOL/${ds_name}_clone1" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must stride_dd -i /dev/urandom -o $clone_mnt/f2 -b 512 -c 64 -s 512 +log_must zfs snapshot $clone@snap1 + +# Do the full resumable send +log_must zfs redact $sendfs@snap book1 $clone@snap1 +resume_test "zfs send --redact book1 $sendfs@snap" $tmpdir $recvfs +log_must mount_redacted -f $recvfs +log_must set_tunable32 zfs_allow_redacted_dataset_mount 1 +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must eval "get_diff $send_mnt/f2 $recv_mnt/f2 >$tmpdir/get_diff.out" +typeset range=$(cat $tmpdir/get_diff.out) +[[ "$RANGE9" = "$range" ]] || log_fail "Unexpected range: $range" + +log_must dd if=/dev/urandom of=$send_mnt/f3 bs=1024k count=3 +log_must zfs snapshot $sendfs@snap2 +log_must zfs clone $sendfs@snap2 $clone1 +typeset clone1_mnt="$(get_prop mountpoint $clone1)" +log_must dd if=/dev/urandom of=$clone1_mnt/f3 bs=128k count=3 conv=notrunc +log_must zfs snapshot $clone1@snap + +# Do the incremental resumable send +log_must zfs redact $sendfs@snap2 book2 $clone1@snap +resume_test "zfs send --redact book2 -i $sendfs#book1 $sendfs@snap2" \ + $tmpdir $recvfs +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must diff $send_mnt/f2 $recv_mnt/f2 +log_must eval "get_diff $send_mnt/f3 $recv_mnt/f3 >$tmpdir/get_diff.out" +range=$(cat $tmpdir/get_diff.out) +[[ "$RANGE10" = "$range" ]] || log_fail "Unexpected range: $range" + +# Test recv -A works properly +log_mustnot zfs recv -A $recvfs +log_must zfs destroy -R $recvfs +log_mustnot zfs recv -A $recvfs +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +dd if=$stream bs=64k count=1 | log_mustnot zfs receive -s $recvfs +[[ "-" = $(get_prop receive_resume_token $recvfs) ]] && \ + log_fail "Receive token not found." +log_must zfs recv -A $recvfs +log_must datasetnonexists $recvfs + +log_pass "Resumable send works correctly with redacted streams." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh new file mode 100755 index 000000000..81e7fe31d --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that send size estimates of redacted sends work correctly +# +# Strategy: +# 1. Perform a redacted send with -nv and without, and verify the +# size estimate is the same as the size of the actual send. +# 2. Receive an incremental send from the redaction bookmark with +# -nv and without, and verify the size estimate is the same as +# the size of the actual send. +# + +ds_name="sizes" +typeset sendfs="$POOL/$ds_name" +typeset clone="$POOL/${ds_name}_clone2" +setup_dataset $ds_name "-o compress=lz4" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset size=$(mktemp $tmpdir/size.XXXX) +typeset size2=$(mktemp $tmpdir/size.XXXX) + +log_onexit redacted_cleanup $sendfs $clone +log_must zfs clone $sendfs@snap $clone +typeset clone_mnt="$(get_prop mountpoint $clone)" +log_must rm -rf $clone_mnt/* +log_must zfs snapshot $clone@snap +log_must zfs redact $sendfs@snap book $clone@snap +log_must eval "zfs send -nvP --redact book $sendfs@snap | \ + grep '^size' | awk '{print \$2}' >$size" +log_must eval "zfs send --redact book $sendfs@snap | wc --bytes \ + >$size2" +bytes1=$(cat $size | tr -d '[[:space:]]') +bytes2=$(cat $size2 | tr -d '[[:space:]]') +[[ "$bytes1" -eq "$bytes2" ]] || \ + log_fail "Full sizes differ: estimate $bytes1 and actual $bytes2" + +log_must zfs snapshot $sendfs@snap2 +log_must eval "zfs send -nvP -i $sendfs#book $sendfs@snap2 | \ + grep '^size' | awk '{print \$2}' >$size" +log_must eval "zfs send -i $sendfs#book $sendfs@snap2 | wc --bytes >$size2" +bytes1=$(cat $size | tr -d '[[:space:]]') +bytes2=$(cat $size2 | tr -d '[[:space:]]') +[[ "$bytes1" -eq "$bytes2" ]] || \ + log_fail "Incremental sizes differ: estimate $bytes1 and actual $bytes2" + +log_pass "Size estimates of redacted sends estimate accurately." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh new file mode 100755 index 000000000..b3c519b7d --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh @@ -0,0 +1,105 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that redacted send works on volumes. +# +# Strategy: +# 1. Write to a volume, then make a clone of that volume. +# 2. Receive a redacted stream that sends all blocks. +# 3. Receive a redacted stream that redacts the first half of the written area. +# + +typeset ds_name="volume" +typeset sendvol="$POOL/$ds_name" +typeset recvvol="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +typeset send_file="/dev/zvol/$sendvol" +typeset recv_file="/dev/zvol/$recvvol" +typeset clone_file="/dev/zvol/$clone" + +log_onexit redacted_cleanup $sendvol $recvvol + +log_must zfs create -b 8k -V 1g $sendvol +sleep 10 +log_must zpool export $POOL +log_must zpool import $POOL +udevadm settle +if [[ ! -b $send_file ]]; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + [[ -b $send_file ]] && break + done +fi +log_must dd if=/dev/urandom of=$send_file bs=8k count=64 +log_must zfs snapshot $sendvol@snap +log_must zfs clone $sendvol@snap $clone +log_must zfs snapshot $clone@snap + +log_must set_tunable32 zfs_allow_redacted_dataset_mount 1 +log_must zfs redact $sendvol@snap book1 $clone@snap +log_must eval "zfs send --redact book1 $sendvol@snap >$stream" +log_must eval "zfs recv $recvvol <$stream" +sleep 10 +log_must zpool export $POOL2 +log_must zpool import $POOL2 +udevadm settle +if [[ ! -b $recv_file ]]; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + [[ -b $recv_file ]] && break + done +fi +log_must dd if=$send_file of=$tmpdir/send.dd bs=8k count=64 +log_must dd if=$recv_file of=$tmpdir/recv.dd bs=8k count=64 +log_must diff $tmpdir/send.dd $tmpdir/recv.dd +log_must zfs destroy -R $recvvol + +log_must dd if=/dev/urandom of=$clone_file bs=8k count=32 +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendvol@snap book2 $clone@snap1 +log_must eval "zfs send --redact book2 $sendvol@snap >$stream" +log_must eval "zfs recv $recvvol <$stream" +sleep 10 +log_must zpool export $POOL2 +log_must zpool import $POOL2 +udevadm settle +if [[ ! -b $recv_file ]]; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + [[ -b $recv_file ]] && break + done +fi +log_must dd if=$send_file of=$tmpdir/send.dd bs=8k count=32 skip=32 +log_must dd if=$recv_file of=$tmpdir/recv.dd bs=8k count=32 skip=32 +log_must diff $tmpdir/send.dd $tmpdir/recv.dd + +log_pass "Redacted send works correctly with volumes." diff --git a/tests/zfs-tests/tests/functional/redacted_send/setup.ksh b/tests/zfs-tests/tests/functional/redacted_send/setup.ksh new file mode 100755 index 000000000..3f537f813 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/setup.ksh @@ -0,0 +1,36 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +verify_disk_count "$DISKS" 2 + +create_pool $POOL $DISK1 +log_must zfs snapshot $POOL@init +create_pool $POOL2 $DISK2 +log_must zfs snapshot $POOL2@init +log_must zfs create $POOL/tmp +log_pass diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib index 521a1c7eb..e6fc9b7d6 100644 --- a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib +++ b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib @@ -25,7 +25,7 @@ # # -# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# Copyright (c) 2013, 2018 by Delphix. All rights reserved. # . $STF_SUITE/include/libtest.shlib @@ -635,7 +635,7 @@ function resume_test stream_num=$((stream_num+1)) token=$(zfs get -Hp -o value receive_resume_token $recvfs) - log_must eval "zfs send -v -t $token >/$streamfs/$stream_num" + log_must eval "zfs send -t $token >/$streamfs/$stream_num" [[ -f /$streamfs/$stream_num ]] || \ log_fail "NO FILE /$streamfs/$stream_num" done diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh new file mode 100755 index 000000000..4610802e9 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh @@ -0,0 +1,33 @@ +#!/usr/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2014, 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that error conditions don't cause panics in zfs send +# +# Strategy: +# 1. Perform a zfs incremental send from a bookmark that doesn't exist +# + +verify_runnable "both" + +log_neg eval "zfs send -i \#bla $POOl/$FS@final > /dev/null" + +log_pass "Ensure that error conditions cause appropriate failures." |