diff options
Diffstat (limited to 'tests')
39 files changed, 2559 insertions, 7 deletions
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 22fc26212..e45868765 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -735,6 +735,15 @@ tests = ['quota_001_pos', 'quota_002_pos', 'quota_003_pos', 'quota_004_pos', 'quota_005_pos', 'quota_006_neg'] tags = ['functional', 'quota'] +[tests/functional/redacted_send] +tests = ['redacted_compressed', 'redacted_contents', 'redacted_deleted', + 'redacted_disabled_feature', 'redacted_embedded', 'redacted_holes', + 'redacted_incrementals', 'redacted_largeblocks', 'redacted_many_clones', + 'redacted_mixed_recsize', 'redacted_mounts', 'redacted_negative', + 'redacted_origin', 'redacted_props', 'redacted_resume', 'redacted_size', + 'redacted_volume'] +tags = ['functional', 'redacted_send'] + [tests/functional/raidz] tests = ['raidz_001_neg', 'raidz_002_pos'] tags = ['functional', 'raidz'] diff --git a/tests/zfs-tests/cmd/Makefile.am b/tests/zfs-tests/cmd/Makefile.am index 39a538d2d..09c59f591 100644 --- a/tests/zfs-tests/cmd/Makefile.am +++ b/tests/zfs-tests/cmd/Makefile.am @@ -8,6 +8,7 @@ SUBDIRS = \ file_check \ file_trunc \ file_write \ + get_diff \ largest_file \ libzfs_input_check \ mkbusy \ @@ -24,4 +25,5 @@ SUBDIRS = \ rename_dir \ rm_lnkcnt_zero_file \ threadsappend \ - xattrtest + xattrtest \ + stride_dd diff --git a/tests/zfs-tests/cmd/get_diff/Makefile.am b/tests/zfs-tests/cmd/get_diff/Makefile.am new file mode 100644 index 000000000..06c39ddd8 --- /dev/null +++ b/tests/zfs-tests/cmd/get_diff/Makefile.am @@ -0,0 +1,6 @@ +include $(top_srcdir)/config/Rules.am + +pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/bin + +pkgexec_PROGRAMS = get_diff +get_diff_SOURCES = get_diff.c diff --git a/tests/zfs-tests/cmd/get_diff/get_diff.c b/tests/zfs-tests/cmd/get_diff/get_diff.c new file mode 100644 index 000000000..2799f46b0 --- /dev/null +++ b/tests/zfs-tests/cmd/get_diff/get_diff.c @@ -0,0 +1,109 @@ +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2018 by Delphix. All rights reserved. + */ + +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <sys/param.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> + +static void +usage(char *msg, int exit_value) +{ + (void) fprintf(stderr, "get_diff file redacted_file\n"); + (void) fprintf(stderr, "%s\n", msg); + exit(exit_value); +} + +/* + * This utility compares two files, an original and its redacted counterpart + * (in that order). It compares the files 512 bytes at a time, printing out + * any ranges (as offset and length) where the redacted file does not match + * the original. This output is used to verify that the expected ranges of + * a redacted file do not contain the original data. + */ +int +main(int argc, char *argv[]) +{ + off_t diff_off = 0, diff_len = 0, off = 0; + int fd1, fd2; + char *fname1, *fname2; + char buf1[DEV_BSIZE], buf2[DEV_BSIZE]; + ssize_t bytes; + + if (argc != 3) + usage("Incorrect number of arguments.", 1); + + if ((fname1 = argv[1]) == NULL) + usage("Filename missing.", 1); + if ((fd1 = open(fname1, O_LARGEFILE | O_RDONLY)) < 0) { + perror("open1 failed"); + exit(1); + } + + if ((fname2 = argv[2]) == NULL) + usage("Redacted filename missing.", 1); + if ((fd2 = open(fname2, O_LARGEFILE | O_RDONLY)) < 0) { + perror("open2 failed"); + exit(1); + } + + while ((bytes = pread(fd1, buf1, DEV_BSIZE, off)) > 0) { + if (pread(fd2, buf2, DEV_BSIZE, off) < 0) { + if (errno == EIO) { + /* + * A read in a redacted section of a file will + * fail with EIO. If we get EIO, continue on + * but ensure that a comparison of buf1 and + * buf2 will fail, indicating a redacted block. + */ + buf2[0] = ~buf1[0]; + } else { + perror("pread failed"); + exit(1); + } + } + if (memcmp(buf1, buf2, bytes) == 0) { + if (diff_len != 0) { + (void) fprintf(stdout, "%lld,%lld\n", + (long long)diff_off, (long long)diff_len); + assert(off == diff_off + diff_len); + diff_len = 0; + } + diff_off = 0; + } else { + if (diff_len == 0) + diff_off = off; + assert(off == diff_off + diff_len); + diff_len += bytes; + } + off += bytes; + } + + if (diff_len != 0 && diff_len != 0) { + (void) fprintf(stdout, "%lld,%lld\n", (long long)diff_off, + (long long)diff_len); + } + + (void) close(fd1); + (void) close(fd2); + + return (0); +} diff --git a/tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c b/tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c index 977b9e2f3..2de1ba20c 100644 --- a/tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c +++ b/tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c @@ -691,6 +691,34 @@ zfs_destroy(const char *dataset) } static void +test_redact(const char *snapshot1, const char *snapshot2) +{ + nvlist_t *required = fnvlist_alloc(); + nvlist_t *snapnv = fnvlist_alloc(); + char bookmark[MAXNAMELEN + 32]; + + fnvlist_add_string(required, "bookname", "testbookmark"); + fnvlist_add_boolean(snapnv, snapshot2); + fnvlist_add_nvlist(required, "snapnv", snapnv); + + IOC_INPUT_TEST(ZFS_IOC_REDACT, snapshot1, required, NULL, 0); + + nvlist_free(snapnv); + nvlist_free(required); + + strncpy(bookmark, snapshot1, sizeof (bookmark) - 1); + *strchr(bookmark, '@') = '\0'; + strncat(bookmark, "#testbookmark", sizeof (bookmark)); + zfs_destroy(bookmark); +} + +static void +test_get_bookmark_props(const char *bookmark) +{ + IOC_INPUT_TEST(ZFS_IOC_GET_BOOKMARK_PROPS, bookmark, NULL, NULL, 0); +} + +static void zfs_ioc_input_tests(const char *pool) { char filepath[] = "/tmp/ioc_test_file_XXXXXX"; @@ -700,6 +728,7 @@ zfs_ioc_input_tests(const char *pool) char bookmark[ZFS_MAX_DATASET_NAME_LEN + 32]; char backup[ZFS_MAX_DATASET_NAME_LEN]; char clone[ZFS_MAX_DATASET_NAME_LEN]; + char clonesnap[ZFS_MAX_DATASET_NAME_LEN + 32]; int tmpfd, err; /* @@ -710,6 +739,7 @@ zfs_ioc_input_tests(const char *pool) (void) snprintf(snapshot, sizeof (snapshot), "%s@snapshot", dataset); (void) snprintf(bookmark, sizeof (bookmark), "%s#bookmark", dataset); (void) snprintf(clone, sizeof (clone), "%s/test-fs-clone", pool); + (void) snprintf(clonesnap, sizeof (clonesnap), "%s@snap", clone); (void) snprintf(backup, sizeof (backup), "%s/backup", pool); err = lzc_create(dataset, DMU_OST_ZFS, NULL, NULL, 0); @@ -747,6 +777,7 @@ zfs_ioc_input_tests(const char *pool) test_bookmark(pool, snapshot, bookmark); test_get_bookmarks(dataset); + test_get_bookmark_props(bookmark); test_destroy_bookmarks(pool, bookmark); test_hold(pool, snapshot); @@ -754,6 +785,9 @@ zfs_ioc_input_tests(const char *pool) test_release(pool, snapshot); test_clone(snapshot, clone); + test_snapshot(pool, clonesnap); + test_redact(snapshot, clonesnap); + zfs_destroy(clonesnap); zfs_destroy(clone); test_rollback(dataset, snapshot); @@ -909,6 +943,8 @@ validate_ioc_values(void) ZFS_IOC_BASE + 78 == ZFS_IOC_POOL_DISCARD_CHECKPOINT && ZFS_IOC_BASE + 79 == ZFS_IOC_POOL_INITIALIZE && ZFS_IOC_BASE + 80 == ZFS_IOC_POOL_TRIM && + ZFS_IOC_BASE + 81 == ZFS_IOC_REDACT && + ZFS_IOC_BASE + 82 == ZFS_IOC_GET_BOOKMARK_PROPS && LINUX_IOC_BASE + 1 == ZFS_IOC_EVENTS_NEXT && LINUX_IOC_BASE + 2 == ZFS_IOC_EVENTS_CLEAR && LINUX_IOC_BASE + 3 == ZFS_IOC_EVENTS_SEEK); diff --git a/tests/zfs-tests/cmd/stride_dd/.gitignore b/tests/zfs-tests/cmd/stride_dd/.gitignore new file mode 100644 index 000000000..7c072ee0d --- /dev/null +++ b/tests/zfs-tests/cmd/stride_dd/.gitignore @@ -0,0 +1 @@ +/stride_dd diff --git a/tests/zfs-tests/cmd/stride_dd/Makefile.am b/tests/zfs-tests/cmd/stride_dd/Makefile.am new file mode 100644 index 000000000..d6f1adbac --- /dev/null +++ b/tests/zfs-tests/cmd/stride_dd/Makefile.am @@ -0,0 +1,7 @@ +include $(top_srcdir)/config/Rules.am + +pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/bin + +pkgexec_PROGRAMS = stride_dd +stride_dd_SOURCES = stride_dd.c +stride_dd_LDADD = -lrt diff --git a/tests/zfs-tests/cmd/stride_dd/stride_dd.c b/tests/zfs-tests/cmd/stride_dd/stride_dd.c new file mode 100644 index 000000000..88bd53292 --- /dev/null +++ b/tests/zfs-tests/cmd/stride_dd/stride_dd.c @@ -0,0 +1,214 @@ +/* + * This file and its contents are supplied under the terms of the + * Common Development and Distribution License ("CDDL"), version 1.0. + * You may only use this file in accordance with the terms of version + * 1.0 of the CDDL. + * + * A full copy of the text of the CDDL should have accompanied this + * source. A copy of the CDDL is also available via the Internet at + * http://www.illumos.org/license/CDDL. + */ + +/* + * Copyright (c) 2018 by Delphix. All rights reserved. + */ + +#include <sys/types.h> +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <unistd.h> +#include <stdlib.h> +#include <string.h> + +static int bsize = 0; +static int count = 0; +static char *ifile = NULL; +static char *ofile = NULL; +static int stride = 0; +static int seek = 0; +static char *execname = "stride_dd"; + +static void usage(void); +static void parse_options(int argc, char *argv[]); + +static void +usage(void) +{ + (void) fprintf(stderr, + "usage: %s -i inputfile -o outputfile -b blocksize -c count \n" + " -s stride [ -k seekblocks]\n" + "\n" + "Simplified version of dd that supports the stride option.\n" + "A stride of n means that for each block written, n - 1 blocks\n" + "are skipped in both the input and output file. A stride of 1\n" + "means that blocks are read and written consecutively.\n" + "All numeric parameters must be integers.\n" + "\n" + " inputfile: File to read from\n" + " outputfile: File to write to\n" + " blocksize: Size of each block to read/write\n" + " count: Number of blocks to read/write\n" + " stride: Read/write a block then skip (stride - 1) blocks\n" + " seekblocks: Number of blocks to skip at start of output\n", + execname); + (void) exit(1); +} + +static void +parse_options(int argc, char *argv[]) +{ + int c; + int errflag = 0; + + execname = argv[0]; + + extern char *optarg; + extern int optind, optopt; + + while ((c = getopt(argc, argv, ":b:c:i:o:s:k:")) != -1) { + switch (c) { + case 'b': + bsize = atoi(optarg); + break; + + case 'c': + count = atoi(optarg); + break; + + case 'i': + ifile = optarg; + break; + + case 'o': + ofile = optarg; + break; + + case 's': + stride = atoi(optarg); + break; + + case 'k': + seek = atoi(optarg); + break; + + case ':': + (void) fprintf(stderr, + "Option -%c requires an operand\n", optopt); + errflag++; + break; + + case '?': + default: + (void) fprintf(stderr, + "Unrecognized option: -%c\n", optopt); + errflag++; + break; + } + + if (errflag) { + (void) usage(); + } + } + + if (bsize <= 0 || count <= 0 || stride <= 0 || ifile == NULL || + ofile == NULL || seek < 0) { + (void) fprintf(stderr, + "Required parameter(s) missing or invalid.\n"); + (void) usage(); + } +} + +int +main(int argc, char *argv[]) +{ + int i; + int ifd; + int ofd; + void *buf; + int c; + + parse_options(argc, argv); + + ifd = open(ifile, O_RDONLY); + if (ifd == -1) { + (void) fprintf(stderr, "%s: %s: ", execname, ifile); + perror("open"); + exit(2); + } + + ofd = open(ofile, O_WRONLY | O_CREAT, 0666); + if (ofd == -1) { + (void) fprintf(stderr, "%s: %s: ", execname, ofile); + perror("open"); + exit(2); + } + + /* + * We use valloc because some character block devices expect a + * page-aligned buffer. + */ + int err = posix_memalign(&buf, 4096, bsize); + if (err != 0) { + (void) fprintf(stderr, + "%s: %s\n", execname, strerror(err)); + exit(2); + } + + if (seek > 0) { + if (lseek(ofd, seek * bsize, SEEK_CUR) == -1) { + perror("output lseek"); + exit(2); + } + } + + for (i = 0; i < count; i++) { + c = read(ifd, buf, bsize); + if (c != bsize) { + + perror("read"); + exit(2); + } + if (c != bsize) { + if (c < 0) { + perror("read"); + } else { + (void) fprintf(stderr, + "%s: unexpected short read, read %d " + "bytes, expected %d\n", execname, + c, bsize); + } + exit(2); + } + + c = write(ofd, buf, bsize); + if (c != bsize) { + if (c < 0) { + perror("write"); + } else { + (void) fprintf(stderr, + "%s: unexpected short write, wrote %d " + "bytes, expected %d\n", execname, + c, bsize); + } + exit(2); + } + + if (stride > 1) { + if (lseek(ifd, (stride - 1) * bsize, SEEK_CUR) == -1) { + perror("input lseek"); + exit(2); + } + if (lseek(ofd, (stride - 1) * bsize, SEEK_CUR) == -1) { + perror("output lseek"); + exit(2); + } + } + } + free(buf); + + (void) close(ofd); + (void) close(ifd); + + return (0); +} diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 127a1477d..f8ad02246 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -1,4 +1,5 @@ # +# Copyright (c) 2016, 2018 by Delphix. All rights reserved. # These variables are used by zfs-tests.sh to constrain which utilities # may be used by the suite. The suite will create a directory which is # the only element of $PATH and create symlinks from that dir to the @@ -163,6 +164,7 @@ export ZFSTEST_FILES='chg_usr_exec file_check file_trunc file_write + get_diff largest_file libzfs_input_check mkbusy @@ -180,4 +182,5 @@ export ZFSTEST_FILES='chg_usr_exec rm_lnkcnt_zero_file threadsappend user_ns_exec - xattrtest' + xattrtest + stride_dd' diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index 1b841d7ba..9c3c20fd9 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -405,7 +405,8 @@ function create_recv_clone log_must eval "zfs send $snap | zfs recv -u $recvfs" log_must mkfile 1m "$mountpoint/data" log_must zfs snapshot $incr - log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 > $sendfile" + log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 \ + iflag=fullblock > $sendfile" log_mustnot eval "zfs recv -su $recvfs < $sendfile" destroy_dataset "$sendfs" "-r" log_must rm -f "$sendfile" diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am index da27673ec..2d23eb296 100644 --- a/tests/zfs-tests/tests/functional/Makefile.am +++ b/tests/zfs-tests/tests/functional/Makefile.am @@ -52,6 +52,7 @@ SUBDIRS = \ projectquota \ quota \ raidz \ + redacted_send \ redundancy \ refquota \ refreserv \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh new file mode 100755 index 000000000..447fbb36b --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# When a snapshot is destroyed, we used to recurse all clones +# that are downstream of the destroyed snapshot (e.g. to remove +# its key and merge its deadlist entries to the previous one). +# This recursion would break the stack on deeply nested clone +# hierarchies. To avoid this problem today, we keep heap-allocated +# records of all the clones as we traverse their hierarchy. +# +# This test ensures and showcases that our new method works with +# deeply nested clone hierarchies. +# +# STRATEGY: +# 1. Create an fs and take a snapshot of it (snapshot foo) +# 2. Take a second snapshot of the same fs (snapshot bar) on +# top of snapshot foo +# 3. Create a clone of snapshot bar and then take a snapshot +# of it. +# 4. Create a clone of the newly-created snapshot and then +# take a snapshot of it. +# 5. Repeat step [4] many times to create a deeply nested hierarchy. +# 6. Destroy snapshot foo. +# + +verify_runnable "both" + +typeset FS0=$TESTPOOL/0 +typeset FOO=foo +typeset BAR=BAR + +typeset FS0SNAPFOO=$FS0@$FOO +typeset FS0SNAPBAR=$FS0@$BAR + +typeset -i numds=300 + +log_must zfs create $FS0 + +function test_cleanup +{ + log_must zfs destroy -Rf $FS0 + + return 0 +} + +log_must zfs snapshot $FS0SNAPFOO +log_must zfs snapshot $FS0SNAPBAR + +log_onexit test_cleanup + +for (( i=1; i<numds; i++ )); do + log_must zfs clone $TESTPOOL/$((i-1))@$BAR $TESTPOOL/$i + log_must zfs snapshot $TESTPOOL/$i@$BAR +done + +log_must zfs destroy $FS0SNAPFOO + +log_pass "Snapshot deletion doesn't break the stack in deeply nested " \ + "clone hierarchies." diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send-b.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send-b.ksh index cd879846c..ab2a9508e 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send-b.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send-b.ksh @@ -65,7 +65,6 @@ done for opt in ${opts[@]}; do log_mustnot eval "zfs send -b$opt $SENDFS > /dev/null" log_mustnot eval "zfs send -b$opt $SENDFS#bm > /dev/null" - log_mustnot eval "zfs send -b$opt -i $SENDFS#bm $SENDFS@s2 > /dev/null" done # Do 3..6 in a loop to verify various combination of "zfs send" options diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh index 7192551b6..5d3d7cf0d 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_send/zfs_send_006_pos.ksh @@ -15,7 +15,7 @@ # # -# Copyright (c) 2012, 2016 by Delphix. All rights reserved. +# Copyright (c) 2012, 2018 by Delphix. All rights reserved. # . $STF_SUITE/include/libtest.shlib @@ -36,6 +36,7 @@ verify_runnable "both" function cleanup { + log_must set_tunable32 zfs_override_estimate_recordsize 8192 for ds in $datasets; do destroy_dataset $ds "-rf" done @@ -90,6 +91,7 @@ function verify_size_estimates log_assert "Verify 'zfs send -nvP' generates valid stream estimates" log_onexit cleanup +log_must set_tunable32 zfs_override_estimate_recordsize 0 typeset -l block_count=0 typeset -l block_size typeset -i PERCENT=1 diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg index fdcce8b56..9e2f00f6a 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg @@ -77,6 +77,9 @@ typeset -a properties=( "feature@obsolete_counts" "feature@zpool_checkpoint" "feature@spacemap_v2" + "feature@redaction_bookmarks" + "feature@redacted_datasets" + "feature@bookmark_written" ) # Additional properties added for Linux. diff --git a/tests/zfs-tests/tests/functional/redacted_send/Makefile.am b/tests/zfs-tests/tests/functional/redacted_send/Makefile.am new file mode 100644 index 000000000..dd6b4eb67 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/Makefile.am @@ -0,0 +1,25 @@ +pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/redacted_send +dist_pkgdata_SCRIPTS = \ + setup.ksh \ + cleanup.ksh \ + redacted_compressed.ksh \ + redacted_contents.ksh \ + redacted_deleted.ksh \ + redacted_disabled_feature.ksh \ + redacted_embedded.ksh \ + redacted_holes.ksh \ + redacted_incrementals.ksh \ + redacted_largeblocks.ksh \ + redacted_many_clones.ksh \ + redacted_mixed_recsize.ksh \ + redacted_mounts.ksh \ + redacted_negative.ksh \ + redacted_origin.ksh \ + redacted_props.ksh \ + redacted_resume.ksh \ + redacted_size.ksh \ + redacted_volume.ksh + +dist_pkgdata_DATA = \ + redacted.cfg \ + redacted.kshlib diff --git a/tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh b/tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh new file mode 100755 index 000000000..596c661ed --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/cleanup.ksh @@ -0,0 +1,33 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +destroy_pool $POOL +destroy_pool $POOL2 +log_must set_tunable32 zfs_allow_redacted_dataset_mount 0 + +log_pass diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted.cfg b/tests/zfs-tests/tests/functional/redacted_send/redacted.cfg new file mode 100644 index 000000000..f964b37ba --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted.cfg @@ -0,0 +1,86 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +export DISK1=$(echo $DISKS | awk '{print $1}') +export DISK2=$(echo $DISKS | awk '{print $2}') + +export POOL=$TESTPOOL +export POOL2=$TESTPOOL2 +export FS=$TESTFS +export FS2=$TESTFS2 + +# +# These are the byte ranges that differ between files and their redacted +# counterparts. See compare_files() for more detail. +# +typeset RANGE0="0,2097152" +typeset RANGE1="0,131072" +typeset RANGE2="1048576,2097152" +typeset RANGE3="0,131072 +1966080,131072 +3932160,131072" +typeset RANGE4="0,131072 +262144,131072 +524288,131072 +786432,131072" +typeset RANGE5="0,1048576 +7340032,1048576" +typeset RANGE6="393216,131072 +655360,131072 +917504,131072 +1179648,131072 +1441792,393216 +1966080,393216 +2621440,262144 +3145728,262144 +3670016,262144 +4194304,262144 +4718592,262144 +5242880,262144" +typeset RANGE7="1048576,6291456" +typeset RANGE8="4063232,131072" +typeset RANGE9="0,131072 +262144,131072 +524288,131072 +786432,131072 +1048576,131072 +1310720,131072 +1572864,131072 +1835008,131072 +2097152,131072 +2359296,131072 +2621440,131072 +2883584,131072 +3145728,131072 +3407872,131072 +3670016,131072 +3932160,131072" +typeset RANGE10="0,393216" +typeset RANGE11="0,1048576" +typeset RANGE12="0,2097152" +typeset RANGE13="0,16384" +typeset RANGE14="" +typeset RANGE15="0,4194304" +typeset RANGE16="0,6291456"
\ No newline at end of file diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib b/tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib new file mode 100644 index 000000000..d3d219bb9 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted.kshlib @@ -0,0 +1,270 @@ +#!/bin/ksh + +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2016, 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/tests/functional/redacted_send/redacted.cfg + +function setup_dataset +{ + typeset ds_name=$1 + typeset opts=$2 + typeset file_create_func=$3 + typeset sendfs="$POOL/$ds_name" + [[ -n $file_create_func ]] || file_create_func=setup_common + + log_must zfs create $opts $sendfs + + $file_create_func $sendfs + + log_must zfs snapshot $sendfs@snap + log_must zfs clone $opts $sendfs@snap $POOL/${ds_name}_clone + log_must zfs snapshot $POOL/${ds_name}_clone@snap +} + +function setup_common +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + typeset bs=$(get_prop recsize $sendfs) + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=$bs count=16 + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=$bs count=32 +} + +function setup_embedded +{ + typeset sendfs=$1 + + typeset recsize + typeset mntpnt=$(get_prop mountpoint $sendfs) + for recsize in 512 1024 2048 4096 8192 16384; do + if is_linux; then + log_must dd if=/dev/urandom of=$mntpnt/$recsize bs=8 \ + count=1 seek=$(((recsize / 8) - 1)) + else + log_must mkholes -d $((recsize - 8)):8 $mntpnt/$recsize + fi + done +} + +function setup_holes +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + typeset M=$((1024 * 1024)) + + if is_linux; then + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=8M count=1 + + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1M count=1 + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1M count=1 seek=7 \ + conv=notrunc + + log_must dd if=/dev/urandom of=$mntpnt/f3 bs=1M count=6 seek=1 + log_must truncate $mntpnt/f3 --size=$((8 * M)) + + log_must truncate $mntpnt/f4 --size=$((8 * M)) + else + log_must mkholes -d 0:$((8 * M)) $mntpnt/f1 + log_must mkholes -d 0:$M -d $((7 * M)):$M $mntpnt/f2 + log_must mkholes -d $M:$((6 * M)) -h $((7 * M)):$M $mntpnt/f3 + log_must mkholes -h 0:$((8 * M)) $mntpnt/f4 + fi + + log_must zfs create $sendfs/manyrm + for i in {1..256}; do + log_must stride_dd -i /dev/urandom -o $mntpnt/manyrm/f$i -b 512 \ + -c $(random 100) -s $(random 4) + done + + log_must zfs snapshot $sendfs/manyrm@snap + log_must zfs clone $sendfs/manyrm@snap $sendfs/manyrm_clone + log_must zfs snapshot $sendfs/manyrm_clone@snap +} + +function setup_incrementals +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + typeset bs=$(get_prop recsize $sendfs) + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=$bs count=16 + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=$bs count=32 + log_must mkdir $mntpnt/d1 + log_must eval "cat $mntpnt/f1 $mntpnt/f2 >$mntpnt/d1/f1" + log_must zfs snapshot $sendfs@snap0 + + log_must zfs clone $sendfs@snap0 $POOL/hole + mntpnt=$(get_prop mountpoint $POOL/hole) + log_must dd if=/dev/zero of=$mntpnt/f2 bs=$bs count=16 conv=notrunc + log_must zfs snapshot $POOL/hole@snap + + log_must zfs clone $sendfs@snap0 $POOL/stride3 + mntpnt=$(get_prop mountpoint $POOL/stride3) + log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $bs -c 11 -s 3 + log_must zfs snapshot $POOL/stride3@snap + + log_must zfs clone $sendfs@snap0 $POOL/stride5 + mntpnt=$(get_prop mountpoint $POOL/stride5) + log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $bs -c 7 -s 5 + log_must zfs snapshot $POOL/stride5@snap + + log_must zfs clone $sendfs@snap0 $POOL/int + log_must zfs snapshot $POOL/int@snap + + log_must zfs clone $POOL/int@snap $POOL/rm + mntpnt=$(get_prop mountpoint $POOL/rm) + log_must rm -rf $mntpnt/[df][12] + log_must zfs snapshot $POOL/rm@snap + + log_must zfs clone $POOL/int@snap $POOL/write + mntpnt=$(get_prop mountpoint $POOL/write) + log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=16 conv=notrunc + log_must dd if=/dev/urandom of=$mntpnt/d1/f1 bs=512 count=16 seek=16 \ + conv=notrunc + log_must zfs snapshot $POOL/write@snap +} + +function setup_mounts +{ + typeset sendfs=$1 + + typeset mntpnt=$(get_prop mountpoint $sendfs) + log_must touch $mntpnt/empty + log_must dd if=/dev/urandom of=$mntpnt/contents1 bs=512 count=2 + log_must dd if=/dev/urandom of=$mntpnt/contents2 bs=512 count=2 + log_must mkdir $mntpnt/dir1 + log_must touch $mntpnt/dir1/empty + log_must dd if=/dev/urandom of=$mntpnt/dir1/contents1 bs=512 count=2 + log_must dd if=/dev/urandom of=$mntpnt/dir1/contents2 bs=512 count=2 + log_must mkdir $mntpnt/dir1/dir2 + log_must touch $mntpnt/dir1/dir2/empty + log_must dd if=/dev/urandom of=$mntpnt/dir1/dir2/file bs=512 count=2 + + log_must zfs create -s -V 16p $sendfs/vol + log_must zfs snapshot $sendfs/vol@snap + log_must zfs clone $sendfs/vol@snap $sendfs/vol_clone + log_must zfs snapshot $sendfs/vol_clone@snap +} + +function mount_redacted +{ + typeset flag='' + while getopts "f" opt; do + case $opt in + f) + flag='-f' + ;; + esac + done + shift $(($OPTIND - 1)) + + typeset ds=$1 + log_must set_tunable32 zfs_allow_redacted_dataset_mount 1 + zfs mount $flag -oro $ds || return 1 + log_must set_tunable32 zfs_allow_redacted_dataset_mount 0 + return 0 +} + +function unmount_redacted +{ + typeset ds=$1 + + zfs unmount $ds +} + +# +# This function calls a utility that prints out the ranges where a file +# and its redacted counterpart differ, each range on a new line like this: +# +# 0,131072 +# 1966080,131072 +# 3932160,131072 +# +# The output is then checked against a variable containing the expected +# output to verify the redacted ranges are the ones expected. +# +function compare_files +{ + typeset sendfs=$1 + typeset recvfs=$2 + typeset file=$3 + typeset expected="$4" + typeset tmpfile="$tmpdir/get_file.out" + + log_must mount_redacted -f $recvfs + + typeset file1="$(get_prop mountpoint $sendfs)/$file" + typeset file2="$(get_prop mountpoint $recvfs)/$file" + log_note "Comparing $file1 and $file2" + [[ -f $file1 ]] || log_fail "File $file1 does not exist." + [[ -f $file2 ]] || log_fail "File $file2 does not exist." + + log_must eval "get_diff $file1 $file2 >$tmpfile" + typeset range="$(cat $tmpfile)" + log_must unmount_redacted $recvfs + [[ "$expected" = "$range" ]] || log_fail "Unexpected range: $range" +} + +function redacted_cleanup +{ + typeset ds_list=$@ + typeset ds + + # Verify the receiving pool can still be exported and imported. + log_must zpool export $POOL2 + log_must zpool import $POOL2 + + for ds in $ds_list; do + datasetexists $ds && log_must zfs destroy -R $ds + done + + log_must set_tunable32 zfs_allow_redacted_dataset_mount 0 + rm -f $(get_prop mountpoint $POOL)/tmp/* +} + +# Retrieve the redaction list of a bookmark or snapshot, using +# the property or zdb output, as requested. +function get_guid_list +{ + typeset filename=$1 + typeset dataset=$2 + typeset use_zdb=${3:-false} + + if $use_zdb; then + guid_list=$(zdb -vvvv $dataset | sed -e 's/,//g' \ + -ne 's/^.*Snapshots: \[\(.*\)\]/\1/p') + else + guid_list=$(get_prop redact_snaps $dataset) + fi + + for guid in $(echo $guid_list | tr ',' ' '); do + echo $guid + done | sort >$filename +} diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh new file mode 100755 index 000000000..0a8bf3903 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_compressed.ksh @@ -0,0 +1,71 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that compressed send streams are redacted correctly. +# +# Strategy: +# 1. Receive a redacted compressed send stream, verifying compression and +# redaction. +# 2. Receive an incremental on the full receive, verifying compression and +# redaction. +# + +typeset ds_name="compressed" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name "-o compress=lz4" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset clone_mnt="$(get_prop mountpoint $clone)" + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must stride_dd -i /dev/urandom -o $clone_mnt/f1 -b $((128 * 1024)) -c 4 -s 2 +log_must zfs snapshot $clone@snap1 +log_must rm $clone_mnt/f2 +log_must zfs snapshot $clone@snap2 + +log_must zfs redact $sendfs@snap book1 $clone@snap1 $clone@snap2 +log_must eval "zfs send -c --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must stream_has_features $stream compressed lz4 redacted +compare_files $sendfs $recvfs "f1" "$RANGE4" +verify_stream_size $stream $sendfs +log_must mount_redacted -f $recvfs +verify_stream_size $stream $recvfs +log_must unmount_redacted $recvfs + +log_must eval "zfs send -c -i $sendfs@snap $clone@snap1 >$stream" +log_must eval "zfs recv $POOL2/inc1 <$stream" +log_must stream_has_features $stream compressed lz4 +typeset mntpnt=$(get_prop mountpoint $POOL2) +log_must diff $clone_mnt/f1 $mntpnt/inc1/f1 +log_must diff $send_mnt/f2 $mntpnt/inc1/f2 + +log_must eval "zfs send -c -i $sendfs@snap $clone@snap2 >$stream" +log_must eval "zfs recv $POOL2/inc2 <$stream" +log_must stream_has_features $stream compressed lz4 +log_must diff $clone_mnt/f1 $mntpnt/inc1/f1 +[[ -f $mntpnt/inc2/f2 ]] && log_fail "File f2 should not exist." + +log_pass "Compressed send streams are redacted correctly." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh new file mode 100755 index 000000000..58dbde7f1 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_contents.ksh @@ -0,0 +1,162 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redaction works as expected for various scenarios. +# +# Strategy: +# 1. An unmodified file does not get redacted at all. +# 2. Empty redaction list redacts everything. +# 3. A file removed in the clone redacts the whole file. +# 4. A file moved in the clone does not redact the file. +# 5. A copied, then removed file in the clone redacts the whole file. +# 6. Overwriting a file with identical contents redacts the file. +# 7. A paritally modified block redacts the entire block. +# 8. Only overlapping areas of modified ranges are redacted. +# 9. Send from the root dataset of a pool work correctly. +# + +typeset ds_name="contents" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +# An unmodified file does not get redacted at all. +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must diff $send_mnt/f2 $recv_mnt/f2 +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Removing a file in the clone redacts the entire file. +log_must rm "$clone_mnt/f1" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book3 $clone@snap1 +log_must eval "zfs send --redact book3 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE0" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Moving a file in the clone does not redact the file. +log_must mv "$clone_mnt/f1" "$clone_mnt/f1.moved" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book4 $clone@snap1 +log_must eval "zfs send --redact book4 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +[[ -f $recv_mnt/f1.moved ]] && log_fail "Found moved file in redacted receive." +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Copying, then removing a file in the clone does redact the file. +log_must cp "$clone_mnt/f1" "$clone_mnt/f1.copied" +log_must rm "$clone_mnt/f1" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book5 $clone@snap1 +log_must eval "zfs send --redact book5 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE0" +log_must mount_redacted -f $recvfs +[[ -f $recv_mnt/f1.copied ]] && log_fail "Found moved file in redacted receive." +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Overwriting the contents of a block with identical contents redacts the file. +log_must cp "$clone_mnt/f1" "$clone_mnt/f1.copied" +log_must cp "$clone_mnt/f1.copied" "$clone_mnt/f1" +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book6 $clone@snap1 +log_must eval "zfs send --redact book6 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE0" +log_must mount_redacted -f $recvfs +[[ -f $recv_mnt/f1.copied ]] && log_fail "Found moved file in redacted receive." +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Modifying some of a block redacts the whole block. +log_must dd if=/dev/urandom of=$clone_mnt/f1 conv=notrunc seek=2 count=1 bs=32k +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book7 $clone@snap1 +log_must eval "zfs send --redact book7 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE1" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Only overlapping areas of modified ranges are redacted. +log_must dd if=/dev/urandom of=$clone_mnt/f2 bs=1024k count=3 conv=notrunc +log_must zfs snapshot $clone@snap1 +log_must zfs clone $sendfs@snap $clone/new +typeset mntpnt="$(get_prop mountpoint $clone/new)" +log_must dd if=/dev/urandom of=$mntpnt/f2 bs=1024k seek=1 count=3 \ + conv=notrunc +log_must zfs snapshot $clone/new@snap +log_must zfs redact $sendfs@snap book8 $clone@snap1 $clone/new@snap +log_must eval "zfs send --redact book8 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE2" +log_must zfs destroy -R $clone/new +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# FizzBuzz version +log_must zfs clone $sendfs@snap $POOL/stride3 +mntpnt="$(get_prop mountpoint $POOL/stride3)" +log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $((128 * 1024)) -c 11 -s 3 +log_must zfs snapshot $POOL/stride3@snap +log_must zfs clone $sendfs@snap $POOL/stride5 +mntpnt="$(get_prop mountpoint $POOL/stride5)" +log_must stride_dd -i /dev/urandom -o $mntpnt/f2 -b $((128 * 1024)) -c 7 -s 5 +log_must zfs snapshot $POOL/stride5@snap +log_must zfs redact $sendfs@snap book8a $POOL/stride3@snap $POOL/stride5@snap +log_must eval "zfs send --redact book8a $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE3" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Send from the root dataset of a pool work correctly. +log_must dd if=/dev/urandom of=/$POOL/f1 bs=128k count=4 +log_must zfs snapshot $POOL@snap +log_must zfs clone $POOL@snap $POOL/clone +log_must dd if=/dev/urandom of=/$POOL/clone/f1 bs=128k count=1 conv=notrunc +log_must zfs snapshot $POOL/clone@snap +log_must zfs redact $POOL@snap book9 $POOL/clone@snap +log_must eval "zfs send --redact book9 $POOL@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $POOL $recvfs "f1" "$RANGE1" +log_must zfs destroy -R $POOL@snap + +log_pass "Redaction works as expected for various scenarios." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh new file mode 100755 index 000000000..e25b51cb9 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh @@ -0,0 +1,103 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2017, 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redaction works as expected with respect to deleted files +# +# Strategy: +# 1. A file on the delete queue counts as deleted when using it to calculate +# redaction. +# 2. A file that is removed in the tosnap of an incremental, where the fromsnap +# is a redaction bookmark that contains references to that file, does not +# result in records for that file. +# + +typeset ds_name="deleted" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset clone2="$POOL/${ds_name}_clone2" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +# +# A file on the delete queue counts as deleted when using it to calculate +# redaction. +# + +# +# Open file descriptor 5 for appending to $clone_mnt/f1 so that it will go on +# the delete queue when we rm it. +# +exec 5>>$clone_mnt/f1 +log_must dd if=/dev/urandom of=$clone_mnt/f1 bs=512 count=1 conv=notrunc +log_must rm $clone_mnt/f1 +log_must zfs snapshot $clone@snap1 +# Close file descriptor 5 +exec 5>&- +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +# +# We have temporarily disabled redaction blkptrs, so this will not +# fail as was originally intended. We should uncomment this line +# when we reenable redaction blkptrs. +# +#log_mustnot dd if=$recv_mnt/f1 of=/dev/null bs=512 count=1 +log_must diff $send_mnt/f2 $recv_mnt/f2 +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# +# A file that is removed in the tosnap of an incremental, where the fromsnap +# is a redaction bookmark that contains references to that file, does not +# result in records for that file. +# +log_must zfs clone $sendfs@snap $clone2 +typeset clone2_mnt="$(get_prop mountpoint $clone2)" +log_must rm -rf $clone2_mnt/* +log_must zfs snapshot $clone2@snap +log_must zfs redact $sendfs@snap book2 $clone2@snap +log_must zfs destroy -R $clone2 +log_must eval "zfs send --redact book2 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must rm $send_mnt/f1 +log_must zfs snapshot $sendfs@snap2 +log_must zfs clone $sendfs@snap2 $clone2 +typeset clone2_mnt="$(get_prop mountpoint $clone2)" +log_must rm $clone2_mnt/* +log_must zfs snapshot $clone2@snap +log_must zfs redact $sendfs@snap2 book3 $clone2@snap +log_must zfs destroy -R $clone2 +log_must eval "zfs send -i $sendfs#book2 --redact book3 $sendfs@snap2 >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +log_must diff <(ls $send_mnt) <(ls $recv_mnt) +log_must zfs destroy -R $recvfs +log_must zfs rollback -R $sendfs@snap + +log_pass "Verify Redaction works as expected with respect to deleted files." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh new file mode 100755 index 000000000..24478f1bc --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_disabled_feature.ksh @@ -0,0 +1,71 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify the functionality of the redaction_bookmarks and redacted_datasets +# features. +# +# Strategy: +# 1. Create a pool with all features disabled. +# 2. Verify redacted send fails. +# 3. Enable redaction_bookmarks and verify redacted sends works. +# 4. Verify recepit of a redacted stream fails. +# 5. Enable recacted_datasets and verify zfs receive works. +# + +typeset ds_name="disabled" +typeset sendfs="$POOL/$ds_name" +typeset sendfs1="$POOL2/${ds_name}1" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset clone1="$POOL2/${ds_name}_clone1" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' + +function cleanup +{ + destroy_pool $POOL2 + create_pool $POOL2 $DISK2 + log_must zfs snapshot $POOL2@init + redacted_cleanup $sendfs $recvfs +} + +log_onexit cleanup + +destroy_pool $POOL2 +log_must zpool create -d $POOL2 $DISK2 + +log_must zfs create $sendfs1 +log_must zfs snapshot $sendfs1@snap +log_must zfs clone $sendfs1@snap $clone1 +log_must zfs snapshot $clone1@snap + +log_mustnot zfs redact $sendfs1@snap book1 $clone1@snap +log_must zpool set feature@redaction_bookmarks=enabled $POOL2 +log_must zfs redact $sendfs1@snap book1 $clone1@snap + +log_must zfs redact $sendfs@snap book1 $clone@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_mustnot eval "zfs recv $recvfs <$stream" +log_must zpool set feature@redacted_datasets=enabled $POOL2 +log_must eval "zfs recv $recvfs <$stream" + +log_pass "The redacted send/recv features work correctly." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh new file mode 100755 index 000000000..94937a2f7 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_embedded.ksh @@ -0,0 +1,103 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify embedded blocks and redacted send work correctly together. +# +# Strategy: +# 1. Create recsize sized files with embedded blocks from size 512b to 16k. +# 2. Receive a redacted send stream with nothing redacted. +# 3. Verify the received files match the source, contain embedded blocks, and +# that the stream has the redacted and embedded data features. +# 4. Receive a redacted send stream with files 512, 2048 and 8192 redacted. +# 5. Verify that the redacted files no longer match, but the others still +# contain embedded blocks and the stream has the redacted and embedded +# data features. +# + +typeset ds_name="embedded" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '-o compress=lz4' setup_embedded +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset recsize send_obj recv_obj + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must zfs redact $sendfs@snap book1 $clone@snap +log_must eval "zfs send -e --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must stream_has_features $stream redacted embed_data + +log_must mount_redacted -f $recvfs +for recsize in 512 1024 2048 4096 8192 16384; do + send_obj=$(get_objnum $send_mnt/$recsize) + recv_obj=$(get_objnum $recv_mnt/$recsize) + + log_must diff $send_mnt/$recsize $recv_mnt/$recsize + log_must eval "zdb -ddddd $sendfs $send_obj >$tmpdir/send.zdb" + log_must eval "zdb -ddddd $recvfs $recv_obj >$tmpdir/recv.zdb" + + grep -q "EMBEDDED" $tmpdir/send.zdb || \ + log_fail "Obj $send_obj not embedded in $sendfs" + grep -q "EMBEDDED" $tmpdir/recv.zdb || \ + log_fail "Obj $recv_obj not embedded in $recvfs" + + cat $stream | zstreamdump -v | log_must grep -q \ + "WRITE_EMBEDDED object = $send_obj offset = 0" +done + +log_must zfs destroy -R $recvfs +for recsize in 512 2048 8192; do + log_must dd if=/dev/urandom of=$clone_mnt/$recsize bs=$recsize count=1 +done +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book2 $clone@snap1 +log_must eval "zfs send -e --redact book2 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must stream_has_features $stream redacted embed_data + +log_must mount_redacted -f $recvfs +for recsize in 512 2048 8192; do + log_mustnot diff $send_mnt/$recsize $recv_mnt/$recsize +done +for recsize in 1024 4096 16384; do + send_obj=$(get_objnum $send_mnt/$recsize) + recv_obj=$(get_objnum $recv_mnt/$recsize) + + log_must diff $send_mnt/$recsize $recv_mnt/$recsize + log_must eval "zdb -ddddd $sendfs $send_obj >$tmpdir/send.zdb" + log_must eval "zdb -ddddd $recvfs $recv_obj >$tmpdir/recv.zdb" + + grep -q "EMBEDDED" $tmpdir/send.zdb || \ + log_fail "Obj $send_obj not embedded in $sendfs" + grep -q "EMBEDDED" $tmpdir/recv.zdb || \ + log_fail "Obj $recv_obj not embedded in $recvfs" + + cat $stream | zstreamdump -v | log_must grep -q \ + "WRITE_EMBEDDED object = $send_obj offset = 0" +done + +log_pass "Embedded blocks and redacted send work correctly together." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh new file mode 100755 index 000000000..47063848f --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_holes.ksh @@ -0,0 +1,120 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redacted send streams reliably handle holes. +# +# Strategy: +# 1. Holes written at the beginning and end of a non-sparse file in the +# redacted list are correctly redacted. +# 2. Holes written throughout a non-sparse file in the redacted list are +# correctly redacted. +# 3. Data written into a hole in a sparse file in the redacted list are +# correctly redacted. +# 4. Holes in metadata blocks. +# + +typeset ds_name="holes" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_holes +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset M=$((1024 * 1024)) + +log_onexit redacted_cleanup $sendfs $recvfs + +# Write holes at the start and end of a non-sparse file. +if is_linux; then + log_must dd if=/dev/zero of=$clone_mnt/f1 bs=1M count=1 conv=notrunc + log_must dd if=/dev/zero of=$clone_mnt/f1 bs=1M count=1 conv=notrunc seek=7 +else + log_must mkholes -h 0:$M -h $((7 * M)):$M $clone_mnt/f1 +fi +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE5" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Write two overlapping sets of holes into the same non-sparse file. +log_must stride_dd -i /dev/zero -o $clone_mnt/f1 -b $((128 * 1024)) -c 8 -s 2 -k 3 +log_must stride_dd -i /dev/zero -o $clone_mnt/f1 -b $((256 * 1024)) -c 8 -s 2 -k 6 +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book2 $clone@snap1 +log_must eval "zfs send --redact book2 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE6" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Write data into the middle of a hole. +if is_linux; then + log_must dd if=/dev/urandom of=$clone_mnt/f2 bs=1M count=2 seek=3 \ + conv=notrunc +else + log_must mkholes -d $((3 * M)):$((2 * M)) $clone_mnt/f2 +fi +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book3 $clone@snap1 +log_must eval "zfs send --redact book3 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE14" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Remove a file with holes. +log_must rm $clone_mnt/f3 +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendfs@snap book4 $clone@snap1 +log_must eval "zfs send --redact book4 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f3" "$RANGE7" +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +# Create a hole in a L0 metadata block by removing files. +log_must rm $send_mnt/manyrm_clone/f{32..96} +log_must zfs snapshot $sendfs/manyrm_clone@snap1 + +log_must zfs redact $sendfs/manyrm@snap book6 $sendfs/manyrm_clone@snap1 +log_must eval "zfs send --redact book6 $sendfs/manyrm@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +log_must mount_redacted -f $recvfs +for i in {1..31} {97..256}; do + diff $send_mnt/manyrm/f$i $recv_mnt/f$i || log_fail \ + "File f$i did not match in the send and recv datasets." +done +for i in {32..96}; do + file_size=$(stat -c %s $send_mnt/manyrm/f$i) + redacted_size=$(stat -c %s $recv_mnt/f$i) + [[ $file_size -eq $redacted_size ]] || log_fail \ + "File f$i has size $file_size and redacted size $redacted_size" +done +log_must zfs rollback -R $clone@snap +log_must zfs destroy -R $recvfs + +log_pass "Redacted send streams reliably handle holes." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh new file mode 100755 index 000000000..1d2ed3a68 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_incrementals.ksh @@ -0,0 +1,152 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that incrementals (redacted and normal) work with redacted datasets. +# +# Strategy: +# 1. Test normal incrementals from the original snap to a subset of the +# redaction list. +# 2. Test receipt of intermediate clones, and their children. +# 3. Test receipt with origin snap specified by '-o origin='. +# 4. Test incrementals from redaction bookmarks. +# + +typeset ds_name="incrementals" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_incrementals +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs $POOL2/rfs + +# Setup a redacted send using a redaction list at varying depth. +log_must zfs redact $sendfs@snap0 book1 $POOL/rm@snap $POOL/stride3@snap \ + $POOL/stride5@snap +log_must eval "zfs send --redact book1 $sendfs@snap0 >$stream" +log_must eval "zfs receive $POOL2/rfs <$stream" + +# Verify receipt of normal incrementals to redaction list members. +log_must eval "zfs send -i $sendfs@snap0 $POOL/stride3@snap >$stream" +log_must eval "zfs recv $POOL2/rstride3 <$stream" +log_must diff -r /$POOL/stride3 /$POOL2/rstride3 +log_must eval "zfs send -i $sendfs@snap0 $POOL/stride5@snap >$stream" +log_must eval "zfs recv $POOL2/rstride5 <$stream" +log_must diff -r /$POOL/stride5 /$POOL2/rstride5 + +# But not a normal child that we weren't redacted with respect to. +log_must eval "zfs send -i $sendfs@snap0 $POOL/hole@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rhole@snap <$stream" + +# Verify we can receive an intermediate clone redacted with respect to a +# subset of the original redaction list. +log_must zfs redact $POOL/int@snap book2 $POOL/rm@snap +log_must eval "zfs send -i $sendfs@snap0 --redact book2 $POOL/int@snap >$stream" +log_must eval "zfs recv $POOL2/rint <$stream" +compare_files $POOL/int $POOL2/rint "f1" "$RANGE0" +compare_files $POOL/int $POOL2/rint "f2" "$RANGE15" +compare_files $POOL/int $POOL2/rint "d1/f1" "$RANGE16" +log_must mount_redacted -f $POOL2/rint + +# Verify we can receive grandchildren on the child. +log_must eval "zfs send -i $POOL/int@snap $POOL/rm@snap >$stream" +log_must eval "zfs receive $POOL2/rrm <$stream" +log_must diff -r /$POOL/rm /$POOL2/rrm + +# But not a grandchild that the received child wasn't redacted with respect to. +log_must eval "zfs send -i $POOL/int@snap $POOL/write@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rwrite<$stream" + +# Verify we cannot receive an intermediate clone that isn't redacted with +# respect to a subset of the original redaction list. +log_must zfs redact $POOL/int@snap book4 $POOL/rm@snap $POOL/write@snap +log_must eval "zfs send -i $sendfs@snap0 --redact book4 $POOL/int@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rint <$stream" +log_must zfs redact $POOL/int@snap book5 $POOL/write@snap +log_must eval "zfs send -i $sendfs@snap0 --redact book5 $POOL/int@snap >$stream" +log_mustnot eval "zfs recv $POOL2/rint <$stream" +log_mustnot zfs redact $POOL/int@snap book6 $POOL/hole@snap + +# Verify we can receive a full clone of the grandchild on the child. +log_must eval "zfs send $POOL/write@snap >$stream" +log_must eval "zfs recv -o origin=$POOL2/rint@snap $POOL2/rwrite <$stream" +log_must diff -r /$POOL/write /$POOL2/rwrite + +# Along with other origins. +log_must eval "zfs recv -o origin=$POOL2/rfs@snap0 $POOL2/rwrite1 <$stream" +log_must diff -r /$POOL/write /$POOL2/rwrite1 +log_must eval "zfs recv -o origin=$POOL2@init $POOL2/rwrite2 <$stream" +log_must diff -r /$POOL/write /$POOL2/rwrite2 +log_must zfs destroy -R $POOL2/rwrite2 + +log_must zfs destroy -R $POOL2/rfs + +# Write some data for tests of incremental sends from bookmarks +log_must zfs snapshot $sendfs@snap1 +log_must zfs clone $sendfs@snap1 $POOL/hole1 +typeset mntpnt=$(get_prop mountpoint $POOL/hole1) +log_must dd if=/dev/zero of=$mntpnt/f2 bs=128k count=16 conv=notrunc +log_must zfs snapshot $POOL/hole1@snap +log_must zfs clone $sendfs@snap1 $POOL/write1 +mntpnt=$(get_prop mountpoint $POOL/write1) +log_must dd if=/dev/urandom of=$mntpnt/f2 bs=128k count=16 conv=notrunc +log_must zfs snapshot $POOL/write1@snap +log_must zfs clone $POOL/int@snap $POOL/write2 +mntpnt=$(get_prop mountpoint $POOL/write2) +log_must dd if=/dev/urandom of=$mntpnt/f2 bs=128k count=16 conv=notrunc +log_must zfs snapshot $POOL/write2@snap + +# Setup a redacted send using a redaction list at varying depth. +log_must zfs redact $sendfs@snap0 book7 $POOL/rm@snap $POOL/stride3@snap \ + $POOL/stride5@snap +log_must eval "zfs send --redact book7 $sendfs@snap0 >$stream" +log_must eval "zfs receive $POOL2/rfs <$stream" + +# Verify we can receive a redacted incremental sending from the bookmark. +log_must zfs redact $sendfs@snap1 book8 $POOL/write1@snap +log_must eval "zfs send -i $sendfs#book7 --redact book8 $sendfs@snap1 >$stream" +log_must eval "zfs receive $POOL2/rfs <$stream" +# The stride3 and stride5 snaps redact 3 128k blocks at block offsets 0 15 and +# 30 of f2. The write1 snap only covers the first two of those three blocks. +compare_files $sendfs $POOL2/rfs "f2" "$RANGE12" +log_must mount_redacted -f $POOL2/rfs +log_must diff $send_mnt/f1 /$POOL2/rfs/f1 +log_must diff $send_mnt/d1/f1 /$POOL2/rfs/d1/f1 +unmount_redacted $POOL2/rfs + +# Verify we can receive a normal child we weren't redacted with respect to by +# sending from the bookmark. +log_must eval "zfs send -i $sendfs#book7 $POOL/hole1@snap >$stream" +log_must eval "zfs recv $POOL2/rhole1 <$stream" +log_must diff -r /$POOL/hole1 /$POOL2/rhole1 + +# Verify we can receive an intermediate clone redacted with respect to a +# non-subset if we send from the bookmark. +log_must zfs redact $POOL/int@snap book9 $POOL/write2@snap +log_must eval "zfs send -i $sendfs#book7 --redact book9 $POOL/int@snap >$stream" +log_must eval "zfs receive $POOL2/rint <$stream" +compare_files $sendfs $POOL2/rint "f2" "$RANGE12" + +log_pass "Incrementals (redacted and normal) work with redacted datasets." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh new file mode 100755 index 000000000..caccdd360 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_largeblocks.ksh @@ -0,0 +1,63 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify large blocks and redacted send work correctly together. +# +# Strategy: +# 1. Create a dataset and clone with a 1m recordsize, modifying a few k +# within the first 1m of a 16m file. +# 2. Verify that the whole first 1m of the file is redacted. +# 3. Receive an incremental stream from the original snap to the snap it +# was redacted with respect to. +# 4. Verify that the received dataset matches the clone +# + +typeset ds_name="largeblocks" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '-o recsize=1m' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must dd if=/dev/urandom of=$clone_mnt/f1 bs=32k count=3 seek=8 conv=notrunc +log_must zfs snapshot $clone@snap1 + +log_must zfs redact $sendfs@snap book1 $clone@snap1 +log_must eval "zfs send -L --redact book1 $sendfs@snap >$stream" +log_must stream_has_features $stream redacted large_blocks +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f1" "$RANGE11" +log_must mount_redacted -f $recvfs +log_must diff $send_mnt/f2 $recv_mnt/f2 +unmount_redacted $recvfs + +log_must eval "zfs send -L -i $sendfs@snap $clone@snap1 >$stream" +log_must stream_has_features $stream large_blocks +log_must eval "zfs recv $recvfs/new <$stream" +log_must diff -r $clone_mnt $recv_mnt/new + +log_pass "Large blocks and redacted send work correctly together." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh new file mode 100755 index 000000000..3386643b2 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_many_clones.ksh @@ -0,0 +1,68 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redacted send can deal with a large redaction list. +# +# Strategy: +# 1. Create 64 clones of sendfs each of which modifies two blocks in a file. +# The first modification is at an offset unique to each clone, and the +# second (the last block in the file) is common to them all. +# 2. Verify a redacted stream with a reasonable redaction list length can +# be correctly processed. +# 3. Verify that if the list is too long, the send fails gracefully. +# + +typeset ds_name="many_clones" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset redaction_list='' +typeset mntpnt + +log_onexit redacted_cleanup $sendfs $recvfs + +# Fill in both the last block, and a different block in every clone. +for i in {1..64}; do + log_must zfs clone $sendfs@snap ${clone}$i + mntpnt=$(get_prop mountpoint ${clone}$i) + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=$i \ + conv=notrunc + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=63 \ + conv=notrunc + log_must zfs snapshot ${clone}$i@snap +done + +# The limit isn't necessarily 32 snapshots. The maximum number of snapshots in +# the redacted list is determined in dsl_bookmark_create_redacted_check(). +log_must zfs redact $sendfs@snap book1 $clone{1..32}@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" +compare_files $sendfs $recvfs "f2" "$RANGE8" + +log_mustnot zfs redact $sendfs@snap book2 $clone{1..64}@snap + +log_pass "Redacted send can deal with a large redaction list." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh new file mode 100755 index 000000000..e1cd09e17 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_mixed_recsize.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify redacted send works with datasets of different sizes. +# +# Strategy: +# 1. Create two dataset one with recsize 512, and one 1m and create a 2m file. +# 2. For each dataset, create clones of both 512 and 1m recsize and modify +# the first 16k of the file. +# 3. Send each original dataset, redacted with respect to each of the clones +# into both a dataset inheriting a 512 recsize and a 1m one. +# 4. Verify that the smallest unit of redaction is that of the origin fs. +# + +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +typeset mntpnt + +log_onexit redacted_cleanup $POOL/512 $POOL/1m $POOL2/512 $POOL2/1m + +# Set up the datasets we'll send and redact from. +log_must zfs create -o recsize=512 $POOL/512 +mntpnt=$(get_prop mountpoint $POOL/512) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=1024k count=2 +log_must zfs snapshot $POOL/512@snap +log_must zfs clone -o recsize=1m $POOL/512@snap $POOL/1mclone +mntpnt=$(get_prop mountpoint $POOL/1mclone) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=32 conv=notrunc +log_must zfs snapshot $POOL/1mclone@snap + +log_must zfs create -o recsize=1m $POOL/1m +mntpnt=$(get_prop mountpoint $POOL/1m) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=1024k count=2 +log_must zfs snapshot $POOL/1m@snap +log_must zfs clone -o recsize=512 $POOL/1m@snap $POOL/512clone +mntpnt=$(get_prop mountpoint $POOL/512clone) +log_must dd if=/dev/urandom of=$mntpnt/f1 bs=512 count=32 conv=notrunc +log_must zfs snapshot $POOL/512clone@snap + +# Create datasets that allow received datasets to inherit recordsize. +log_must zfs create -o recsize=512 $POOL2/512 +log_must zfs create -o recsize=1m $POOL2/1m + +# Do the sends and verify the contents. +log_must zfs redact $POOL/512@snap book1 $POOL/1mclone@snap +log_must eval "zfs send --redact book1 $POOL/512@snap>$stream" +log_must eval "zfs recv $POOL2/512/recva <$stream" +compare_files $POOL/512 $POOL2/512/recva "f1" "$RANGE13" +log_must eval "zfs recv $POOL2/1m/recvb <$stream" +compare_files $POOL/512 $POOL2/1m/recvb "f1" "$RANGE13" + +log_must zfs redact $POOL/1m@snap book2 $POOL/512clone@snap +log_must eval "zfs send --redact book2 $POOL/1m@snap >$stream" +log_must eval "zfs recv $POOL2/512/recvc <$stream" +compare_files $POOL/1m $POOL2/512/recvc "f1" "$RANGE11" +log_must eval "zfs recv $POOL2/1m/recvd <$stream" +compare_files $POOL/1m $POOL2/1m/recvd "f1" "$RANGE11" + +log_pass "Redaction works correctly with different recordsizes." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh new file mode 100755 index 000000000..b24f5ccad --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_mounts.ksh @@ -0,0 +1,109 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that received redacted datasets are not mounted by default, but +# can still be mounted after setting zfs_allow_redacted_dataset_mount. +# +# Strategy: +# 1. Verify a received redacted stream isn't mounted by default. +# 2. Set zfs_allow_redacted_dataset_mount and verify it can't be mounted +# without the -f flag, but can with -f. +# 3. Receive a redacted volume. +# 4. Verify the device file isn't present until the kernel variable is set. +# 5. Verify the files in the send fs are also present in the recv fs. +# + +typeset ds_name="mounts" +typeset sendfs="$POOL/$ds_name" +typeset sendvol="$sendfs/vol" +typeset recvfs="$POOL2/$ds_name" +typeset recvvol="$POOL2/vol" +typeset clone="$POOL/${ds_name}_clone" +typeset clonevol="${sendvol}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_mounts +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" +typeset recv_vol_file="/dev/zvol/$recvvol" + +log_onexit redacted_cleanup $sendfs $recvfs $recvvol + +log_must rm $clone_mnt/empty $clone_mnt/contents1 +log_must dd if=/dev/urandom of=$clone_mnt/contents2 bs=512 count=1 conv=notrunc +log_must rm $clone_mnt/dir1/contents1 +log_must rm -rf $clone_mnt/dir1/dir2 +log_must dd if=/dev/urandom of=$clone_mnt/dir1/contents2 bs=512 count=1 \ + conv=notrunc +log_must dd if=/dev/urandom of=$clone_mnt/dir1/empty bs=512 count=1 +log_must zfs snapshot $clone@snap1 + +log_must zfs redact $sendfs@snap book1 $clone@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs receive $recvfs <$stream" +log_mustnot ismounted $recvfs +log_mustnot mount_redacted $recvfs +log_mustnot ismounted $recvfs +log_must mount_redacted -f $recvfs +log_must ismounted $recvfs + +# Verify that the send and recv fs both have the same files under their +# mountpoints by comparing find output with the name of the mountpoint +# deleted. +contents=$(log_must find $recv_mnt) +contents_orig=$(log_must find $send_mnt) +log_must diff <(echo ${contents//$recv_mnt/}) \ + <(echo ${contents_orig//$send_mnt/}) +log_must zfs redact $sendvol@snap book2 $clonevol@snap +log_must eval "zfs send --redact book2 $sendvol@snap >$stream" +log_must eval "zfs receive $recvvol <$stream" +[[ -b $recv_vol_file ]] && log_fail "Volume device file should not exist." +log_must set_tunable32 zfs_allow_redacted_dataset_mount 1 +log_must zpool export $POOL2 +log_must zpool import $POOL2 +udevadm settle + +# The device file isn't guaranteed to show up right away. +if [[ ! -b $recv_vol_file ]]; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + [[ -b $recv_vol_file ]] && break + done +fi +[[ -b $recv_vol_file ]] || log_fail "Volume device file should exist." + +log_must dd if=/dev/urandom of=$send_mnt/dir1/contents1 bs=512 count=2 +log_must rm $send_mnt/dir1/dir2/empty +log_must zfs snapshot $sendfs@snap2 +log_must eval "zfs send -i $sendfs#book1 $sendfs@snap2 >$stream" +log_must eval "zfs receive $recvfs <$stream" +log_must mount_redacted -f $recvfs +log_must ismounted $recvfs +contents=$(log_must find $recv_mnt) +contents_orig=$(log_must find $send_mnt) +log_must diff <(echo ${contents//$recv_mnt/}) \ + <(echo ${contents_orig//$send_mnt/}) + +log_pass "Received redacted streams can be mounted." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh new file mode 100755 index 000000000..e27eb601e --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_negative.ksh @@ -0,0 +1,80 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Test that redacted send correctly detects invalid arguments. +# + +typeset sendfs="$POOL2/sendfs" +typeset recvfs="$POOL2/recvfs" +typeset clone1="$POOL2/clone1" +typeset clone2="$POOL2/clone2" +typeset clone3="$POOL2/clone3" +typeset clone3="$POOL2/clone4" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) + +log_onexit redacted_cleanup $sendfs $recvfs $clone3 + +log_must zfs create $sendfs +log_must zfs snapshot $sendfs@snap1 +log_must zfs snapshot $sendfs@snap2 +log_must zfs snapshot $sendfs@snap3 +log_must zfs clone $sendfs@snap2 $clone1 +log_must zfs snapshot $clone1@snap +log_must zfs bookmark $clone1@snap $clone1#book +log_must zfs clone $sendfs@snap2 $clone2 +log_must zfs snapshot $clone2@snap + +# Incompatible flags +log_must zfs redact $sendfs@snap2 book $clone1@snap +log_mustnot eval "zfs send -R --redact book $sendfs@snap2 >/dev/null" + +typeset arg +for arg in "$sendfs" "$clone1#book"; do + log_mustnot eval "zfs send --redact book $arg >/dev/null" +done + +# Bad redaction list arguments +log_mustnot zfs redact $sendfs@snap1 +log_mustnot zfs redact $sendfs@snap1 book +log_mustnot zfs redact $sendfs#book1 book4 $clone1 +log_mustnot eval "zfs send --redact $sendfs#book $sendfs@snap >/dev/null" + +# Redaction snapshots not a descendant of tosnap +log_mustnot zfs redact $sendfs@snap2 book $sendfs@snap2 +log_must zfs redact $sendfs@snap2 book2 $clone1@snap $clone2@snap +log_must eval "zfs send --redact book2 $sendfs@snap2 >$stream" +log_must zfs redact $sendfs@snap2 book3 $clone1@snap $clone2@snap +log_must eval "zfs send -i $sendfs@snap1 --redact book3 $sendfs@snap2 \ + >/dev/null" +log_mustnot zfs redact $sendfs@snap3 $sendfs@snap3 $clone1@snap + +# Full redacted sends of redacted datasets are not allowed. +log_must eval "zfs recv $recvfs <$stream" +log_must zfs snapshot $recvfs@snap +log_must zfs clone $recvfs@snap $clone3 +log_must zfs snapshot $clone3@snap +log_mustnot zfs redact $recvfs@snap book5 $clone3@snap + +# Nor may a redacted dataset appear in the redaction list. +log_mustnot zfs redact testpool2/recvfs@snap2 book7 testpool2/recvfs@snap + +log_pass "Verify that redacted send correctly detects invalid arguments." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh new file mode 100755 index 000000000..74e5914f2 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_origin.ksh @@ -0,0 +1,87 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Test that receiving sends from redaction bookmarks and redacted datasets +# works correctly in certain edge cases. +# 1. Send A(B,C,D) to pool2. +# 2. Verify send from A(B, C, D) can be received onto it. +# 3. Verify send from A(B, C) can be received onto it. +# 4. Verify send from A() can be received onto it. +# 5. Verify send from A(E) cannot be received onto it. +# 6. Verify send from redaction bookmark for A(B, C) can be received onto it. +# 7. Verify send from redaction bookmark for A() can be received onto it. +# 8. Verify send from redaction bookmark for A(E) cannot be received onto it. +# + +typeset ds_name="origin" +typeset sendfs="$POOL/$ds_name" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' setup_incrementals +typeset dsA=$sendfs@snap0 +typeset dsB=$POOL/hole@snap +typeset dsC=$POOL/rm@snap +typeset dsD=$POOL/write@snap +typeset dsE=$POOL/stride3@snap +typeset dsF=$POOL/stride5@snap +typeset targ=$POOL2/targfs@snap + +log_onexit redacted_cleanup $sendfs $POOL2/rBCD $POOL2/targfs \ + $POOL2/rBC $POOL2/rE + +# Set up all the filesystems and clones. +log_must zfs redact $dsA BCD $dsB $dsC $dsD +log_must eval "zfs send --redact BCD $dsA >$stream" +log_must eval "zfs receive $POOL2/rBCD <$stream" +log_must eval "zfs receive $targ <$stream" + +log_must zfs redact $dsA BC $dsB $dsC +log_must eval "zfs send --redact BC $dsA >$stream" +log_must eval "zfs receive $POOL2/rBC <$stream" + +log_must zfs redact $dsA E $dsE +log_must eval "zfs send --redact E $dsA >$stream" +log_must eval "zfs receive $POOL2/rE <$stream" + +log_must eval "zfs send $dsF >$stream" +log_must eval "zfs receive -o origin=$POOL2/rBCD@snap0 $POOL2/BCDrF <$stream" +log_must eval "zfs receive -o origin=$POOL2/rBC@snap0 $POOL2/BCrF <$stream" +log_must eval "zfs receive -o origin=$POOL2/rE@snap0 $POOL2/ErF <$stream" + +# Run tests from redacted datasets. +log_must eval "zfs send -i $POOL2/rBCD@snap0 $POOL2/BCDrF@snap >$stream" +log_must eval "zfs receive -o origin=$targ $POOL2/tdBCD <$stream" + +log_must eval "zfs send -i $POOL2/rBC@snap0 $POOL2/BCrF@snap >$stream" +log_must eval "zfs receive -o origin=$targ $POOL2/tdBC <$stream" + +log_must eval "zfs send -i $POOL2/rE@snap0 $POOL2/ErF@snap >$stream" +log_mustnot eval "zfs receive -o origin=$targ $POOL2/tdE <$stream" + +# Run tests from redaction bookmarks. +log_must eval "zfs send -i $sendfs#BC $dsF >$stream" +log_must eval "zfs receive -o origin=$targ $POOL2/tbBC <$stream" + +log_must eval "zfs send -i $sendfs#E $dsF >$stream" +log_mustnot eval "zfs receive -o origin=$targ $POOL2/tbE <$stream" + +log_pass "Verify sends from redacted datasets and bookmarks work correctly." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh new file mode 100755 index 000000000..e4163c4ef --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_props.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify the list of redacted snapshot guids as properties. +# +# Strategy: +# 1. Create a redacted dataset and receive it into another pool. +# 2. Verify that the redaction list in the book mark (according to zdb) +# matches the list shown in the redact_snaps property. +# 3. Verify that the received snapshot has a matching redaction list. +# + +typeset ds_name="props" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset mntpnt + +log_onexit redacted_cleanup $sendfs $recvfs + +# Verify a plain dataset, snapshot or bookmark has an empty list. +log_must zfs snapshot $sendfs@empty_snapshot +log_must zfs bookmark $sendfs@empty_snapshot $sendfs#empty_bookmark +found_list=$(get_prop redact_snaps $sendfs) +[[ $found_list = "-" ]] || log_fail "Unexpected dataset list: $found_list" +found_list=$(get_prop redact_snaps $sendfs@empty_snapshot) +[[ $found_list = "-" ]] || log_fail "Unexpected snapshot list: $found_list" +found_list=$(get_prop redact_snaps $sendfs#empty_bookmark) +[[ $found_list = "-" ]] || log_fail "Unexpected bookmark list: $found_list" + +# Fill in a different block in every clone. +for i in {1..16}; do + log_must zfs clone $sendfs@snap ${clone}$i + mntpnt=$(get_prop mountpoint ${clone}$i) + log_must dd if=/dev/urandom of=$mntpnt/f2 bs=64k count=1 seek=$i \ + conv=notrunc + log_must zfs snapshot ${clone}$i@snap +done + +log_must zfs redact $sendfs@snap book1 $clone{1..16}@snap +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +log_must eval "zfs recv $recvfs <$stream" + +get_guid_list $tmpdir/prop_list $sendfs#book1 +get_guid_list $tmpdir/zdb_list $sendfs#book1 true +get_guid_list $tmpdir/recvd_prop_list $recvfs@snap + +count=$(wc -l $tmpdir/prop_list | awk '{print $1}') +[[ $count -eq 16 ]] || log_fail "Found incorrect number of redaction snapshots." + +diff $tmpdir/prop_list $tmpdir/zdb_list || \ + log_fail "Property list differed from zdb output" +diff $tmpdir/prop_list $tmpdir/recvd_prop_list || \ + log_fail "Received property list differed from sent" + +log_pass "The redaction list is consistent between sent and received datasets." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh new file mode 100755 index 000000000..9a3c5329d --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_resume.ksh @@ -0,0 +1,87 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that resumable send works correctly with redacted streams. +# +# Strategy: +# 1. Do a full redacted resumable send. +# 2. Verify the received contents are correct. +# 3. Do an incremental redacted resumable send. +# 4. Verify the received contents are correct. +# 5. Verify that recv -A removes a partially received dataset. +# + +typeset ds_name="resume" +typeset sendfs="$POOL/$ds_name" +typeset recvfs="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset clone1="$POOL/${ds_name}_clone1" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +setup_dataset $ds_name '' +typeset clone_mnt="$(get_prop mountpoint $clone)" +typeset send_mnt="$(get_prop mountpoint $sendfs)" +typeset recv_mnt="/$POOL2/$ds_name" + +log_onexit redacted_cleanup $sendfs $recvfs + +log_must stride_dd -i /dev/urandom -o $clone_mnt/f2 -b 512 -c 64 -s 512 +log_must zfs snapshot $clone@snap1 + +# Do the full resumable send +log_must zfs redact $sendfs@snap book1 $clone@snap1 +resume_test "zfs send --redact book1 $sendfs@snap" $tmpdir $recvfs +log_must mount_redacted -f $recvfs +log_must set_tunable32 zfs_allow_redacted_dataset_mount 1 +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must eval "get_diff $send_mnt/f2 $recv_mnt/f2 >$tmpdir/get_diff.out" +typeset range=$(cat $tmpdir/get_diff.out) +[[ "$RANGE9" = "$range" ]] || log_fail "Unexpected range: $range" + +log_must dd if=/dev/urandom of=$send_mnt/f3 bs=1024k count=3 +log_must zfs snapshot $sendfs@snap2 +log_must zfs clone $sendfs@snap2 $clone1 +typeset clone1_mnt="$(get_prop mountpoint $clone1)" +log_must dd if=/dev/urandom of=$clone1_mnt/f3 bs=128k count=3 conv=notrunc +log_must zfs snapshot $clone1@snap + +# Do the incremental resumable send +log_must zfs redact $sendfs@snap2 book2 $clone1@snap +resume_test "zfs send --redact book2 -i $sendfs#book1 $sendfs@snap2" \ + $tmpdir $recvfs +log_must diff $send_mnt/f1 $recv_mnt/f1 +log_must diff $send_mnt/f2 $recv_mnt/f2 +log_must eval "get_diff $send_mnt/f3 $recv_mnt/f3 >$tmpdir/get_diff.out" +range=$(cat $tmpdir/get_diff.out) +[[ "$RANGE10" = "$range" ]] || log_fail "Unexpected range: $range" + +# Test recv -A works properly +log_mustnot zfs recv -A $recvfs +log_must zfs destroy -R $recvfs +log_mustnot zfs recv -A $recvfs +log_must eval "zfs send --redact book1 $sendfs@snap >$stream" +dd if=$stream bs=64k count=1 | log_mustnot zfs receive -s $recvfs +[[ "-" = $(get_prop receive_resume_token $recvfs) ]] && \ + log_fail "Receive token not found." +log_must zfs recv -A $recvfs +log_must datasetnonexists $recvfs + +log_pass "Resumable send works correctly with redacted streams." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh new file mode 100755 index 000000000..81e7fe31d --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_size.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that send size estimates of redacted sends work correctly +# +# Strategy: +# 1. Perform a redacted send with -nv and without, and verify the +# size estimate is the same as the size of the actual send. +# 2. Receive an incremental send from the redaction bookmark with +# -nv and without, and verify the size estimate is the same as +# the size of the actual send. +# + +ds_name="sizes" +typeset sendfs="$POOL/$ds_name" +typeset clone="$POOL/${ds_name}_clone2" +setup_dataset $ds_name "-o compress=lz4" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset size=$(mktemp $tmpdir/size.XXXX) +typeset size2=$(mktemp $tmpdir/size.XXXX) + +log_onexit redacted_cleanup $sendfs $clone +log_must zfs clone $sendfs@snap $clone +typeset clone_mnt="$(get_prop mountpoint $clone)" +log_must rm -rf $clone_mnt/* +log_must zfs snapshot $clone@snap +log_must zfs redact $sendfs@snap book $clone@snap +log_must eval "zfs send -nvP --redact book $sendfs@snap | \ + grep '^size' | awk '{print \$2}' >$size" +log_must eval "zfs send --redact book $sendfs@snap | wc --bytes \ + >$size2" +bytes1=$(cat $size | tr -d '[[:space:]]') +bytes2=$(cat $size2 | tr -d '[[:space:]]') +[[ "$bytes1" -eq "$bytes2" ]] || \ + log_fail "Full sizes differ: estimate $bytes1 and actual $bytes2" + +log_must zfs snapshot $sendfs@snap2 +log_must eval "zfs send -nvP -i $sendfs#book $sendfs@snap2 | \ + grep '^size' | awk '{print \$2}' >$size" +log_must eval "zfs send -i $sendfs#book $sendfs@snap2 | wc --bytes >$size2" +bytes1=$(cat $size | tr -d '[[:space:]]') +bytes2=$(cat $size2 | tr -d '[[:space:]]') +[[ "$bytes1" -eq "$bytes2" ]] || \ + log_fail "Incremental sizes differ: estimate $bytes1 and actual $bytes2" + +log_pass "Size estimates of redacted sends estimate accurately." diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh new file mode 100755 index 000000000..b3c519b7d --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_volume.ksh @@ -0,0 +1,105 @@ +#!/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +# +# Description: +# Verify that redacted send works on volumes. +# +# Strategy: +# 1. Write to a volume, then make a clone of that volume. +# 2. Receive a redacted stream that sends all blocks. +# 3. Receive a redacted stream that redacts the first half of the written area. +# + +typeset ds_name="volume" +typeset sendvol="$POOL/$ds_name" +typeset recvvol="$POOL2/$ds_name" +typeset clone="$POOL/${ds_name}_clone" +typeset tmpdir="$(get_prop mountpoint $POOL)/tmp" +typeset stream=$(mktemp $tmpdir/stream.XXXX) +typeset send_file="/dev/zvol/$sendvol" +typeset recv_file="/dev/zvol/$recvvol" +typeset clone_file="/dev/zvol/$clone" + +log_onexit redacted_cleanup $sendvol $recvvol + +log_must zfs create -b 8k -V 1g $sendvol +sleep 10 +log_must zpool export $POOL +log_must zpool import $POOL +udevadm settle +if [[ ! -b $send_file ]]; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + [[ -b $send_file ]] && break + done +fi +log_must dd if=/dev/urandom of=$send_file bs=8k count=64 +log_must zfs snapshot $sendvol@snap +log_must zfs clone $sendvol@snap $clone +log_must zfs snapshot $clone@snap + +log_must set_tunable32 zfs_allow_redacted_dataset_mount 1 +log_must zfs redact $sendvol@snap book1 $clone@snap +log_must eval "zfs send --redact book1 $sendvol@snap >$stream" +log_must eval "zfs recv $recvvol <$stream" +sleep 10 +log_must zpool export $POOL2 +log_must zpool import $POOL2 +udevadm settle +if [[ ! -b $recv_file ]]; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + [[ -b $recv_file ]] && break + done +fi +log_must dd if=$send_file of=$tmpdir/send.dd bs=8k count=64 +log_must dd if=$recv_file of=$tmpdir/recv.dd bs=8k count=64 +log_must diff $tmpdir/send.dd $tmpdir/recv.dd +log_must zfs destroy -R $recvvol + +log_must dd if=/dev/urandom of=$clone_file bs=8k count=32 +log_must zfs snapshot $clone@snap1 +log_must zfs redact $sendvol@snap book2 $clone@snap1 +log_must eval "zfs send --redact book2 $sendvol@snap >$stream" +log_must eval "zfs recv $recvvol <$stream" +sleep 10 +log_must zpool export $POOL2 +log_must zpool import $POOL2 +udevadm settle +if [[ ! -b $recv_file ]]; then + udevadm settle + for t in 10 5 3 2 1; do + log_note "Polling $t seconds for device file." + udevadm settle + sleep $t + [[ -b $recv_file ]] && break + done +fi +log_must dd if=$send_file of=$tmpdir/send.dd bs=8k count=32 skip=32 +log_must dd if=$recv_file of=$tmpdir/recv.dd bs=8k count=32 skip=32 +log_must diff $tmpdir/send.dd $tmpdir/recv.dd + +log_pass "Redacted send works correctly with volumes." diff --git a/tests/zfs-tests/tests/functional/redacted_send/setup.ksh b/tests/zfs-tests/tests/functional/redacted_send/setup.ksh new file mode 100755 index 000000000..3f537f813 --- /dev/null +++ b/tests/zfs-tests/tests/functional/redacted_send/setup.ksh @@ -0,0 +1,36 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/redacted_send/redacted.kshlib + +verify_disk_count "$DISKS" 2 + +create_pool $POOL $DISK1 +log_must zfs snapshot $POOL@init +create_pool $POOL2 $DISK2 +log_must zfs snapshot $POOL2@init +log_must zfs create $POOL/tmp +log_pass diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib index 521a1c7eb..e6fc9b7d6 100644 --- a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib +++ b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib @@ -25,7 +25,7 @@ # # -# Copyright (c) 2013, 2016 by Delphix. All rights reserved. +# Copyright (c) 2013, 2018 by Delphix. All rights reserved. # . $STF_SUITE/include/libtest.shlib @@ -635,7 +635,7 @@ function resume_test stream_num=$((stream_num+1)) token=$(zfs get -Hp -o value receive_resume_token $recvfs) - log_must eval "zfs send -v -t $token >/$streamfs/$stream_num" + log_must eval "zfs send -t $token >/$streamfs/$stream_num" [[ -f /$streamfs/$stream_num ]] || \ log_fail "NO FILE /$streamfs/$stream_num" done diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh new file mode 100755 index 000000000..4610802e9 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/rsend_016_neg.ksh @@ -0,0 +1,33 @@ +#!/usr/bin/ksh + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2014, 2018 by Delphix. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that error conditions don't cause panics in zfs send +# +# Strategy: +# 1. Perform a zfs incremental send from a bookmark that doesn't exist +# + +verify_runnable "both" + +log_neg eval "zfs send -i \#bla $POOl/$FS@final > /dev/null" + +log_pass "Ensure that error conditions cause appropriate failures." |