diff options
Diffstat (limited to 'tests')
32 files changed, 1553 insertions, 37 deletions
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index b8c0efa36..406491aa1 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -568,15 +568,21 @@ tests = ['reservation_001_pos', 'reservation_002_pos', 'reservation_003_pos', tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos'] # DISABLED: -# rsend_008_pos - Fails for OpenZFS on illumos +# rsend_008_pos - https://github.com/zfsonlinux/zfs/issues/6066 # rsend_009_pos - https://github.com/zfsonlinux/zfs/issues/5887 -# rsend_020_pos - ASSERTs in dump_record() [tests/functional/rsend] tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', - 'rsend_013_pos', 'rsend_014_pos', 'rsend_019_pos', - 'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos'] + 'rsend_013_pos', 'rsend_014_pos', + 'rsend_019_pos', 'rsend_020_pos', + 'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos', + 'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props', + 'send-c_incremental', 'send-c_volume', 'send-c_zstreamdump', + 'send-c_lz4_disabled', 'send-c_recv_lz4_disabled', + 'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD', + 'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize', + 'send-c_recv_dedup'] [tests/functional/scrub_mirror] tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos', diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg index 5dc8e32aa..aa8f69821 100644 --- a/tests/zfs-tests/include/commands.cfg +++ b/tests/zfs-tests/include/commands.cfg @@ -134,7 +134,8 @@ export ZFS_FILES='zdb arc_summary.py arcstat.py dbufstat.py - zed' + zed + zstreamdump' export ZFSTEST_FILES='chg_usr_exec devname2devid diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index 8984071c8..07cf9a1e6 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -29,6 +29,7 @@ # . ${STF_TOOLS}/include/logapi.shlib +. ${STF_SUITE}/include/math.shlib # # Apply constrained path when available. This is required since the @@ -3112,6 +3113,73 @@ function get_min } # +# Generate a random number between 1 and the argument. +# +function random +{ + typeset max=$1 + echo $(( ($RANDOM % $max) + 1 )) +} + +# Write data that can be compressed into a directory +function write_compressible +{ + typeset dir=$1 + typeset megs=$2 + typeset nfiles=${3:-1} + typeset bs=${4:-1024k} + typeset fname=${5:-file} + + [[ -d $dir ]] || log_fail "No directory: $dir" + + # Under Linux fio is not currently used since its behavior can + # differ significantly across versions. This includes missing + # command line options and cases where the --buffer_compress_* + # options fail to behave as expected. + if is_linux; then + typeset file_bytes=$(to_bytes $megs) + typeset bs_bytes=4096 + typeset blocks=$(($file_bytes / $bs_bytes)) + + for (( i = 0; i < $nfiles; i++ )); do + truncate -s $file_bytes $dir/$fname.$i + + # Write every third block to get 66% compression. + for (( j = 0; j < $blocks; j += 3 )); do + dd if=/dev/urandom of=$dir/$fname.$i \ + seek=$j bs=$bs_bytes count=1 \ + conv=notrunc >/dev/null 2>&1 + done + done + else + log_must eval "fio \ + --name=job \ + --fallocate=0 \ + --minimal \ + --randrepeat=0 \ + --buffer_compress_percentage=66 \ + --buffer_compress_chunk=4096 \ + --directory=$dir \ + --numjobs=$nfiles \ + --nrfiles=$nfiles \ + --rw=write \ + --bs=$bs \ + --filesize=$megs \ + --filename_format='$fname.\$jobnum' >/dev/null" + fi +} + +function get_objnum +{ + typeset pathname=$1 + typeset objnum + + [[ -e $pathname ]] || log_fail "No such file or directory: $pathname" + objnum=$(stat -c %i $pathname) + echo $objnum +} + +# # Synchronize all the data in pool # # $1 pool name diff --git a/tests/zfs-tests/include/math.shlib b/tests/zfs-tests/include/math.shlib index ca1cbcb4e..4ed11fb4b 100644 --- a/tests/zfs-tests/include/math.shlib +++ b/tests/zfs-tests/include/math.shlib @@ -41,3 +41,28 @@ function within_percent return 1 } + +# +# Return 0 if the human readable string of the form <value>[suffix] can +# be converted to bytes. Allow suffixes are shown in the table below. +# +function to_bytes +{ + typeset size=$1 + typeset value=$(echo "$size" | grep -o '[0-9]\+') + + case $size in + *PB|*pb|*P|*p) factor='1024^5' ;; + *TB|*tb|*T|*t) factor='1024^4' ;; + *GB|*gb|*G|*g) factor='1024^3' ;; + *MB|*mb|*M|*m) factor='1024^2' ;; + *KB|*kb|*K|*k) factor='1024^1' ;; + *B|*b) factor='1024^0' ;; + *[!0-9.]*) return 1 ;; + *) factor='1024^0' ;; + esac + + echo "$value * ($factor)" | bc + + return 0 +} diff --git a/tests/zfs-tests/include/properties.shlib b/tests/zfs-tests/include/properties.shlib index 8817b6f2c..6a3a0e1ce 100644 --- a/tests/zfs-tests/include/properties.shlib +++ b/tests/zfs-tests/include/properties.shlib @@ -13,11 +13,26 @@ # Copyright (c) 2012, 2016 by Delphix. All rights reserved. # -typeset -a compress_props=('on' 'off' 'lzjb' 'gzip' 'gzip-1' 'gzip-2' 'gzip-3' - 'gzip-4' 'gzip-5' 'gzip-6' 'gzip-7' 'gzip-8' 'gzip-9' 'zle') +typeset -a compress_prop_vals=('on' 'off' 'lzjb' 'gzip' 'gzip-1' 'gzip-2' + 'gzip-3' 'gzip-4' 'gzip-5' 'gzip-6' 'gzip-7' 'gzip-8' 'gzip-9' 'zle' 'lz4') +typeset -a checksum_prop_vals=('on' 'off' 'fletcher2' 'fletcher4' 'sha256' + 'noparity' 'sha512' 'skein' 'edonr') +typeset -a recsize_prop_vals=('512' '1024' '2048' '4096' '8192' '16384' + '32768' '65536' '131072' '262144' '524288' '1048576') +typeset -a canmount_prop_vals=('on' 'off' 'noauto') +typeset -a copies_prop_vals=('1' '2' '3') +typeset -a logbias_prop_vals=('latency' 'throughput') +typeset -a primarycache_prop_vals=('all' 'none' 'metadata') +typeset -a redundant_metadata_prop_vals=('all' 'most') +typeset -a secondarycache_prop_vals=('all' 'none' 'metadata') +typeset -a snapdir_prop_vals=('hidden' 'visible') +typeset -a sync_prop_vals=('standard' 'always' 'disabled') -typeset -a checksum_props=('on' 'off' 'fletcher2' 'fletcher4' 'sha256' 'sha512' - 'edonr' 'skein' 'noparity') +typeset -a fs_props=('compress' 'checksum' 'recsize' + 'canmount' 'copies' 'logbias' 'primarycache' 'redundant_metadata' + 'secondarycache' 'snapdir' 'sync') +typeset -a vol_props=('compress' 'checksum' 'copies' 'logbias' 'primarycache' + 'secondarycache' 'redundant_metadata' 'sync') # # Given the property array passed in, return 'num_props' elements to the @@ -45,20 +60,81 @@ function get_rand_prop function get_rand_compress { - get_rand_prop compress_props $1 2 + get_rand_prop compress_prop_vals $1 2 } function get_rand_compress_any { - get_rand_prop compress_props $1 0 + get_rand_prop compress_prop_vals $1 0 } function get_rand_checksum { - get_rand_prop checksum_props $1 2 + get_rand_prop checksum_prop_vals $1 2 } function get_rand_checksum_any { - get_rand_prop checksum_props $1 0 + get_rand_prop checksum_prop_vals $1 0 +} + +function get_rand_recsize +{ + get_rand_prop recsize_prop_vals $1 0 +} + +function get_rand_large_recsize +{ + get_rand_prop recsize_prop_vals $1 9 +} + +# +# Functions to toggle on/off properties +# +typeset -a binary_props=('atime' 'devices' 'exec' 'nbmand' 'readonly' 'setuid' + 'xattr' 'zoned') + +function toggle_prop +{ + typeset ds=$1 + typeset prop=$2 + + datasetexists $ds || log_fail "$ds does not exist" + typeset val=$(get_prop $prop $ds) + typeset newval='off' + + [[ $val = $newval ]] && newval='on' + log_must zfs set $prop=$newval $ds +} + +function toggle_binary_props +{ + typeset ds=$1 + typeset prop + + for prop in "${binary_props[@]}"; do + toggle_prop $ds $prop + done +} + +function randomize_ds_props +{ + typeset ds=$1 + typeset prop proplist val + + datasetexists $ds || log_fail "$ds does not exist" + if ds_is_volume $ds; then + toggle_prop $ds readonly + proplist="${vol_props[@]}" + elif ds_is_filesystem $ds; then + toggle_binary_props $ds + proplist="${fs_props[@]}" + else + log_fail "$ds is neither a volume nor a file system" + fi + + for prop in $proplist; do + typeset val=$(get_rand_prop "${prop}_prop_vals" 1 0) + log_must zfs set $prop=$val $ds + done } diff --git a/tests/zfs-tests/tests/functional/rsend/Makefile.am b/tests/zfs-tests/tests/functional/rsend/Makefile.am index 6be0000be..b9f8dba65 100644 --- a/tests/zfs-tests/tests/functional/rsend/Makefile.am +++ b/tests/zfs-tests/tests/functional/rsend/Makefile.am @@ -22,4 +22,19 @@ dist_pkgdata_SCRIPTS = \ rsend_020_pos.ksh \ rsend_021_pos.ksh \ rsend_022_pos.ksh \ - rsend_024_pos.ksh + rsend_024_pos.ksh \ + send-cD.ksh \ + send-c_embedded_blocks.ksh \ + send-c_incremental.ksh \ + send-c_lz4_disabled.ksh \ + send-c_mixed_compression.ksh \ + send-c_props.ksh \ + send-c_recv_dedup.ksh \ + send-c_recv_lz4_disabled.ksh \ + send-c_resume.ksh \ + send-c_stream_size_estimate.ksh \ + send-c_verify_contents.ksh \ + send-c_verify_ratio.ksh \ + send-c_volume.ksh \ + send-c_zstreamdump.ksh \ + send-cpL_varied_recsize.ksh diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.cfg b/tests/zfs-tests/tests/functional/rsend/rsend.cfg index 8984ee2a7..8400ecfe3 100644 --- a/tests/zfs-tests/tests/functional/rsend/rsend.cfg +++ b/tests/zfs-tests/tests/functional/rsend/rsend.cfg @@ -29,11 +29,11 @@ export BACKDIR=${TEST_BASE_DIR%%/}/backdir-rsend -export DISK1=${DISKS%% *} +export DISK1=$(echo $DISKS | awk '{print $1}') export DISK2=$(echo $DISKS | awk '{print $2}') export DISK3=$(echo $DISKS | awk '{print $3}') export POOL=$TESTPOOL -export POOL2=$TESTPOOL1 -export POOL3=$TESTPOOL2 +export POOL2=$TESTPOOL2 +export POOL3=$TESTPOOL3 export FS=$TESTFS diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib index ac07dcf90..fde4123be 100644 --- a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib +++ b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib @@ -29,6 +29,7 @@ # . $STF_SUITE/include/libtest.shlib +. $STF_SUITE/include/math.shlib . $STF_SUITE/tests/functional/rsend/rsend.cfg # @@ -207,11 +208,10 @@ function cmp_ds_prop typeset dtst1=$1 typeset dtst2=$2 - for item in "type" "origin" "volblocksize" "aclinherit" "acltype" \ + for item in "type" "origin" "volblocksize" "acltype" "dnodesize" \ "atime" "canmount" "checksum" "compression" "copies" "devices" \ - "dnodesize" "exec" "quota" "readonly" "recordsize" "reservation" \ - "setuid" "snapdir" "version" "volsize" "xattr" "zoned" \ - "mountpoint"; + "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \ + "snapdir" "version" "volsize" "xattr" "zoned" "mountpoint"; do zfs get -H -o property,value,source $item $dtst1 >> \ $BACKDIR/dtst1 @@ -525,20 +525,20 @@ function test_fs_setup mk_files 100 1048576 0 $sendfs & mk_files 10 10485760 0 $sendfs & mk_files 1 104857600 0 $sendfs & - wait + log_must wait log_must zfs snapshot $sendfs@a rm_files 200 256 0 $sendfs & rm_files 200 131072 0 $sendfs & rm_files 20 1048576 0 $sendfs & rm_files 2 10485760 0 $sendfs & - wait + log_must wait mk_files 400 256 0 $sendfs & mk_files 400 131072 0 $sendfs & mk_files 40 1048576 0 $sendfs & mk_files 4 10485760 0 $sendfs & - wait + log_must wait log_must zfs snapshot $sendfs@b log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend" @@ -551,3 +551,121 @@ function test_fs_setup fi log_must zfs create -o compress=lz4 $sendpool/stream } + +# +# Check to see if the specified features are set in a send stream. +# The values for these features are found in include/sys/zfs_ioctl.h +# +# $1 The stream file +# $2-$n The flags expected in the stream +# +function stream_has_features +{ + typeset file=$1 + shift + + [[ -f $file ]] || log_fail "Couldn't find file: $file" + typeset flags=$(cat $file | zstreamdump | \ + awk '/features =/ {features = $3} END {print features}') + typeset -A feature + feature[dedup]="1" + feature[dedupprops]="2" + feature[sa_spill]="4" + feature[embed_data]="10000" + feature[lz4]="20000" + feature[mooch_byteswap]="40000" + feature[large_blocks]="80000" + feature[resuming]="100000" + feature[redacted]="200000" + feature[compressed]="400000" + + typeset flag known derived=0 + for flag in "$@"; do + known=${feature[$flag]} + [[ -z $known ]] && log_fail "Unknown feature: $flag" + + derived=$(printf "%x" $((0x${flags} & 0x${feature[$flag]}))) + [[ $derived = $known ]] || return 1 + done + + return 0 +} + +# +# Parse zstreamdump -v output. The output varies for each kind of record: +# BEGIN records are simply output as "BEGIN" +# END records are output as "END" +# OBJECT records become "OBJECT <object num>" +# FREEOBJECTS records become "FREEOBJECTS <startobj> <numobjs>" +# FREE records become "<record type> <start> <length>" +# WRITE records become: +# "<record type> <compression type> <start> <logical size> <compressed size> +# <data size>" +# +function parse_dump +{ + sed '/^WRITE/{N;s/\n/ /;}' | grep "^[A-Z]" | awk '{ + if ($1 == "BEGIN" || $1 == "END") print $1 + if ($1 == "OBJECT") print $1" "$4 + if ($1 == "FREEOBJECTS") print $1" "$4" "$7 + if ($1 == "FREE") print $1" "$7" "$10 + if ($1 == "WRITE") print $1" "$15" "$18" "$21" "$24" "$27}' +} + +# +# Given a send stream, verify that the size of the stream matches what's +# expected based on the source or target dataset. If the stream is an +# incremental stream, subtract the size of the source snapshot before +# comparing. This function does not currently handle incremental streams +# that remove data. +# +# $1 The zstreamdump output file +# $2 The dataset to compare against +# This can be a source of a send or recv target (fs, not snapshot) +# $3 The percentage below which verification is deemed a failure +# $4 The source snapshot of an incremental send +# + +function verify_stream_size +{ + typeset stream=$1 + typeset ds=$2 + typeset percent=${3:-90} + typeset inc_src=$4 + + [[ -f $stream ]] || log_fail "No such file: $stream" + datasetexists $ds || log_fail "No such dataset: $ds" + + typeset stream_size=$(cat $stream | zstreamdump | sed -n \ + 's/ Total write size = \(.*\) (0x.*)/\1/p') + + typeset inc_size=0 + if [[ -n $inc_src ]]; then + inc_size=$(get_prop lrefer $inc_src) + if stream_has_features $stream compressed; then + inc_size=$(get_prop refer $inc_src) + fi + fi + + if stream_has_features $stream compressed; then + ds_size=$(get_prop refer $ds) + else + ds_size=$(get_prop lrefer $ds) + fi + ds_size=$((ds_size - inc_size)) + + within_percent $stream_size $ds_size $percent || log_fail \ + "$stream_size $ds_size differed by too much" +} + +# Cleanup function for tests involving resumable send +function resume_cleanup +{ + typeset sendfs=$1 + typeset streamfs=$2 + + datasetexists $sendfs && log_must zfs destroy -r $sendfs + datasetexists $streamfs && log_must zfs destroy -r $streamfs + cleanup_pool $POOL2 + rm -f /$POOL/initial.zsend /$POOL/incremental.zsend +} diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_019_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_019_pos.ksh index 5858bd2f2..8969ba9e8 100755 --- a/tests/zfs-tests/tests/functional/rsend/rsend_019_pos.ksh +++ b/tests/zfs-tests/tests/functional/rsend/rsend_019_pos.ksh @@ -37,8 +37,9 @@ verify_runnable "both" log_assert "Verify resumability of a full and incremental ZFS send/receive " \ "in the presence of a corrupted stream" -log_onexit cleanup_pools $POOL2 $POOL3 +log_onexit resume_cleanup $sendfs $streamfs +sendfs=$POOL/sendfs recvfs=$POOL3/recvfs streamfs=$POOL2/stream diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh index 25e125b4f..29bdc4f84 100755 --- a/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh +++ b/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh @@ -35,12 +35,13 @@ verify_runnable "both" log_assert "Verify resumability of full ZFS send/receive with the -D " \ "(dedup) flag" -log_onexit cleanup_pool $POOL2 sendfs=$POOL/sendfs recvfs=$POOL2/recvfs streamfs=$POOL/stream +log_onexit resume_cleanup $sendfs $streamfs + test_fs_setup $sendfs $recvfs resume_test "zfs send -D -v $sendfs@a" $streamfs $recvfs file_check $sendfs $recvfs diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh index 30af5ba43..d74844890 100755 --- a/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh +++ b/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh @@ -37,12 +37,13 @@ verify_runnable "both" log_assert "Verify resumability of a full and incremental ZFS send/receive " \ "with the -e (embedded) flag" -log_onexit cleanup_pool $POOL2 sendfs=$POOL/sendfs recvfs=$POOL2/recvfs streamfs=$POOL/stream +log_onexit resume_cleanup $sendfs $streamfs + test_fs_setup $sendfs $recvfs resume_test "zfs send -v -e $sendfs@a" $streamfs $recvfs resume_test "zfs send -v -e -i @a $sendfs@b" $streamfs $recvfs diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh index 278f1535f..487a01426 100755 --- a/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh +++ b/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh @@ -45,12 +45,13 @@ fi log_assert "Verify resumability of an incremental ZFS send/receive with ZFS " \ "bookmarks" -log_onexit cleanup_pool $POOL2 sendfs=$POOL/sendfs recvfs=$POOL2/recvfs streamfs=$POOL/stream +log_onexit resume_cleanup $sendfs $streamfs + test_fs_setup $sendfs $recvfs log_must zfs bookmark $sendfs@a $sendfs#bm_a log_must zfs destroy $sendfs@a diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh index f61be9442..e7453a0e1 100755 --- a/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh +++ b/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh @@ -42,12 +42,13 @@ fi log_assert "Verify resumability of a full ZFS send/receive with the source " \ "filesystem unmounted" -log_onexit cleanup_pool $POOL2 sendfs=$POOL/sendfs recvfs=$POOL2/recvfs streamfs=$POOL/stream +log_onexit resume_cleanup $sendfs $streamfs + test_fs_setup $sendfs $recvfs log_must zfs unmount $sendfs resume_test "zfs send $sendfs" $streamfs $recvfs diff --git a/tests/zfs-tests/tests/functional/rsend/send-cD.ksh b/tests/zfs-tests/tests/functional/rsend/send-cD.ksh new file mode 100755 index 000000000..c51788f81 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-cD.ksh @@ -0,0 +1,77 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that the -c and -D flags do not interfere with each other. +# +# Strategy: +# 1. Write unique data to a filesystem and create a compressed, deduplicated +# full stream. +# 2. Verify that the stream and send dataset show the same size +# 3. Make several copies of the original data, and create both full and +# incremental compressed, deduplicated send streams +# 4. Verify the full stream is no bigger than the stream from step 1 +# 5. Verify the streams can be received correctly. +# + +verify_runnable "both" + +log_assert "Verify that the -c and -D flags do not interfere with each other" +log_onexit cleanup_pool $POOL2 + +typeset sendfs=$POOL2/sendfs +typeset recvfs=$POOL2/recvfs +typeset stream0=$BACKDIR/stream.0 +typeset stream1=$BACKDIR/stream.1 +typeset inc=$BACKDIR/stream.inc + +log_must zfs create -o compress=lz4 $sendfs +log_must zfs create -o compress=lz4 $recvfs +typeset dir=$(get_prop mountpoint $sendfs) +# Don't use write_compressible: we want compressible but undedupable data here. +log_must file_write -o overwrite -f $dir/file -d R -b 4096 -c 1000 +log_must zfs snapshot $sendfs@snap0 +log_must eval "zfs send -D -c $sendfs@snap0 >$stream0" + +# The stream size should match at this point because the data is all unique +verify_stream_size $stream0 $sendfs + +for i in {0..3}; do + log_must cp $dir/file $dir/file.$i +done +log_must zfs snapshot $sendfs@snap1 + +# The stream sizes should match, since the second stream contains no new blocks +log_must eval "zfs send -D -c $sendfs@snap1 >$stream1" +typeset size0=$(stat -c %s $stream0) +typeset size1=$(stat -c %s $stream1) +within_percent $size0 $size1 90 || log_fail "$size0 and $size1" + +# Finally, make sure the receive works correctly. +log_must eval "zfs send -D -c -i snap0 $sendfs@snap1 >$inc" +log_must eval "zfs recv -d $recvfs <$stream0" +log_must eval "zfs recv -d $recvfs <$inc" +cmp_ds_cont $sendfs $recvfs + +# The size of the incremental should be the same as the initial send. +typeset size2=$(stat -c %s $inc) +within_percent $size0 $size2 90 || log_fail "$size0 and $size1" + +log_pass "The -c and -D flags do not interfere with each other" diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_embedded_blocks.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_embedded_blocks.ksh new file mode 100755 index 000000000..1983a3ea1 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_embedded_blocks.ksh @@ -0,0 +1,109 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify that compressed streams can contain embedded blocks. +# +# Strategy: +# 1. Create a filesystem with compressible data and embedded blocks. +# 2. Verify the created streams can be received correctly. +# 3. Verify the presence / absence of embedded blocks in the compressed stream, +# as well as the receiving file system. +# + +verify_runnable "both" + +log_assert "Verify that compressed streams can contain embedded blocks." +log_onexit cleanup_pool $POOL2 + +typeset objs obj recsize +typeset sendfs=$POOL2/sendfs +typeset recvfs=$POOL2/recvfs +typeset stream=$BACKDIR/stream +typeset dump=$BACKDIR/dump +typeset recvfs2=$POOL2/recvfs2 +typeset stream2=$BACKDIR/stream2 +typeset dump2=$BACKDIR/dump2 +log_must zfs create -o compress=lz4 $sendfs +log_must zfs create -o compress=lz4 $recvfs +log_must zfs create -o compress=lz4 $recvfs2 +typeset dir=$(get_prop mountpoint $sendfs) + +# Populate the send dataset with compressible data and embedded block files. +write_compressible $dir 16m +for recsize in "${recsize_prop_vals[@]}"; do + # For lz4, this method works for blocks up to 16k, but not larger + [[ $recsize -eq $((32 * 1024)) ]] && break + + if is_linux; then + log_must truncate -s $recsize $dir/$recsize + log_must dd if=/dev/urandom of=$dir/$recsize \ + seek=$((recsize - 8)) bs=1 count=8 conv=notrunc + else + log_must mkholes -h 0:$((recsize - 8)) -d $((recsize - 8)):8 \ + $dir/$recsize + fi +done + +# Generate the streams and zstreamdump output. +log_must zfs snapshot $sendfs@now +log_must eval "zfs send -c $sendfs@now >$stream" +log_must eval "zstreamdump -v <$stream >$dump" +log_must eval "zfs recv -d $recvfs <$stream" +cmp_ds_cont $sendfs $recvfs +verify_stream_size $stream $sendfs +log_mustnot stream_has_features $stream embed_data + +log_must eval "zfs send -c -e $sendfs@now >$stream2" +log_must eval "zstreamdump -v <$stream2 >$dump2" +log_must eval "zfs recv -d $recvfs2 <$stream2" +cmp_ds_cont $sendfs $recvfs2 +verify_stream_size $stream2 $sendfs +log_must stream_has_features $stream2 embed_data + +# Verify embedded blocks are present only when expected. +for recsize in "${recsize_prop_vals[@]}"; do + [[ $recsize -eq $((32 * 1024)) ]] && break + + typeset send_obj=$(get_objnum $(get_prop mountpoint $sendfs)/$recsize) + typeset recv_obj=$(get_objnum \ + $(get_prop mountpoint $recvfs/sendfs)/$recsize) + typeset recv2_obj=$(get_objnum \ + $(get_prop mountpoint $recvfs2/sendfs)/$recsize) + + log_must eval "zdb -ddddd $sendfs $send_obj >$BACKDIR/sendfs.zdb" + log_must eval "zdb -ddddd $recvfs/sendfs $recv_obj >$BACKDIR/recvfs.zdb" + log_must eval "zdb -ddddd $recvfs2/sendfs $recv2_obj >$BACKDIR/recvfs2.zdb" + + grep -q "EMBEDDED" $BACKDIR/sendfs.zdb || \ + log_fail "Obj $send_obj not embedded in $sendfs" + grep -q "EMBEDDED" $BACKDIR/recvfs.zdb || \ + log_fail "Obj $recv_obj not embedded in $recvfs" + grep -q "EMBEDDED" $BACKDIR/recvfs2.zdb || \ + log_fail "Obj $recv2_obj not embedded in $recvfs2" + + grep -q "WRITE_EMBEDDED object = $send_obj offset = 0" $dump && \ + log_fail "Obj $obj embedded in zstreamdump output" + grep -q "WRITE_EMBEDDED object = $send_obj offset = 0" $dump2 || \ + log_fail "Obj $obj not embedded in zstreamdump output" +done + +log_pass "Compressed streams can contain embedded blocks." diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_incremental.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_incremental.ksh new file mode 100755 index 000000000..d673282c9 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_incremental.ksh @@ -0,0 +1,100 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that compressed send works correctly with incremental sends. +# +# Strategy: +# 1. Randomly choose either a -i or -I incremental. +# 2. Generate compressed incremental replication streams for a pool, a +# descendant dataset, and a volume. +# 3. Receive these streams verifying both the contents, and intermediate +# snapshots are present or absent as appropriate to the -i or -I option. +# + +verify_runnable "both" + +log_assert "Verify compressed send works with incremental send streams." +log_onexit cleanup_pool $POOL2 + +typeset opt=$(random_get "-i" "-I") +typeset final dstlist list vol + +log_must eval "zfs send -R $POOL@final > $BACKDIR/final" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/final" + +function do_checks +{ + log_must cmp_ds_cont $POOL $POOL2 + [[ $opt = "-I" ]] && log_must cmp_ds_subs $POOL $POOL2 + [[ $opt = "-i" ]] && log_mustnot cmp_ds_subs $POOL $POOL2 + + [[ $1 != "clean" ]] && return + + cleanup_pool $POOL2 + log_must eval "zfs send -R $POOL@final > $BACKDIR/final" + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/final" +} + +if is_global_zone; then + # Send from the pool root + final=$(getds_with_suffix $POOL2 @final) + list="$final $(getds_with_suffix $POOL2 @snapA)" + list="$list $(getds_with_suffix $POOL2 @snapB)" + list="$list $(getds_with_suffix $POOL2 @snapC)" + + log_must eval "zfs send -c -R $opt @init $POOL2@final >$BACKDIR/pool" + log_must destroy_tree $list + log_must eval "zfs recv -d -F $POOL2 <$BACKDIR/pool" + + dstlist=$(getds_with_suffix $POOL2 @final) + [[ $final != $dstlist ]] && log_fail "$final != $dstlist" + + do_checks clean + + # Send of a volume + vol=$POOL2/$FS/vol + final=$(getds_with_suffix $vol @final) + log_must eval "zfs send -c -R $opt @init $vol@final >$BACKDIR/vol" + log_must destroy_tree $vol@snapB $vol@snapC $vol@final + log_must eval "zfs recv -d -F $POOL2 <$BACKDIR/vol" + + dstlist=$(getds_with_suffix $POOL2/$FS/vol @final) + [[ $final != $dstlist ]] && log_fail "$final != $dstlist" + + do_checks clean +fi + +# Send of a descendant fs +final=$(getds_with_suffix $POOL2/$FS @final) +list="$final $(getds_with_suffix $POOL2/$FS @snapA)" +list="$list $(getds_with_suffix $POOL2/$FS @snapB)" +list="$list $(getds_with_suffix $POOL2/$FS @snapC)" + +log_must eval "zfs send -c -R $opt @init $POOL2/$FS@final >$BACKDIR/fs" +log_must destroy_tree $list +log_must eval "zfs recv -d -F $POOL2 <$BACKDIR/fs" + +dstlist=$(getds_with_suffix $POOL2/$FS @final) +[[ $final != $dstlist ]] && log_fail "$final != $dstlist" + +do_checks + +log_pass "Compressed send works with incremental send streams." diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh new file mode 100755 index 000000000..70e200bad --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh @@ -0,0 +1,73 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify a pool without the lz4 feature enabled can create compressed send +# streams, and that they can be received into pools with or without the +# lz4 feature. +# +# Strategy: +# 1. For each of an uncompressed, and gzip dataset created from a pool with +# the lz4 feature disabled, receive the stream into a pool with and without +# the feature enabled. +# + +verify_runnable "both" + +log_assert "Verify compressed streams are rejected if incompatible." + +typeset send_ds=$POOL2/testds +typeset recv_ds=$POOL3/testds + +function cleanup +{ + poolexists $POOL2 && destroy_pool $POOL2 + poolexists $POOL3 && destroy_pool $POOL3 + log_must zpool create $POOL2 $DISK2 +} +log_onexit cleanup + +datasetexists $POOL2 && log_must zpool destroy $POOL2 +log_must zpool create -d $POOL2 $DISK2 + +for compress in off gzip; do + for pool_opt in '' -d; do + poolexists $POOL3 && destroy_pool $POOL3 + log_must zpool create $pool_opt $POOL3 $DISK3 + + datasetexists $send_ds && log_must zfs destroy -r $send_ds + datasetexists $recv_ds && log_must zfs destroy -r $recv_ds + + log_must zfs create -o compress=$compress $send_ds + typeset dir=$(get_prop mountpoint $send_ds) + write_compressible $dir 16m + log_must zfs snapshot $send_ds@full + + log_must eval "zfs send -c $send_ds@full >$BACKDIR/full-c" + log_must eval "zfs recv $recv_ds <$BACKDIR/full-c" + + log_must zfs destroy -r $recv_ds + + log_must eval "zfs send $send_ds@full >$BACKDIR/full" + log_must eval "zfs recv $recv_ds <$BACKDIR/full" + done +done + +log_pass "Compressed streams are rejected if incompatible." diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_mixed_compression.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_mixed_compression.ksh new file mode 100755 index 000000000..22e91c011 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_mixed_compression.ksh @@ -0,0 +1,54 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify datasets using mixed compression algorithms can be received. +# +# Strategy: +# 1. Write data with each of the available compression algorithms +# 2. Receive a full compressed send, and verify the data and compression ratios +# + +verify_runnable "both" + +log_assert "Verify datasets using mixed compression algorithms can be received." +log_onexit cleanup_pool $POOL2 + +send_ds=$POOL2/sendfs +recv_ds=$POOL2/recvfs + +log_must zfs create $send_ds + +for prop in "${compress_prop_vals[@]}"; do + log_must zfs set compress=$prop $send_ds + write_compressible $(get_prop mountpoint $send_ds) 16m +done + +log_must zfs set compress=off $send_ds +log_must zfs snapshot $send_ds@full +log_must eval "zfs send -c $send_ds@full >$BACKDIR/full" +log_must eval "zfs recv $recv_ds <$BACKDIR/full" + +verify_stream_size $BACKDIR/full $send_ds +verify_stream_size $BACKDIR/full $recv_ds +log_must cmp_ds_cont $send_ds $recv_ds + +log_pass "Datasets using mixed compression algorithms can be received." diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_props.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_props.ksh new file mode 100755 index 000000000..6e95c2c30 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_props.ksh @@ -0,0 +1,67 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify compressed send streams can still preserve properties +# +# Strategy: +# 1. Randomly modify the properties in the src pool +# 2. Send a full compressed stream with -p to preserve properties +# 3. Verify all the received properties match the source datasets +# 4. Repeat the process with -R instead of -p +# + +verify_runnable "global" + +function cleanup +{ + destroy_pool $POOL + destroy_pool $POOL2 + log_must zpool create $POOL $DISK1 + log_must zpool create $POOL2 $DISK2 + log_must setup_test_model $POOL +} + +log_assert "Compressed send doesn't interfere with preservation of properties" +log_onexit cleanup + +typeset -a datasets=("" "/pclone" "/$FS" "/$FS/fs1" "/$FS/fs1/fs2" + "/$FS/fs1/fclone" "/vol" "/$FS/vol") + +typeset ds +for opt in "-p" "-R"; do + for ds in ${datasets[@]}; do + randomize_ds_props $POOL$ds + done + + log_must eval "zfs send -c $opt $POOL@final > $BACKDIR/pool-final$opt" + log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final$opt" + + for ds in ${datasets[@]}; do + log_must cmp_ds_prop $POOL$ds $POOL2$ds + log_must cmp_ds_prop $POOL$ds@final $POOL2$ds@final + done + + # Don't cleanup the second time, since we do that on exit anyway. + [[ $opt = "-p" ]] && cleanup +done + +log_pass "Compressed send doesn't interfere with preservation of properties" diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_recv_dedup.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_recv_dedup.ksh new file mode 100755 index 000000000..cef81617b --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_recv_dedup.ksh @@ -0,0 +1,55 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2016 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that we can receive a compressed stream into a deduped filesystem. +# +# Strategy: +# 1. Write heavily duplicated data to a filesystem and create a compressed +# full stream. +# 2. Verify that the stream can be received correctly into a dedup=verify +# filesystem. +# + +verify_runnable "both" + +log_pass "Verify a compressed stream can be received into a deduped filesystem" +log_onexit cleanup_pool $POOL2 + +typeset sendfs=$POOL2/sendfs +typeset recvfs=$POOL2/recvfs +typeset stream0=$BACKDIR/stream.0 +typeset stream1=$BACKDIR/stream.1 +typeset inc=$BACKDIR/stream.inc + +log_must zfs create -o compress=lz4 $sendfs +log_must zfs create -o compress=lz4 -o dedup=verify $recvfs +typeset dir=$(get_prop mountpoint $sendfs) +for i in {0..10}; do + log_must file_write -o overwrite -f $dir/file.$i -d R -b 4096 -c 1000 +done +log_must zfs snapshot $sendfs@snap0 +log_must eval "zfs send -c $sendfs@snap0 >$stream0" + +# Finally, make sure the receive works correctly. +log_must eval "zfs recv -d $recvfs <$stream0" +cmp_ds_cont $sendfs $recvfs + +log_pass "The compressed stream could be received into a deduped filesystem" diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh new file mode 100755 index 000000000..53c7ce8bd --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh @@ -0,0 +1,68 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify a pool without the lz4 feature gracefully rejects a compressed stream +# because on any sending pool that supports it, metadata will be compressed +# with lz4 even if user data is not compressed. +# +# Strategy: +# 1. For each of an uncompressed, gzip and lz4 dataset, do the following +# receives into a pool without the lz4 feature: +# 2. Attempt to receive the compressed stream (should fail) +# 3. Attempt to receive the uncompressed stream (should succeed) +# + +verify_runnable "both" + +log_assert "Verify compressed streams are rejected if incompatible." + +typeset compress_types="off gzip lz4" +typeset send_ds=$POOL2/testds +typeset recv_ds=$POOL3/testds + +function cleanup +{ + poolexists $POOL2 && destroy_pool $POOL2 + poolexists $POOL3 && destroy_pool $POOL3 + log_must zpool create $POOL2 $DISK2 +} +log_onexit cleanup + +datasetexists $POOL3 && log_must zpool destroy $POOL3 +log_must zpool create -d $POOL3 $DISK3 + +for compress in $compress_types; do + datasetexists $send_ds && log_must zfs destroy -r $send_ds + datasetexists $recv_ds && log_must zfs destroy -r $recv_ds + + log_must zfs create -o compress=$compress $send_ds + typeset dir=$(get_prop mountpoint $send_ds) + write_compressible $dir 16m + log_must zfs snapshot $send_ds@full + + log_must eval "zfs send -c $send_ds@full >$BACKDIR/full-c" + log_mustnot eval "zfs recv $recv_ds <$BACKDIR/full-c" + + log_must eval "zfs send $send_ds@full >$BACKDIR/full" + log_must eval "zfs recv $recv_ds <$BACKDIR/full" +done + +log_pass "Compressed streams are rejected if incompatible." diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_resume.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_resume.ksh new file mode 100755 index 000000000..9f43b7929 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_resume.ksh @@ -0,0 +1,49 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify resumability of full and incremental ZFS send/receive with the -c +# (compress) flag in the presence of a corrupted stream. +# +# Strategy: +# 1. Start a full ZFS send with the -c flag (compress), redirect output to +# a file +# 2. Mess up the contents of the stream state file on disk +# 3. Try ZFS receive, which should fail with a checksum mismatch error +# 4. ZFS send to the stream state file again using the receive_resume_token +# 5. ZFS receieve and verify the receive completes successfully +# 6. Repeat steps on an incremental ZFS send +# + +verify_runnable "both" + +sendfs=$POOL/sendfs +recvfs=$POOL2/recvfs +streamfs=$POOL/stream + +log_assert "Verify compressed send streams can be resumed if interrupted" +log_onexit resume_cleanup $sendfs $streamfs + +test_fs_setup $sendfs $recvfs +resume_test "zfs send -c -v $sendfs@a" $streamfs $recvfs +resume_test "zfs send -c -v -i @a $sendfs@b" $streamfs $recvfs +file_check $sendfs $recvfs + +log_pass "Compressed send streams can be resumed if interrupted" diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh new file mode 100755 index 000000000..e810f712a --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh @@ -0,0 +1,91 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify the stream size estimate given by -P accounts for compressed send. +# Verify the stream size given by -P accounts for compressed send." +# +# Strategy: +# 1. For datasets of varied compression types do the following: +# 2. Write data, verify stream size estimates with and without -c +# + +verify_runnable "both" +typeset compress_types="off gzip lz4" +typeset send_ds="$POOL2/testfs" +typeset send_vol="$POOL2/vol" +typeset send_voldev="$ZVOL_DEVDIR/$POOL2/vol" +typeset file="$BACKDIR/file.0" +typeset megs="16" +typeset compress + +function get_estimated_size +{ + typeset cmd=$1 + typeset ds=${cmd##* } + typeset tmpfile=$(mktemp -p $BACKDIR) + + eval "$cmd >$tmpfile" + [[ $? -eq 0 ]] || log_fail "get_estimated_size: $cmd" + typeset size=$(eval "awk '\$2 == \"$ds\" {print \$3}' $tmpfile") + rm -f $tmpfile + + echo $size +} + +log_assert "Verify the stream size given by -P accounts for compressed send." +log_onexit cleanup_pool $POOL2 + +write_compressible $BACKDIR ${megs}m + +for compress in $compress_types; do + datasetexists $send_ds && log_must zfs destroy -r $send_ds + datasetexists $send_vol && log_must zfs destroy -r $send_vol + log_must zfs create -o compress=$compress $send_ds + log_must zfs create -V 1g -o compress=$compress $send_vol + + typeset dir=$(get_prop mountpoint $send_ds) + log_must cp $file $dir + log_must zfs snapshot $send_ds@snap + log_must dd if=$file of=$send_voldev + log_must zfs snapshot $send_vol@snap + + typeset ds_size=$(get_estimated_size "zfs send -nP $send_ds@snap") + typeset ds_lrefer=$(get_prop lrefer $send_ds) + within_percent $ds_size $ds_lrefer 90 || log_fail \ + "$ds_size and $ds_lrefer differed by too much" + + typeset vol_size=$(get_estimated_size "zfs send -nP $send_vol@snap") + typeset vol_lrefer=$(get_prop lrefer $send_vol) + within_percent $vol_size $vol_lrefer 90 || log_fail \ + "$vol_size and $vol_lrefer differed by too much" + + typeset ds_csize=$(get_estimated_size "zfs send -nP -c $send_ds@snap") + typeset ds_refer=$(get_prop refer $send_ds) + within_percent $ds_csize $ds_refer 90 || log_fail \ + "$ds_csize and $ds_refer differed by too much" + + typeset vol_csize=$(get_estimated_size "zfs send -nP -c $send_vol@snap") + typeset vol_refer=$(get_prop refer $send_vol) + within_percent $vol_csize $vol_refer 90 || log_fail \ + "$vol_csize and $vol_refer differed by too much" +done + +log_pass "The the stream size given by -P accounts for compressed send." diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_verify_contents.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_verify_contents.ksh new file mode 100755 index 000000000..0a5d56beb --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_verify_contents.ksh @@ -0,0 +1,55 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify compressed send streams replicate data and datasets +# +# Strategy: +# 1. Back up all the data from POOL/FS +# 2. Verify all the datasets and data can be recovered in POOL2 +# 3. Back up all the data from root filesystem POOL2 +# 4. Verify all the data can be recovered, too +# + +verify_runnable "both" + +log_assert "zfs send -c -R send replication stream up to the named snap." +log_onexit cleanup_pool $POOL2 + +# Verify the entire pool and descendants can be backed up and restored. +log_must eval "zfs send -c -R $POOL@final > $BACKDIR/pool-final-R" +log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-final-R" + +dstds=$(get_dst_ds $POOL $POOL2) +log_must cmp_ds_subs $POOL $dstds +log_must cmp_ds_cont $POOL $dstds + +# Cleanup POOL2 +log_must cleanup_pool $POOL2 + +# Verify all the filesystems and descendants can be backed up and restored. +log_must eval "zfs send -c -R $POOL/$FS@final > $BACKDIR/fs-final-R" +log_must eval "zfs receive -d $POOL2 < $BACKDIR/fs-final-R" + +dstds=$(get_dst_ds $POOL/$FS $POOL2) +log_must cmp_ds_subs $POOL/$FS $dstds +log_must cmp_ds_cont $POOL/$FS $dstds + +log_pass "zfs send -c -R send replication stream up to the named snap." diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh new file mode 100755 index 000000000..17bf1f7c0 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh @@ -0,0 +1,66 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/properties.shlib + +# +# Description: +# Verify that the amount of data in a send -c stream matches compressratio. +# +# Strategy: +# 1. For random compression types, and compressible / incompressible data: +# 2. Create a snap with data +# 3. Compare the size of the stream with the data on the dataset, adjusted +# by compressratio for normal send, and compared to used for send -c. +# + +verify_runnable "both" + +log_assert "Verify send -c streams are compressed" +log_onexit cleanup_pool $POOL2 + +typeset sendfs=$POOL2/$FS +typeset megs=128 + +for prop in $(get_rand_compress_any 6); do + for compressible in 'yes' 'no'; do + log_must zfs create -o compress=$prop $sendfs + + if [[ $compressible = 'yes' ]]; then + write_compressible $(get_prop mountpoint $sendfs) \ + ${megs}m + else + typeset file="$(get_prop mountpoint $sendfs)/ddfile" + log_must dd if=/dev/urandom of=$file bs=1024k count=$megs + fi + + log_must zfs snapshot $sendfs@snap + + # Calculate the sizes and verify the compression ratio. + log_must eval "zfs send $sendfs@snap >$BACKDIR/uncompressed" + verify_stream_size $BACKDIR/uncompressed $sendfs + + log_must eval "zfs send -c $sendfs@snap >$BACKDIR/compressed" + verify_stream_size $BACKDIR/compressed $sendfs + + log_must rm $BACKDIR/uncompressed $BACKDIR/compressed + log_must zfs destroy -r $sendfs + done +done + +log_pass "Verify send -c streams are compressed" diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh new file mode 100755 index 000000000..99aee8699 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh @@ -0,0 +1,80 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify that compressed send correctly handles volumes +# +# Strategy: +# 1. Write compressible data into a volume, take a snap +# 2. Verify the compressed stream is the correct size, and has the correct data +# 3. Repeat step 2 for an incremental compressed stream +# + +function cleanup +{ + log_must zfs destroy -r $vol + cleanup_pool $POOL2 +} + +verify_runnable "both" + +log_assert "Verify compressed send works with volumes" +log_onexit cleanup + +typeset vol="$POOL/newvol" +typeset vol2="$POOL2/newvol" +typeset voldev="$ZVOL_DEVDIR/$POOL/newvol" +typeset voldev2="$ZVOL_DEVDIR/$POOL2/newvol" +typeset data1=$BACKDIR/file.0 +typeset data2=$BACKDIR/file.1 +typeset megs=8 + +log_must zfs create -V 256m -o compress=lz4 $vol + +write_compressible $BACKDIR ${megs}m 2 +md5_1=$(md5sum $data1 | awk '{print $1}') +md5_2=$(md5sum $data2 | awk '{print $1}') + +log_must dd if=$data1 of=$voldev bs=1024k +log_must zfs snapshot $vol@snap + +log_must eval "zfs send -c $vol@snap >$BACKDIR/full" +log_must eval "zfs recv -d $POOL2 <$BACKDIR/full" + +verify_stream_size $BACKDIR/full $vol +verify_stream_size $BACKDIR/full $vol2 +md5=$(dd if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5sum | \ + awk '{print $1}') +[[ $md5 = $md5_1 ]] || log_fail "md5 mismatch: $md5 != $md5_1" + +# Repeat, for an incremental send +log_must dd seek=$megs if=$data2 of=$voldev bs=1024k +log_must zfs snapshot $vol@snap2 + +log_must eval "zfs send -c -i snap $vol@snap2 >$BACKDIR/inc" +log_must eval "zfs recv -d $POOL2 <$BACKDIR/inc" + +verify_stream_size $BACKDIR/inc $vol 90 $vol@snap +verify_stream_size $BACKDIR/inc $vol2 90 $vol2@snap +md5=$(dd skip=$megs if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5sum | \ + awk '{print $1}') +[[ $md5 = $md5_2 ]] || log_fail "md5 mismatch: $md5 != $md5_2" + +log_pass "Verify compressed send works with volumes" diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh new file mode 100755 index 000000000..158223ccf --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh @@ -0,0 +1,59 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib +. $STF_SUITE/include/math.shlib + +# +# Description: +# Verify compression features show up in zstreamdump +# +# Strategy: +# 1. Create a full compressed send stream +# 2. Verify zstreamdump shows this stream has the relevant features +# 3. Verify zstreamdump's accounting of logical and compressed size is correct +# + +verify_runnable "both" + +log_assert "Verify zstreamdump correctly interprets compressed send streams." +log_onexit cleanup_pool $POOL2 + +typeset sendfs=$POOL2/fs + +log_must zfs create -o compress=lz4 $sendfs +typeset dir=$(get_prop mountpoint $sendfs) +write_compressible $dir 16m +log_must zfs snapshot $sendfs@full + +log_must eval "zfs send -c $sendfs@full >$BACKDIR/full" +log_must stream_has_features $BACKDIR/full lz4 compressed +cat $BACKDIR/full | zstreamdump -v | parse_dump > $BACKDIR/dump.out + +lsize=$(awk '/^WRITE [^0]/ {lsize += $4} END {printf("%d", lsize)}' \ + $BACKDIR/dump.out) +lsize_prop=$(get_prop logicalused $sendfs) +within_percent $lsize $lsize_prop 90 || log_fail \ + "$lsize and $lsize_prop differed by too much" + +csize=$(awk '/^WRITE [^0]/ {csize += $5} END {printf("%d", csize)}' \ + $BACKDIR/dump.out) +csize_prop=$(get_prop used $sendfs) +within_percent $csize $csize_prop 90 || log_fail \ + "$csize and $csize_prop differed by too much" + +log_pass "zstreamdump correctly interprets compressed send streams." diff --git a/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh b/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh new file mode 100755 index 000000000..ee7421d08 --- /dev/null +++ b/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh @@ -0,0 +1,199 @@ +#!/bin/ksh -p + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2015 by Delphix. All rights reserved. +# + +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# +# Description: +# Verify compressed send works correctly with datasets of varying recsize. +# +# Strategy: +# 1. Check the recv behavior (into pools with features enabled and disabled) +# of all combinations of -c -p and -L. Verify the stream is compressed, +# and that the recsize property and that of a received file is correct +# according to this matrix: +# +# +---------+--------+------------+------------+-----------+-----------+ +# | send | send | received | received | received | received | +# | stream | stream | file bs | prop | file bs | props | +# | recsize | flags | (disabled) | (disabled) | (enabled) | (enabled) | +# +---------+--------+------------+------------+-----------+-----------+ +# | 128k | | 128k | 128k | 128k | 128k | +# | 128k | -c | Fails | Fails | 128k | 128k | +# | 128k | -p | 128k | 128k | 128k | 128k | +# | 128k | -L | 128k | 128k | 128k | 128k | +# | 128k | -cp | Fails | Fails | 128k | 128k | +# | 128k | -cL | Fails | Fails | 128k | 128k | +# | 128k | -pL | 128k | 128k | 128k | 128k | +# | 128k | -cpL | Fails | Fails | 128k | 128k | +# | 1m | | Fails | Fails | 128k | 128k | +# | 1m | -c | Fails | Fails | 128k | 128k | +# | 1m | -p | 128k | 128k | 128k | 1m | +# | 1m | -L | Fails | Fails | 1m | 128k | +# | 1m | -cp | Fails | Fails | 128k | 1m | +# | 1m | -cL | Fails | Fails | 1m | 128k | +# | 1m | -pL | Fails | Fails | 1m | 1m | +# | 1m | -cpL | Fails | Fails | 1m | 1m | +# +---------+--------+------------+------------+-----------+-----------+ +# + +verify_runnable "both" + +function cleanup +{ + datasetexists $TESTPOOL/128k && log_must zfs destroy $TESTPOOL/128k + datasetexists $TESTPOOL/1m && log_must zfs destroy $TESTPOOL/1m + cleanup_pool $POOL2 + destroy_pool $POOL3 +} + +# For a received stream, verify the recsize (prop and file) match expectations. +function check_recsize +{ + typeset recv_ds=$1 + typeset expected_file_bs=$2 + typeset expected_recsize=$3 + typeset file="$(get_prop mountpoint $recv_ds)/testfile" + + [[ -f $file ]] || log_fail "file '$file' doesn't exist" + + typeset read_recsize=$(get_prop recsize $recv_ds) + typeset read_file_bs=$(stat $file | sed -n \ + 's/.*IO Block: \([0-9]*\).*/\1/p') + + [[ $read_recsize = $expected_recsize ]] || log_fail \ + "read_recsize: $read_recsize expected_recsize: $expected_recsize" + [[ $read_file_bs = $expected_file_bs ]] || log_fail \ + "read_file_bs: $read_file_bs expected_file_bs: $expected_file_bs" +} + +# +# This function does a zfs send and receive according to the parameters +# below, and verifies the data shown in the strategy section. +# +# -[cpL] flags to pass through to 'zfs send' +# -d Receive into a pool with all features disabled +# +# $1 The recordsize of the send dataset +# $2 Whether or not the recv should work. +# $3 The blocksize expected in a received file (default 128k) +# $4 The recordsize property expected in a received dataset (default 128k) +# +function check +{ + typeset recv_pool=$POOL2 + typeset flags='-' + + while getopts "cdpL" opt; do + case $opt in + c) + flags+='c' + ;; + d) + recv_pool=$POOL3 + ;; + p) + flags+='p' + ;; + L) + flags+='L' + ;; + esac + done + shift $(($OPTIND - 1)) + [[ ${#flags} -eq 1 ]] && flags='' + + typeset recsize=$1 + typeset verify=$2 + typeset expected_file_bs=${3-131072} + typeset expected_recsize=${4-131072} + typeset send_ds=$TESTPOOL/$recsize + typeset send_snap=$send_ds@snap + typeset recv_ds=$recv_pool/$recsize + typeset stream=$BACKDIR/stream.out + + datasetexists $send_ds || log_fail "send ds: $send_ds doesn't exist" + [[ -f $stream ]] && log_must rm $stream + log_must eval "zfs send $flags $send_snap >$stream" + $verify eval "zfs recv $recv_ds <$stream" + typeset stream_size=$(cat $stream | zstreamdump | sed -n \ + 's/ Total write size = \(.*\) (0x.*)/\1/p') + + # + # Special case: For a send dataset with large blocks, don't try to + # verify the stream size is correct if the compress flag is present + # but the large blocks flag isn't. In these cases, the user data + # isn't compressed in the stream (though metadata is) so the + # verification would fail. + # + typeset do_size_test=true + [[ $recsize = $large && $flags =~ 'c' && ! $flags =~ 'L' ]] && \ + do_size_test=false + + $do_size_test && verify_stream_size $stream $send_ds + + if [[ $verify = "log_mustnot" ]]; then + datasetnonexists $recv_ds || log_fail "$recv_ds shouldn't exist" + return + fi + + check_recsize $recv_ds $expected_file_bs $expected_recsize + $do_size_test && verify_stream_size $stream $recv_ds + log_must zfs destroy -r $recv_ds +} + +log_assert "Verify compressed send works with datasets of varying recsize." +log_onexit cleanup +typeset recsize opts dir +typeset small=$((128 * 1024)) +typeset large=$((1024 * 1024)) + +# Create POOL3 with features disabled and datasets to create test send streams +datasetexists $POOL3 && log_must zpool destroy $POOL3 +log_must zpool create -d $POOL3 $DISK3 +write_compressible $BACKDIR 32m +for recsize in $small $large; do + log_must zfs create -o compress=gzip -o recsize=$recsize \ + $TESTPOOL/$recsize + dir=$(get_prop mountpoint $TESTPOOL/$recsize) + log_must cp $BACKDIR/file.0 $dir/testfile + log_must zfs snapshot $TESTPOOL/$recsize@snap +done + +# Run tests for send streams without large blocks +for opts in '' -d -c -p -dp -L -dL -cp -cL -pL -dpL -cpL; do + check $opts $small log_must +done +for opts in -dc -dcp -dcL -dcpL; do + check $opts $small log_mustnot +done + +# Run tests for send streams with large blocks +for opts in '' -d -dp -c; do + check $opts $large log_must +done +for opts in -dc -dL -dcp -dcL -dpL -dcpL; do + check $opts $large log_mustnot +done +check -p $large log_must $small $large +check -L $large log_must $large $small +check -cp $large log_must $small $large +check -cL $large log_must $large $small +check -pL $large log_must $large $large +check -cpL $large log_must $large $large + +log_pass "Compressed send works with datasets of varying recsize." diff --git a/tests/zfs-tests/tests/perf/fio/mkfiles.fio b/tests/zfs-tests/tests/perf/fio/mkfiles.fio index f876bd63d..8289d546d 100644 --- a/tests/zfs-tests/tests/perf/fio/mkfiles.fio +++ b/tests/zfs-tests/tests/perf/fio/mkfiles.fio @@ -10,7 +10,7 @@ # # -# Copyright (c) 2015 by Delphix. All rights reserved. +# Copyright (c) 2016 by Delphix. All rights reserved. # [global] @@ -24,7 +24,7 @@ thread=1 directory=/${TESTFS} numjobs=${NUMJOBS} filesize=${FILE_SIZE} -buffer_compress_percentage=33 +buffer_compress_percentage=66 buffer_compress_chunk=4096 [job] diff --git a/tests/zfs-tests/tests/perf/fio/random_readwrite.fio b/tests/zfs-tests/tests/perf/fio/random_readwrite.fio index 0b750260f..07090d4dc 100644 --- a/tests/zfs-tests/tests/perf/fio/random_readwrite.fio +++ b/tests/zfs-tests/tests/perf/fio/random_readwrite.fio @@ -10,7 +10,7 @@ # # -# Copyright (c) 2015 by Delphix. All rights reserved. +# Copyright (c) 2015, 2016 by Delphix. All rights reserved. # [global] @@ -29,7 +29,7 @@ bssplit=4k/50:8k/30:128k/10:1m/10 ioengine=psync sync=${SYNC_TYPE} numjobs=${NUMJOBS} -buffer_compress_percentage=33 +buffer_compress_percentage=66 buffer_compress_chunk=4096 [job] diff --git a/tests/zfs-tests/tests/perf/fio/random_writes.fio b/tests/zfs-tests/tests/perf/fio/random_writes.fio index b1860a71d..9233a8426 100644 --- a/tests/zfs-tests/tests/perf/fio/random_writes.fio +++ b/tests/zfs-tests/tests/perf/fio/random_writes.fio @@ -10,7 +10,7 @@ # # -# Copyright (c) 2015 by Delphix. All rights reserved. +# Copyright (c) 2015, 2016 by Delphix. All rights reserved. # [global] @@ -27,7 +27,7 @@ ioengine=psync sync=${SYNC_TYPE} numjobs=${NUMJOBS} filesize=${FILESIZE} -buffer_compress_percentage=33 +buffer_compress_percentage=66 buffer_compress_chunk=4096 [job] diff --git a/tests/zfs-tests/tests/perf/fio/sequential_writes.fio b/tests/zfs-tests/tests/perf/fio/sequential_writes.fio index df1590cf1..0ee6d091d 100644 --- a/tests/zfs-tests/tests/perf/fio/sequential_writes.fio +++ b/tests/zfs-tests/tests/perf/fio/sequential_writes.fio @@ -10,7 +10,7 @@ # # -# Copyright (c) 2015 by Delphix. All rights reserved. +# Copyright (c) 2015, 2016 by Delphix. All rights reserved. # [global] @@ -27,7 +27,7 @@ ioengine=psync sync=${SYNC_TYPE} numjobs=${NUMJOBS} filesize=${FILESIZE} -buffer_compress_percentage=33 +buffer_compress_percentage=66 buffer_compress_chunk=4096 [job] |