Remove deduplicated send/receive code

Deduplicated send streams (i.e. `zfs send -D` and `zfs receive` of such
streams) are deprecated.  Deduplicated send streams can be received by
first converting them to non-deduplicated with the `zstream redup`
command.

This commit removes the code for sending and receiving deduplicated send
streams.  `zfs send -D` will now print a warning, ignore the `-D` flag,
and generate a regular (non-deduplicated) send stream.  `zfs receive` of
a deduplicated send stream will print an error message and fail.

The resulting code simplification (especially in the kernel's support
for receiving dedup streams) should help enable future performance
enhancements.

Several new tests are added which leverage `zstream redup`.

Reviewed-by: Paul Dagnelie <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Matthew Ahrens <[email protected]>
Issue #7887
Issue #10117
Issue #10156
Closes #10212 

11 files changed, 136 insertions, 106 deletions

diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run
index b75be778b..6f83dcd8b 100644
--- a/tests/runfiles/common.run
+++ b/tests/runfiles/common.run
@@ -766,22 +766,22 @@ tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
 tags = ['functional', 'rootpool']
 
 [tests/functional/rsend]
-tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
-    'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos',
-    'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos',
-    'rsend_013_pos', 'rsend_014_pos', 'rsend_016_neg',
-    'rsend_019_pos', 'rsend_020_pos',
+tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
+    'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos',
+    'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', 'rsend_009_pos',
+    'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', 'rsend_013_pos',
+    'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos',
     'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos',
     'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props',
     'send-c_incremental', 'send-c_volume', 'send-c_zstreamdump',
     'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
-    'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
+    'send-c_mixed_compression', 'send-c_stream_size_estimate',
     'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
     'send-c_recv_dedup', 'send_encrypted_hierarchy',
     'send_encrypted_props', 'send_encrypted_truncated_files',
     'send_freeobjects', 'send_realloc_files',
     'send_realloc_encrypted_files', 'send_spill_block', 'send_holds',
-    'send_hole_birth', 'send_mixed_raw', 'send-wDR_encrypted_zvol',
+    'send_hole_birth', 'send_mixed_raw', 'send-wR_encrypted_zvol',
     'send_partial_dataset']
 tags = ['functional', 'rsend']
 
diff --git a/tests/zfs-tests/tests/functional/rsend/Makefile.am b/tests/zfs-tests/tests/functional/rsend/Makefile.am
index 8e16f8847..7728a6481 100644
--- a/tests/zfs-tests/tests/functional/rsend/Makefile.am
+++ b/tests/zfs-tests/tests/functional/rsend/Makefile.am
@@ -2,6 +2,8 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/rsend
 dist_pkgdata_SCRIPTS = \
 	setup.ksh \
 	cleanup.ksh \
+	recv_dedup.ksh \
+	recv_dedup_encrypted_zvol.ksh \
 	rsend_001_pos.ksh \
 	rsend_002_pos.ksh \
 	rsend_003_pos.ksh \
@@ -25,7 +27,6 @@ dist_pkgdata_SCRIPTS = \
 	send_encrypted_hierarchy.ksh \
 	send_encrypted_props.ksh \
 	send_encrypted_truncated_files.ksh \
-	send-cD.ksh \
 	send-c_embedded_blocks.ksh \
 	send-c_incremental.ksh \
 	send-c_lz4_disabled.ksh \
@@ -49,8 +50,12 @@ dist_pkgdata_SCRIPTS = \
 	send_holds.ksh \
 	send_hole_birth.ksh \
 	send_mixed_raw.ksh \
-	send-wDR_encrypted_zvol.ksh
+	send-wR_encrypted_zvol.ksh
 
 dist_pkgdata_DATA = \
+	dedup.zsend.bz2 \
+	dedup_encrypted_zvol.bz2 \
+	dedup_encrypted_zvol.zsend.bz2 \
+	fs.tar.gz \
 	rsend.cfg \
 	rsend.kshlib
diff --git a/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 b/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2
new file mode 100644
index 000000000..585e14852
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2
diff --git a/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 b/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2
new file mode 100644
index 000000000..73a5742fc
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2
diff --git a/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 b/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2
new file mode 100644
index 000000000..04a6cb53f
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2
diff --git a/tests/zfs-tests/tests/functional/rsend/fs.tar.gz b/tests/zfs-tests/tests/functional/rsend/fs.tar.gz
new file mode 100644
index 000000000..cb6861c15
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/rsend/fs.tar.gz
diff --git a/tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh b/tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh
new file mode 100755
index 000000000..e6e282a1c
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh
@@ -0,0 +1,53 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# DESCRIPTION:
+# Verifies that we can receive a dedup send stream by processing it with
+# "zstream redup".
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	destroy_dataset $TESTPOOL/recv "-r"
+	rm -r /$TESTPOOL/tar
+	rm $sendfile
+}
+log_onexit cleanup
+
+log_assert "Verify zfs can receive dedup send streams with 'zstream redup'"
+
+typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup.zsend.bz2
+typeset sendfile=/$TESTPOOL/dedup.zsend
+typeset tarfile=$STF_SUITE/tests/functional/rsend/fs.tar.gz
+
+log_must eval "bzcat <$sendfile_compressed >$sendfile"
+log_must zfs create $TESTPOOL/recv
+log_must eval "zstream redup $sendfile | zfs recv -d $TESTPOOL/recv"
+
+log_must mkdir /$TESTPOOL/tar
+log_must tar --directory /$TESTPOOL/tar -xzf $tarfile
+log_must diff -r /$TESTPOOL/tar /$TESTPOOL/recv
+
+log_pass "zfs can receive dedup send streams with 'zstream redup'"
diff --git a/tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh b/tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh
new file mode 100755
index 000000000..569fcd893
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh
@@ -0,0 +1,60 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020 by Delphix. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# DESCRIPTION:
+# Verifies that we can receive a dedup send stream of a zvol by processing it
+# with "zstream redup".
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	destroy_dataset $TESTPOOL/recv "-r"
+	rm $sendfile
+	rm $volfile
+	rm $keyfile
+}
+log_onexit cleanup
+
+log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams"
+
+typeset keyfile=/$TESTPOOL/pkey
+typeset recvdev=$ZVOL_DEVDIR/$TESTPOOL/recv
+typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2
+typeset sendfile=/$TESTPOOL/dedup_encrypted_zvol.zsend
+typeset volfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.bz2
+typeset volfile=/$TESTPOOL/dedup_encrypted_zvol
+
+log_must eval "echo 'password' > $keyfile"
+
+log_must eval "bzcat <$sendfile_compressed >$sendfile"
+log_must eval "zstream redup $sendfile | zfs recv $TESTPOOL/recv"
+
+log_must zfs load-key $TESTPOOL/recv
+block_device_wait
+
+log_must eval "bzcat <$volfile_compressed >$volfile"
+log_must diff $volfile $recvdev
+
+log_pass "zfs can receive raw, recursive, and deduplicated send streams"
diff --git a/tests/zfs-tests/tests/functional/rsend/send-cD.ksh b/tests/zfs-tests/tests/functional/rsend/send-cD.ksh
deleted file mode 100755
index fcbec2d9e..000000000
--- a/tests/zfs-tests/tests/functional/rsend/send-cD.ksh
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/bin/ksh -p
-
-#
-# This file and its contents are supplied under the terms of the
-# Common Development and Distribution License ("CDDL"), version 1.0.
-# You may only use this file in accordance with the terms of version
-# 1.0 of the CDDL.
-#
-# A full copy of the text of the CDDL should have accompanied this
-# source.  A copy of the CDDL is also available via the Internet at
-# http://www.illumos.org/license/CDDL.
-#
-
-#
-# Copyright (c) 2015, 2018 by Delphix. All rights reserved.
-#
-
-. $STF_SUITE/tests/functional/rsend/rsend.kshlib
-
-#
-# Description:
-# Verify that the -c and -D flags do not interfere with each other.
-#
-# Strategy:
-# 1. Write unique data to a filesystem and create a compressed, deduplicated
-#    full stream.
-# 2. Verify that the stream and send dataset show the same size
-# 3. Make several copies of the original data, and create both full and
-#    incremental compressed, deduplicated send streams
-# 4. Verify the full stream is no bigger than the stream from step 1
-# 5. Verify the streams can be received correctly.
-#
-
-verify_runnable "both"
-
-log_assert "Verify that the -c and -D flags do not interfere with each other"
-log_onexit cleanup_pool $POOL2
-
-typeset sendfs=$POOL2/sendfs
-typeset recvfs=$POOL2/recvfs
-typeset stream0=$BACKDIR/stream.0
-typeset stream1=$BACKDIR/stream.1
-typeset inc=$BACKDIR/stream.inc
-
-log_must zfs create -o compress=lz4 $sendfs
-log_must zfs create -o compress=lz4 $recvfs
-typeset dir=$(get_prop mountpoint $sendfs)
-# Don't use write_compressible: we want compressible but undeduplicable data.
-log_must eval "dd if=/dev/urandom bs=1024k count=4 | base64 >$dir/file"
-log_must zfs snapshot $sendfs@snap0
-log_must eval "zfs send -D -c $sendfs@snap0 >$stream0"
-
-# The stream size should match at this point because the data is all unique
-verify_stream_size $stream0 $sendfs
-
-for i in {0..3}; do
-	log_must cp $dir/file $dir/file.$i
-done
-log_must zfs snapshot $sendfs@snap1
-
-# The stream sizes should match, since the second stream contains no new blocks
-log_must eval "zfs send -D -c $sendfs@snap1 >$stream1"
-typeset size0=$(stat_size $stream0)
-typeset size1=$(stat_size $stream1)
-within_percent $size0 $size1 90 || log_fail "$size0 and $size1"
-
-# make sure the receive works correctly.
-log_must eval "zfs send -D -c -i snap0 $sendfs@snap1 >$inc"
-log_must eval "zfs recv -d $recvfs <$stream0"
-log_must eval "zfs recv -d $recvfs <$inc"
-cmp_ds_cont $sendfs $recvfs
-
-# check receive with redup.
-log_must zfs destroy -r $recvfs
-log_must zfs create -o compress=lz4 $recvfs
-log_must eval "zstream redup $stream0 | zfs recv -d $recvfs"
-log_must eval "zstream redup $inc | zfs recv -d $recvfs"
-cmp_ds_cont $sendfs $recvfs
-
-# The size of the incremental should be the same as the initial send.
-typeset size2=$(stat_size $inc)
-within_percent $size0 $size2 90 || log_fail "$size0 and $size1"
-
-# The redup'ed size should be 4x
-typeset size3=$(zstream redup $inc | wc -c)
-let size4=size0*4
-within_percent $size4 $size3 90 || log_fail "$size4 and $size3"
-
-log_pass "The -c and -D flags do not interfere with each other"
diff --git a/tests/zfs-tests/tests/functional/rsend/send-wDR_encrypted_zvol.ksh b/tests/zfs-tests/tests/functional/rsend/send-wR_encrypted_zvol.ksh
index f9cfeee21..b95fc3da3 100755
--- a/tests/zfs-tests/tests/functional/rsend/send-wDR_encrypted_zvol.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-wR_encrypted_zvol.ksh
@@ -16,20 +16,21 @@
 
 #
 # Copyright (c) 2018 by Datto Inc. All rights reserved.
+# Copyright (c) 2020 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/tests/functional/rsend/rsend.kshlib
 
 #
 # DESCRIPTION:
-# Verify that zvols with dedup=on and encryption=on can be sent and received
-# with a deduplicated raw send stream.
+# Verify that zvols with encryption=on can be sent and received with a raw
+# send stream.
 #
 # STRATEGY:
-# 1. Create a zvol with dedup and encryption on and put a filesystem on it
+# 1. Create a zvol with encryption on and put a filesystem on it
 # 2. Copy a file into the zvol a few times and take a snapshot
 # 3. Repeat step 2 a few times to create more snapshots
-# 4. Send all snapshots in a recursive, raw, deduplicated send stream
+# 4. Send all snapshots in a recursive, raw send stream
 # 5. Mount the received zvol and verify that all of the data there is correct
 #
 
@@ -48,7 +49,7 @@ function cleanup
 }
 log_onexit cleanup
 
-log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams"
+log_assert "Verify zfs can receive raw, recursive send streams"
 
 typeset keyfile=/$TESTPOOL/pkey
 typeset snap_count=5
@@ -93,7 +94,7 @@ for ((i = 1; i <= $snap_count; i++)); do
 	log_must mount $remount_rw $zdev $mntpnt
 done
 
-log_must eval "zfs send -wDR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile"
+log_must eval "zfs send -wR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile"
 log_must eval "zfs recv $TESTPOOL/recv < $sendfile"
 log_must zfs load-key $TESTPOOL/recv
 block_device_wait
@@ -104,4 +105,4 @@ md5_1=$(cat $mntpnt/* | md5digest)
 md5_2=$(cat $recvmnt/* | md5digest)
 [[ "$md5_1" == "$md5_2" ]] || log_fail "md5 mismatch: $md5_1 != $md5_2"
 
-log_pass "zfs can receive raw, recursive, and deduplicated send streams"
+log_pass "zfs can receive raw, recursive send streams"
diff --git a/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh b/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh
index 8a0971596..d5eb9a0ed 100755
--- a/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh
@@ -13,6 +13,7 @@
 
 #
 # Copyright (c) 2019 Datto Inc.
+# Copyright (c) 2020 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -98,7 +99,6 @@ set -A badargs \
 	"-R $POOL/recvfs" \
 	"-p $POOL/recvfs" \
 	"-I $POOL/recvfs" \
-	"-D $POOL/recvfs" \
 	"-h $POOL/recvfs"
 
 while (( i < ${#badargs[*]} ))