Receive checks should allow unencrypted child datasets

dmu_recv_begin_check() unconditionally sets the DS_HOLD_FLAG_DECRYPT
flag before calling dsl_dataset_hold_flags(). If the key on the
receiving side isn't loaded or the send stream contains embedded
blocks, the receive check fails for a stream which is perfectly
valid and could be received without any problem. This seems like
a remnant of the initial design, where unencrypted datasets below
encrypted ones weren't allowed.

Add a condition to set `DS_HOLD_FLAG_DECRYPT` only for encrypted
datasets, modify an existing test to detect this regression and add
a test for raw replication streams.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: George Amanakis <[email protected]>
Co-authored-by: George Amanakis <[email protected]>
Signed-off-by: Attila Fülöp <[email protected]>
Closes #13033 
Closes #13076 

4 files changed, 93 insertions, 6 deletions

diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run
index 18995511e..bad2df2f9 100644
--- a/tests/runfiles/common.run
+++ b/tests/runfiles/common.run
@@ -234,7 +234,8 @@ tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos',
     'zfs_receive_016_pos', 'receive-o-x_props_override',
     'zfs_receive_from_encrypted', 'zfs_receive_to_encrypted',
     'zfs_receive_raw', 'zfs_receive_raw_incremental', 'zfs_receive_-e',
-    'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props']
+    'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props',
+    'zfs_receive_-wR-encrypted-mix']
 tags = ['functional', 'cli_root', 'zfs_receive']
 
 [tests/functional/cli_root/zfs_rename]
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am
index 2b281e71c..773e9f5a5 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile.am
@@ -26,7 +26,8 @@ dist_pkgdata_SCRIPTS = \
 	zfs_receive_raw.ksh \
 	zfs_receive_raw_incremental.ksh \
 	zfs_receive_raw_-d.ksh \
-	zfs_receive_-e.ksh
+	zfs_receive_-e.ksh \
+	zfs_receive_-wR-encrypted-mix.ksh
 
 dist_pkgdata_DATA = \
 	zstd_test_data.txt
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh
new file mode 100755
index 000000000..6e27130e0
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh
@@ -0,0 +1,75 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2022 by Attila Fülöp <[email protected]>
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+#      ZFS should receive a raw send of a mix of unencrypted and encrypted
+#      child datasets
+#
+#      The layout of the datasets is:  enc/unenc/enc/unenc
+#
+# STRATEGY:
+# 1. Create the dataset hierarchy
+# 2. Snapshot the dataset hierarchy
+# 3. Send -Rw the dataset hierarchy and receive into a top-level dataset
+# 4. Check the encryption property of the received datasets
+
+verify_runnable "both"
+
+function cleanup
+{
+	datasetexists "$TESTPOOL/$TESTFS1" && \
+		destroy_dataset "$TESTPOOL/$TESTFS1" -r
+
+	datasetexists "$TESTPOOL/$TESTFS2" && \
+		destroy_dataset "$TESTPOOL/$TESTFS2" -r
+}
+
+log_onexit cleanup
+
+log_assert "ZFS should receive a mix of un/encrypted childs"
+
+typeset src="$TESTPOOL/$TESTFS1"
+typeset dst="$TESTPOOL/$TESTFS2"
+typeset snap="snap"
+
+echo "password" | \
+	create_dataset "$src" -o encryption=on -o keyformat=passphrase
+create_dataset "$src/u" "-o encryption=off"
+echo "password" | \
+	create_dataset "$src/u/e" -o encryption=on -o keyformat=passphrase
+create_dataset "$src/u/e/u" -o encryption=off
+
+log_must zfs snapshot -r "$src@$snap"
+log_must eval "zfs send -Rw $src@$snap | zfs receive -u $dst"
+log_must test "$(get_prop 'encryption' $dst)" != "off"
+log_must test "$(get_prop 'encryption' $dst/u)" == "off"
+log_must test "$(get_prop 'encryption' $dst/u/e)" != "off"
+log_must test "$(get_prop 'encryption' $dst/u/e/u)" == "off"
+
+log_pass "ZFS can receive a mix of un/encrypted childs"
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh
index 526497401..5d76c220f 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh
@@ -59,19 +59,29 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
 
 log_note "Verifying ZFS will receive to an encrypted child"
 log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c1"
+log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c1)" != "off"
 
-log_note "Verifying 'send -p' will receive to an encrypted child"
+# Unload the key, the following tests won't require it and we will test
+# the receive checks as well.
+log_must zfs unmount $TESTPOOL/$TESTFS1
+log_must zfs unload-key $TESTPOOL/$TESTFS1
+
+log_note "Verifying 'send -p' will receive to an unencrypted child"
 log_must eval "zfs send -p $snap | zfs receive $TESTPOOL/$TESTFS1/c2"
 log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c2)" == "off"
 
-log_note "Verifying 'send -R' will receive to an encrypted child"
+# For completeness add the property override case.
+log_note "Verifying recv -o encyption=off' will receive to an unencrypted child"
+log_must eval "zfs send $snap | \
+	zfs receive -o encryption=off $TESTPOOL/$TESTFS1/c2o"
+log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c2o)" == "off"
+
+log_note "Verifying 'send -R' will receive to an unencrypted child"
 log_must eval "zfs send -R $snap | zfs receive $TESTPOOL/$TESTFS1/c3"
 log_must test "$(get_prop 'encryption' $TESTPOOL/$TESTFS1/c3)" == "off"
 
 log_note "Verifying ZFS will not receive to an encrypted child when the" \
 	"parent key is unloaded"
-log_must zfs unmount $TESTPOOL/$TESTFS1
-log_must zfs unload-key $TESTPOOL/$TESTFS1
 log_mustnot eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c4"
 
 log_pass "ZFS can receive encrypted filesystems into child dataset"