libzfs: add keylocation=https://, backed by fetch(3) or libcurl

Add support for http and https to the keylocation properly to
allow encryption keys to be fetched from the specified URL.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Ryan Moeller <[email protected]>
Signed-off-by: Ahelenia Ziemiańska <[email protected]>
Issue #9543
Closes #9947 
Closes #11956 

10 files changed, 207 insertions, 14 deletions

diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am
index 03c291638..7dfec435c 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile.am
@@ -5,6 +5,7 @@ dist_pkgdata_SCRIPTS = \
 	zfs_load-key.ksh \
 	zfs_load-key_all.ksh \
 	zfs_load-key_file.ksh \
+	zfs_load-key_https.ksh \
 	zfs_load-key_location.ksh \
 	zfs_load-key_noop.ksh \
 	zfs_load-key_recursive.ksh
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/cleanup.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/cleanup.ksh
index 79cd6e9f9..d397bcf4e 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/cleanup.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/cleanup.ksh
@@ -26,5 +26,7 @@
 #
 
 . $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
 
+cleanup_https
 default_cleanup
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/setup.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/setup.ksh
index 6a9af3bc2..6cc5528ce 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/setup.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/setup.ksh
@@ -26,7 +26,10 @@
 #
 
 . $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
 
 DISK=${DISKS%% *}
 
-default_setup $DISK
+default_setup_noexit $DISK
+setup_https
+log_pass
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg
index 2f01aac7c..cc1e3b330 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key.cfg
@@ -27,3 +27,31 @@ export HEXKEY="000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F"
 export HEXKEY1="201F1E1D1C1B1A191817161514131211100F0E0D0C0B0A090807060504030201"
 export RAWKEY="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 export RAWKEY1="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+
+export SSL_CA_CERT_FILE="/$TESTPOOL/snakeoil.crt"
+export HTTPS_PORT_FILE="/$TESTPOOL/snakeoil.port"
+export HTTPS_HOSTNAME="localhost"
+export HTTPS_PORT=
+export HTTPS_BASE_URL=
+
+function get_https_port
+{
+	if [ -z "$HTTPS_PORT" ]; then
+		read -r HTTPS_PORT < "$HTTPS_PORT_FILE" || return
+	fi
+
+	echo "$HTTPS_PORT"
+}
+
+function get_https_base_url
+{
+	if [ -z "$HTTPS_BASE_URL" ]; then
+		HTTPS_BASE_URL="https://$HTTPS_HOSTNAME:$(get_https_port)" || {
+			typeset ret=$?
+			HTTPS_BASE_URL=
+			return $ret
+		}
+	fi
+
+	echo "$HTTPS_BASE_URL"
+}
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_all.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_all.ksh
index 5e331fd12..96710bf9a 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_all.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_all.ksh
@@ -39,6 +39,8 @@ function cleanup
 {
 	datasetexists $TESTPOOL/$TESTFS1 && \
 		log_must zfs destroy $TESTPOOL/$TESTFS1
+	datasetexists $TESTPOOL/$TESTFS2 && \
+		log_must zfs destroy $TESTPOOL/$TESTFS2
 	datasetexists $TESTPOOL/zvol && log_must zfs destroy $TESTPOOL/zvol
 	poolexists $TESTPOOL1 && log_must destroy_pool $TESTPOOL1
 }
@@ -50,6 +52,9 @@ log_must eval "echo $PASSPHRASE1 > /$TESTPOOL/pkey"
 log_must zfs create -o encryption=on -o keyformat=passphrase \
 	-o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1
 
+log_must zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS2
+
 log_must zfs create -V 64M -o encryption=on -o keyformat=passphrase \
 	-o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/zvol
 
@@ -60,6 +65,9 @@ log_must zpool create -O encryption=on -O keyformat=passphrase \
 log_must zfs unmount $TESTPOOL/$TESTFS1
 log_must zfs unload-key $TESTPOOL/$TESTFS1
 
+log_must zfs unmount $TESTPOOL/$TESTFS2
+log_must zfs unload-key $TESTPOOL/$TESTFS2
+
 log_must zfs unload-key $TESTPOOL/zvol
 
 log_must zfs unmount $TESTPOOL1
@@ -70,8 +78,10 @@ log_must zfs load-key -a
 log_must key_available $TESTPOOL1
 log_must key_available $TESTPOOL/zvol
 log_must key_available $TESTPOOL/$TESTFS1
+log_must key_available $TESTPOOL/$TESTFS2
 
 log_must zfs mount $TESTPOOL1
 log_must zfs mount $TESTPOOL/$TESTFS1
+log_must zfs mount $TESTPOOL/$TESTFS2
 
 log_pass "'zfs load-key -a' loads keys for all datasets"
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
index d9066f9cb..f7461437c 100644
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
@@ -99,3 +99,66 @@ function verify_origin
 
 	return 0
 }
+
+function setup_https
+{
+	log_must openssl req -x509 -newkey rsa:4096 -sha256 -days 1 -nodes -keyout "/$TESTPOOL/snakeoil.key" -out "$SSL_CA_CERT_FILE" -subj "/CN=$HTTPS_HOSTNAME"
+
+	python3 -uc "
+import http.server, ssl, sys, os, time, random
+
+sys.stdin.close()
+
+httpd, err, port = None, None, None
+for i in range(1, 100):
+	port = random.randint(0xC000, 0xFFFF) # ephemeral range
+	try:
+		httpd = http.server.HTTPServer(('$HTTPS_HOSTNAME', port), http.server.SimpleHTTPRequestHandler)
+		break
+	except:
+		err = sys.exc_info()[1]
+		time.sleep(i / 100)
+if not httpd:
+	raise err
+
+with open('$HTTPS_PORT_FILE', 'w') as portf:
+	print(port, file=portf)
+
+httpd.socket = ssl.wrap_socket(httpd.socket, server_side=True, keyfile='/$TESTPOOL/snakeoil.key', certfile='$SSL_CA_CERT_FILE', ssl_version=ssl.PROTOCOL_TLS)
+
+os.chdir('$STF_SUITE/tests/functional/cli_root/zfs_load-key')
+
+with open('/$TESTPOOL/snakeoil.pid', 'w') as pidf:
+	if os.fork() != 0:
+	  os._exit(0)
+	print(os.getpid(), file=pidf)
+
+sys.stdout.close()
+sys.stderr.close()
+try:
+	sys.stdout = sys.stderr = open('/tmp/ZTS-snakeoil.log', 'w', buffering=1) # line
+except:
+	sys.stdout = sys.stderr = open('/dev/null', 'w')
+
+print('{} start on {}'.format(os.getpid(), port))
+httpd.serve_forever()
+" || log_fail
+
+	typeset https_pid=
+	for d in $(seq 0 0.1 5); do
+		read -r https_pid 2>/dev/null < "/$TESTPOOL/snakeoil.pid" && [ -n "$https_pid" ] && break
+		sleep "$d"
+	done
+	[ -z "$https_pid" ] && log_fail "Couldn't start HTTPS server"
+	log_note "Started HTTPS server as $https_pid on port $(get_https_port)"
+}
+
+function cleanup_https
+{
+	typeset https_pid=
+	read -r https_pid 2>/dev/null < "/$TESTPOOL/snakeoil.pid" || return 0
+
+	log_must kill "$https_pid"
+	cat /tmp/ZTS-snakeoil.log
+	rm -f "/$TESTPOOL/snakeoil.pid" "/tmp/ZTS-snakeoil.log"
+}
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_https.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_https.ksh
new file mode 100755
index 000000000..cac9c4140
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_https.ksh
@@ -0,0 +1,78 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
+
+#
+# DESCRIPTION:
+# 'zfs load-key' should load a dataset's key from an https:// URL,
+# but fail to do so if the domain doesn't exist or the file 404s.
+#
+# STRATEGY:
+# 1. Try to create a dataset pointing to an RFC6761-guaranteed unresolvable domain,
+#    one to the sshd port (which will be either unoccupied (ECONNREFUSED)
+#                          or have sshd on it ("wrong version number")).
+#    and one pointing to an URL that will always 404.
+# 2. Create encrypted datasets with keylocation=https://address
+# 3. Unmount the datasets and unload their keys
+# 4. Attempt to load the keys
+# 5. Verify the keys are loaded
+# 6. Attempt to mount the datasets
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	for fs in "$TESTFS1" "$TESTFS2" "$TESTFS3"; do
+		datasetexists $TESTPOOL/$fs && \
+			log_must zfs destroy $TESTPOOL/$fs
+	done
+}
+log_onexit cleanup
+
+log_assert "'zfs load-key' should load a key from a file"
+
+log_mustnot zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=https://invalid./where-ever $TESTPOOL/$TESTFS1
+
+log_mustnot zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=https://$HTTPS_HOSTNAME:22 $TESTPOOL/$TESTFS1
+
+log_mustnot zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=$(get_https_base_url)/ENOENT $TESTPOOL/$TESTFS1
+
+log_must zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1
+
+log_must zfs create -o encryption=on -o keyformat=hex \
+	-o keylocation=$(get_https_base_url)/HEXKEY $TESTPOOL/$TESTFS2
+
+log_must zfs create -o encryption=on -o keyformat=raw \
+	-o keylocation=$(get_https_base_url)/RAWKEY $TESTPOOL/$TESTFS3
+
+for fs in "$TESTFS1" "$TESTFS2" "$TESTFS3"; do
+	log_must zfs unmount $TESTPOOL/$fs
+	log_must zfs unload-key $TESTPOOL/$fs
+done
+for fs in "$TESTFS1" "$TESTFS2" "$TESTFS3"; do
+	log_must zfs load-key $TESTPOOL/$fs
+	log_must key_available $TESTPOOL/$fs
+	log_must zfs mount $TESTPOOL/$fs
+done
+
+log_pass "'zfs load-key' loads a key from a file"
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_location.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_location.ksh
index d0b1cdb20..2b0a6a292 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_location.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_location.ksh
@@ -70,4 +70,9 @@ log_must eval "echo $PASSPHRASE | zfs load-key -L prompt $TESTPOOL/$TESTFS1"
 log_must key_available $TESTPOOL/$TESTFS1
 log_must verify_keylocation $TESTPOOL/$TESTFS1 "file://$key_location"
 
+log_must zfs unload-key $TESTPOOL/$TESTFS1
+log_must zfs load-key -L $(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1
+log_must key_available $TESTPOOL/$TESTFS1
+log_must verify_keylocation $TESTPOOL/$TESTFS1 "file://$key_location"
+
 log_pass "'zfs load-key -L' overrides keylocation with provided value"
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh
index 7385b69cf..0435383ad 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh
@@ -52,15 +52,21 @@ log_must zfs create -o encryption=on -o keyformat=passphrase \
 log_must zfs create -o keyformat=passphrase \
 	-o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1/child
 
+log_must zfs create -o keyformat=passphrase \
+	-o keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1/child/child
+
 log_must zfs unmount $TESTPOOL/$TESTFS1
+log_must zfs unload-key $TESTPOOL/$TESTFS1/child/child
 log_must zfs unload-key $TESTPOOL/$TESTFS1/child
 log_must zfs unload-key $TESTPOOL/$TESTFS1
 
 log_must zfs load-key -r $TESTPOOL
 log_must key_available $TESTPOOL/$TESTFS1
 log_must key_available $TESTPOOL/$TESTFS1/child
+log_must key_available $TESTPOOL/$TESTFS1/child/child
 
 log_must zfs mount $TESTPOOL/$TESTFS1
 log_must zfs mount $TESTPOOL/$TESTFS1/child
+log_must zfs mount $TESTPOOL/$TESTFS1/child/child
 
 log_pass "'zfs load-key -r' recursively loads keys"
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_keylocation.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_keylocation.ksh
index 0d2e7ab8f..7a6bf75da 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_keylocation.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_set/zfs_set_keylocation.ksh
@@ -46,11 +46,12 @@ function cleanup
 {
 	datasetexists $TESTPOOL/$TESTFS1 && \
 		log_must zfs destroy -r $TESTPOOL/$TESTFS1
+	cleanup_https
 }
 log_onexit cleanup
 
-log_assert "Key location can only be 'prompt' or a file path for encryption" \
-	"roots, and 'none' for unencrypted volumes"
+log_assert "Key location can only be 'prompt', 'file://', or 'https://'" \
+	"for encryption roots, and 'none' for unencrypted volumes"
 
 log_must eval "echo $PASSPHRASE > /$TESTPOOL/pkey"
 
@@ -64,19 +65,15 @@ log_must zfs create -o encryption=on -o keyformat=passphrase \
 	-o keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1
 
 log_mustnot zfs set keylocation=none $TESTPOOL/$TESTFS1
-if true; then
-	log_mustnot zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1
-else
-	### SOON: ###
-	# file:///$TESTPOOL/pkey and /$TESTPOOL/pkey are equivalent on FreeBSD
-	# thanks to libfetch. Eventually we want to make the other platforms
-	# work this way as well, either by porting libfetch or by other means.
-	log_must zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1
-fi
+log_mustnot zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1
 
 log_must zfs set keylocation=file:///$TESTPOOL/pkey $TESTPOOL/$TESTFS1
 log_must verify_keylocation $TESTPOOL/$TESTFS1 "file:///$TESTPOOL/pkey"
 
+setup_https
+log_must zfs set keylocation=$(get_https_base_url)/PASSPHRASE $TESTPOOL/$TESTFS1
+log_must verify_keylocation $TESTPOOL/$TESTFS1 "$(get_https_base_url)/PASSPHRASE"
+
 log_must zfs set keylocation=prompt $TESTPOOL/$TESTFS1
 log_must verify_keylocation $TESTPOOL/$TESTFS1 "prompt"
 
@@ -97,5 +94,5 @@ log_mustnot zfs set keylocation=/$TESTPOOL/pkey $TESTPOOL/$TESTFS1/child
 
 log_must verify_keylocation $TESTPOOL/$TESTFS1/child "none"
 
-log_pass "Key location can only be 'prompt' or a file path for encryption" \
-	"roots, and 'none' for unencrypted volumes"
+log_pass "Key location can only be 'prompt', 'file://', or 'https://'" \
+	"for encryption roots, and 'none' for unencrypted volumes"