Fix /etc/hostid on root pool deadlock

Accidentally introduced by dc04a8c which now takes the SCL_VDEV lock
as a reader in zfs_blkptr_verify().  A deadlock can occur if the
/etc/hostid file resides on a dataset in the same pool.  This is
because reading the /etc/hostid file may occur while the caller is
holding the SCL_VDEV lock as a writer.  For example, to perform a
`zpool attach` as shown in the abbreviated stack below.

To resolve the issue we cache the system's hostid when initializing
the spa_t, or when modifying the multihost property.  The cached
value is then relied upon for subsequent accesses.

Call Trace:
    spa_config_enter+0x1e8/0x350 [zfs]
    zfs_blkptr_verify+0x33c/0x4f0 [zfs] <--- trying read lock
    zio_read+0x6c/0x140 [zfs]
    ...
    vfs_read+0xfc/0x1e0
    kernel_read+0x50/0x90
    ...
    spa_get_hostid+0x1c/0x38 [zfs]
    spa_config_generate+0x1a0/0x610 [zfs]
    vdev_label_init+0xa0/0xc80 [zfs]
    vdev_create+0x98/0xe0 [zfs]
    spa_vdev_attach+0x14c/0xb40 [zfs] <--- grabbed write lock

Reviewed-by: loli10K <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #9256 
Closes #9285 

2 files changed, 91 insertions, 0 deletions

diff --git a/tests/zfs-tests/tests/functional/mmp/Makefile.am b/tests/zfs-tests/tests/functional/mmp/Makefile.am
index e39a0a5aa..2848fd4ce 100644
--- a/tests/zfs-tests/tests/functional/mmp/Makefile.am
+++ b/tests/zfs-tests/tests/functional/mmp/Makefile.am
@@ -12,6 +12,7 @@ dist_pkgdata_SCRIPTS = \
 	mmp_reset_interval.ksh \
 	mmp_on_zdb.ksh \
 	mmp_write_distribution.ksh \
+	mmp_hostid.ksh \
 	setup.ksh \
 	cleanup.ksh
 
diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_hostid.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_hostid.ksh
new file mode 100755
index 000000000..b492b1070
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/mmp/mmp_hostid.ksh
@@ -0,0 +1,90 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
+#
+
+# DESCRIPTION:
+#	Verify the hostid file can reside on a ZFS dataset.
+#
+# STRATEGY:
+#	1. Create a non-redundant pool
+#	2. Create an 'etc' dataset containing a valid hostid file
+#	3. Create a file so the pool will have some contents
+#	4. Verify multihost cannot be enabled until the /etc/hostid is linked
+#	5. Verify vdevs may be attached and detached
+#	6. Verify normal, cache, log and special vdevs can be added
+#	7. Verify normal, cache, and log vdevs can be removed
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/mmp/mmp.cfg
+. $STF_SUITE/tests/functional/mmp/mmp.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+	default_cleanup_noexit
+	log_must rm $MMP_DIR/file.{0,1,2,3,4,5}
+	log_must rmdir $MMP_DIR
+	log_must mmp_clear_hostid
+}
+
+log_assert "Verify hostid file can reside on a ZFS dataset"
+log_onexit cleanup
+
+log_must mkdir -p $MMP_DIR
+log_must truncate -s $MINVDEVSIZE $MMP_DIR/file.{0,1,2,3,4,5}
+
+# 1. Create a non-redundant pool
+log_must zpool create $MMP_POOL $MMP_DIR/file.0
+
+# 2. Create an 'etc' dataset containing a valid hostid file; caching is
+#    disabled on the dataset to force the hostid to be read from disk.
+log_must zfs create -o primarycache=none -o secondarycache=none $MMP_POOL/etc
+mntpnt_etc=$(get_prop mountpoint $MMP_POOL/etc)
+log_must mmp_set_hostid $HOSTID1
+log_must mv $HOSTID_FILE $mntpnt_etc/hostid
+
+# 3. Create a file so the pool will have some contents
+log_must zfs create $MMP_POOL/fs
+mntpnt_fs=$(get_prop mountpoint $MMP_POOL/fs)
+log_must mkfile 1M $fs_mntpnt/file
+
+# 4. Verify multihost cannot be enabled until the /etc/hostid is linked
+log_mustnot zpool set multihost=on $MMP_POOL
+log_must ln -s $mntpnt_etc/hostid $HOSTID_FILE
+log_must zpool set multihost=on $MMP_POOL
+
+# 5. Verify vdevs may be attached and detached
+log_must zpool attach $MMP_POOL $MMP_DIR/file.0 $MMP_DIR/file.1
+log_must zpool detach $MMP_POOL $MMP_DIR/file.1
+
+# 6. Verify normal, cache, log and special vdevs can be added
+log_must zpool add $MMP_POOL $MMP_DIR/file.1
+log_must zpool add $MMP_POOL $MMP_DIR/file.2
+log_must zpool add $MMP_POOL cache $MMP_DIR/file.3
+log_must zpool add $MMP_POOL log $MMP_DIR/file.4
+log_must zpool add $MMP_POOL special $MMP_DIR/file.5
+
+# 7. Verify normal, cache, and log vdevs can be removed
+log_must zpool remove $MMP_POOL $MMP_DIR/file.2
+log_must zpool remove $MMP_POOL $MMP_DIR/file.3
+log_must zpool remove $MMP_POOL $MMP_DIR/file.4
+
+log_pass "Verify hostid file can reside on a ZFS dataset."