OpenZFS 9403 - assertion failed in arc_buf_destroy()

Assertion failed in arc_buf_destroy() when concurrently reading
block with checksum error.

Porting notes:
* The ability to zinject decompression errors has been added, but
  this only works at the zio_decompress() level, where we have all
  of the info we need to match against the user's zinject options.
* The decompress_fault test has been added to test the new zinject
  functionality
* We attempted to set zio_decompress_fail_fraction to (1 << 18) in
  ztest for further test coverage. Although this did uncover a few
  low priority issues, this unfortuantely also causes ztest to
  ASSERT in many locations where the code is working correctly since
  it is designed to fail on IO errors. Developers can manually set
  this variable with the '-o' option to find and debug issues.

Authored by: Matt Ahrens <[email protected]>
Reviewed by: George Wilson <[email protected]>
Reviewed by: Paul Dagnelie <[email protected]>
Reviewed by: Pavel Zakharov <[email protected]>
Reviewed by: Brian Behlendorf <[email protected]>
Approved by: Matt Ahrens <[email protected]>
Ported-by: Tom Caputi <[email protected]>

OpenZFS-issue: https://illumos.org/issues/9403
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/fa98e487a9
Closes #7822 

4 files changed, 59 insertions, 3 deletions

diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index a9594aeff..0a64e8cd5 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -523,7 +523,7 @@ tags = ['functional', 'exec']
 [tests/functional/fault]
 tests = ['auto_online_001_pos', 'auto_replace_001_pos', 'auto_spare_001_pos',
     'auto_spare_002_pos', 'auto_spare_ashift', 'auto_spare_multiple',
-    'scrub_after_resilver', 'decrypt_fault']
+    'scrub_after_resilver', 'decrypt_fault', 'decompress_fault']
 tags = ['functional', 'fault']
 
 [tests/functional/features/async_destroy]
diff --git a/tests/zfs-tests/tests/functional/fault/Makefile.am b/tests/zfs-tests/tests/functional/fault/Makefile.am
index 1153ad8d6..285e331a1 100644
--- a/tests/zfs-tests/tests/functional/fault/Makefile.am
+++ b/tests/zfs-tests/tests/functional/fault/Makefile.am
@@ -9,6 +9,7 @@ dist_pkgdata_SCRIPTS = \
 	auto_spare_ashift.ksh \
 	auto_spare_multiple.ksh \
 	decrypt_fault.ksh \
+	decompress_fault.ksh \
 	scrub_after_resilver.ksh
 
 dist_pkgdata_DATA = \
diff --git a/tests/zfs-tests/tests/functional/fault/decompress_fault.ksh b/tests/zfs-tests/tests/functional/fault/decompress_fault.ksh
new file mode 100755
index 000000000..ea831efdf
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/fault/decompress_fault.ksh
@@ -0,0 +1,55 @@
+#!/bin/ksh -p
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2018 by Datto Inc.
+# All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/fault/fault.cfg
+
+#
+# DESCRIPTION:
+# Test that injected decompression errors are handled correctly.
+#
+# STRATEGY:
+# 1. Create an compressed dataset with a test file
+# 2. Inject decompression errors on the file 20% of the time
+# 3. Read the file to confirm that errors are handled correctly
+# 4. Confirm that the decompression injection was added to the ZED logs
+#
+
+log_assert "Testing that injected decompression errors are handled correctly"
+
+function cleanup
+{
+	log_must set_tunable64 zfs_compressed_arc_enabled 1
+	log_must zinject -c all
+	default_cleanup_noexit
+}
+
+log_onexit cleanup
+
+default_mirror_setup_noexit $DISK1 $DISK2
+log_must set_tunable64 zfs_compressed_arc_enabled 0
+log_must zfs create -o compression=on $TESTPOOL/fs
+mntpt=$(get_prop mountpoint $TESTPOOL/fs)
+write_compressible $mntpt 32m 1 0 "testfile"
+log_must sync
+log_must zfs umount $TESTPOOL/fs
+log_must zfs mount $TESTPOOL/fs
+log_must zinject -a -t data -e decompress -f 20 $mntpt/testfile.0
+log_mustnot eval "cat $mntpt/testfile.0 > /dev/null"
+log_must eval "zpool events $TESTPOOL | grep -q 'data'"
+
+log_pass "Injected decompression errors are handled correctly"
diff --git a/tests/zfs-tests/tests/functional/fault/decrypt_fault.ksh b/tests/zfs-tests/tests/functional/fault/decrypt_fault.ksh
index 10008f22e..ca698f778 100755
--- a/tests/zfs-tests/tests/functional/fault/decrypt_fault.ksh
+++ b/tests/zfs-tests/tests/functional/fault/decrypt_fault.ksh
@@ -11,7 +11,7 @@
 #
 
 #
-# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+# Copyright (c) 2018 by Datto Inc.
 # All rights reserved.
 #
 
@@ -23,7 +23,7 @@
 # Test that injected decryption errors are handled correctly.
 #
 # STRATEGY:
-# 1. Create an encrypted dataset with an test file
+# 1. Create an encrypted dataset with a test file
 # 2. Inject decryption errors on the file 20% of the time
 # 3. Read the file to confirm that errors are handled correctly
 # 4. Confirm that the decryption injection was added to the ZED logs