Ratelimit deadman zevents as with delay zevents

Just as delay zevents can flood the zevent pipe when a vdev becomes
unresponsive, so do the deadman zevents.

Ratelimit deadman zevents according to the same tunable as for delay
zevents.

Enable deadman tests on FreeBSD and add a test for deadman event
ratelimiting. 

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Don Brady <[email protected]>
Signed-off-by: Ryan Moeller <[email protected]>
Closes #11786 

5 files changed, 90 insertions, 7 deletions

diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run
index cc2414550..32be9ef05 100644
--- a/tests/runfiles/common.run
+++ b/tests/runfiles/common.run
@@ -577,6 +577,12 @@ tags = ['functional', 'cp_files']
 tests = ['ctime_001_pos' ]
 tags = ['functional', 'ctime']
 
+[tests/functional/deadman]
+tests = ['deadman_ratelimit', 'deadman_sync', 'deadman_zio']
+pre =
+post =
+tags = ['functional', 'deadman']
+
 [tests/functional/delegate]
 tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos', 'zfs_allow_003_pos',
     'zfs_allow_004_pos', 'zfs_allow_005_pos', 'zfs_allow_006_pos',
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index 9f6bd856a..4952f3013 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -85,12 +85,6 @@ tags = ['functional', 'cli_root', 'zpool_split']
 tests = ['compress_004_pos']
 tags = ['functional', 'compression']
 
-[tests/functional/deadman:Linux]
-tests = ['deadman_sync', 'deadman_zio']
-pre =
-post =
-tags = ['functional', 'deadman']
-
 [tests/functional/devices:Linux]
 tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos']
 tags = ['functional', 'devices']
diff --git a/tests/zfs-tests/tests/functional/deadman/Makefile.am b/tests/zfs-tests/tests/functional/deadman/Makefile.am
index 7b70ca09d..097f23e88 100644
--- a/tests/zfs-tests/tests/functional/deadman/Makefile.am
+++ b/tests/zfs-tests/tests/functional/deadman/Makefile.am
@@ -1,5 +1,6 @@
 pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/deadman
 dist_pkgdata_SCRIPTS = \
+	deadman_ratelimit.ksh \
 	deadman_sync.ksh \
 	deadman_zio.ksh
 
diff --git a/tests/zfs-tests/tests/functional/deadman/deadman_ratelimit.ksh b/tests/zfs-tests/tests/functional/deadman/deadman_ratelimit.ksh
new file mode 100755
index 000000000..469117a56
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/deadman/deadman_ratelimit.ksh
@@ -0,0 +1,78 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Portions Copyright 2021 iXsystems, Inc.
+#
+
+# DESCRIPTION:
+#	Verify spa deadman events are rate limited
+#
+# STRATEGY:
+#	1. Reduce the zfs_slow_io_events_per_second to 1.
+#	2. Reduce the zfs_deadman_ziotime_ms to 1ms.
+#	3. Write data to a pool and read it back.
+#	4. Verify deadman events have been produced at a reasonable rate.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/deadman/deadman.cfg
+
+verify_runnable "both"
+
+function cleanup
+{
+	zinject -c all
+	default_cleanup_noexit
+
+	set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS
+	set_tunable64 DEADMAN_ZIOTIME_MS $ZIOTIME_DEFAULT
+}
+
+log_assert "Verify spa deadman events are rate limited"
+log_onexit cleanup
+
+OLD_SLOW_IO_EVENTS=$(get_tunable SLOW_IO_EVENTS_PER_SECOND)
+log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND 1
+log_must set_tunable64 DEADMAN_ZIOTIME_MS 1
+
+# Create a new pool in order to use the updated deadman settings.
+default_setup_noexit $DISK1
+log_must zpool events -c
+
+mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS)
+log_must file_write -b 1048576 -c 8 -o create -d 0 -f $mntpnt/file
+log_must zpool export $TESTPOOL
+log_must zpool import $TESTPOOL
+log_must zinject -d $DISK1 -D 5:1 $TESTPOOL
+log_must dd if=$mntpnt/file of=$TEST_BASE_DIR/devnull oflag=sync
+
+events=$(zpool events $TESTPOOL | grep -c ereport.fs.zfs.deadman)
+log_note "events=$events"
+if [ "$events" -lt 1 ]; then
+	log_fail "Expect >= 1 deadman events, $events found"
+fi
+if [ "$events" -gt 10 ]; then
+	log_fail "Expect <= 10 deadman events, $events found"
+fi
+
+log_pass "Verify spa deadman events are rate limited"
diff --git a/tests/zfs-tests/tests/functional/deadman/deadman_sync.ksh b/tests/zfs-tests/tests/functional/deadman/deadman_sync.ksh
index 5d803af85..b0b03f5d5 100755
--- a/tests/zfs-tests/tests/functional/deadman/deadman_sync.ksh
+++ b/tests/zfs-tests/tests/functional/deadman/deadman_sync.ksh
@@ -73,7 +73,11 @@ log_must zinject -c all
 log_must zpool sync
 
 # Log txg sync times for reference and the zpool event summary.
-log_must cat /proc/spl/kstat/zfs/$TESTPOOL/txgs
+if is_freebsd; then
+	log_must sysctl -n kstat.zfs.$TESTPOOL.txgs
+else
+	log_must cat /proc/spl/kstat/zfs/$TESTPOOL/txgs
+fi
 log_must zpool events
 
 # Verify at least 5 deadman events were logged.  The first after 5 seconds,