Allow to limit zed's syslog chattiness

Some usage patterns like send/recv of replication streams can
produce a large number of events. In such a case, the current
all-syslog.sh zedlet will hold up to its name, and flood the
logs with mostly redundant information. Two mitigate this
situation, this changeset introduces to new variables
ZED_SYSLOG_SUBCLASS_INCLUDE and ZED_SYSLOG_SUBCLASS_EXCLUDE
to zed.rc that give more control over which event classes end
up in the syslog.

Reviewed-by: loli10K <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Giuseppe Di Natale <[email protected]>
Signed-off-by: Tony Hutter <[email protected]>
Signed-off-by: Daniel Kobras <[email protected]>
Closes #6886 
Closes #7260 

9 files changed, 178 insertions, 7 deletions

diff --git a/cmd/zed/zed.d/all-debug.sh b/cmd/zed/zed.d/all-debug.sh
index 057e39b50..14b39caac 100755
--- a/cmd/zed/zed.d/all-debug.sh
+++ b/cmd/zed/zed.d/all-debug.sh
@@ -10,6 +10,8 @@
 
 : "${ZED_DEBUG_LOG:="${TMPDIR:="/tmp"}/zed.debug.log"}"
 
+zed_exit_if_ignoring_this_event
+
 lockfile="$(basename -- "${ZED_DEBUG_LOG}").lock"
 
 umask 077
diff --git a/cmd/zed/zed.d/all-syslog.sh b/cmd/zed/zed.d/all-syslog.sh
index 68d3cf360..cb9286500 100755
--- a/cmd/zed/zed.d/all-syslog.sh
+++ b/cmd/zed/zed.d/all-syslog.sh
@@ -5,6 +5,8 @@
 [ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
 . "${ZED_ZEDLET_DIR}/zed-functions.sh"
 
+zed_exit_if_ignoring_this_event
+
 zed_log_msg "eid=${ZEVENT_EID}" "class=${ZEVENT_SUBCLASS}" \
     "${ZEVENT_POOL_GUID:+"pool_guid=${ZEVENT_POOL_GUID}"}" \
     "${ZEVENT_VDEV_PATH:+"vdev_path=${ZEVENT_VDEV_PATH}"}" \
diff --git a/cmd/zed/zed.d/zed-functions.sh b/cmd/zed/zed.d/zed-functions.sh
index ed6a95914..fb16e9d36 100644
--- a/cmd/zed/zed.d/zed-functions.sh
+++ b/cmd/zed/zed.d/zed-functions.sh
@@ -438,3 +438,23 @@ zed_guid_to_pool()
 		$ZPOOL get -H -ovalue,name guid | awk '$1=='"$guid"' {print $2}'
 	fi
 }
+
+# zed_exit_if_ignoring_this_event
+#
+# Exit the script if we should ignore this event, as determined by
+# $ZED_SYSLOG_SUBCLASS_INCLUDE and $ZED_SYSLOG_SUBCLASS_EXCLUDE in zed.rc.
+# This function assumes you've imported the normal zed variables.
+zed_exit_if_ignoring_this_event()
+{
+	if [ -n "${ZED_SYSLOG_SUBCLASS_INCLUDE}" ]; then
+	    eval "case ${ZEVENT_SUBCLASS} in
+	    ${ZED_SYSLOG_SUBCLASS_INCLUDE});;
+	    *) exit 0;;
+	    esac"
+	elif [ -n "${ZED_SYSLOG_SUBCLASS_EXCLUDE}" ]; then
+	    eval "case ${ZEVENT_SUBCLASS} in
+	    ${ZED_SYSLOG_SUBCLASS_EXCLUDE}) exit 0;;
+	    *);;
+	    esac"
+	fi
+}
diff --git a/cmd/zed/zed.d/zed.rc b/cmd/zed/zed.d/zed.rc
index 8b0e476d5..35a4d1275 100644
--- a/cmd/zed/zed.d/zed.rc
+++ b/cmd/zed/zed.d/zed.rc
@@ -100,3 +100,14 @@ ZED_USE_ENCLOSURE_LEDS=1
 #
 #ZED_SYSLOG_TAG="zed"
 
+##
+# Which set of event subclasses to log
+# By default, events from all subclasses are logged.
+# If ZED_SYSLOG_SUBCLASS_INCLUDE is set, only subclasses
+# matching the pattern are logged. Use the pipe symbol (|)
+# or shell wildcards (*, ?) to match multiple subclasses.
+# Otherwise, if ZED_SYSLOG_SUBCLASS_EXCLUDE is set, the
+# matching subclasses are excluded from logging.
+#ZED_SYSLOG_SUBCLASS_INCLUDE="checksum|scrub_*|vdev.*"
+#ZED_SYSLOG_SUBCLASS_EXCLUDE="statechange|config_*|history_event"
+
diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index d0a6c8b73..8606fd8f0 100644
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -490,7 +490,7 @@ tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos']
 tags = ['functional', 'devices']
 
 [tests/functional/events]
-tests = ['events_001_pos', 'events_002_pos']
+tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter']
 tags = ['functional', 'events']
 
 [tests/functional/exec]
diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index 8918dd885..464b59d3f 100644
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -3109,6 +3109,21 @@ function wait_scrubbed
 	return 1
 }
 
+# Backup the zed.rc in our test directory so that we can edit it for our test.
+#
+# Returns: Backup file name.  You will need to pass this to zed_rc_restore().
+function zed_rc_backup
+{
+	zedrc_backup="$(mktemp)"
+	cp $ZEDLET_DIR/zed.rc $zedrc_backup
+	echo $zedrc_backup
+}
+
+function zed_rc_restore
+{
+	mv $1 $ZEDLET_DIR/zed.rc
+}
+
 #
 # Setup custom environment for the ZED.
 #
@@ -3247,6 +3262,23 @@ function zed_events_drain
 	done
 }
 
+# Set a variable in zed.rc to something, un-commenting it in the process.
+#
+# $1 variable
+# $2 value
+function zed_rc_set
+{
+	var="$1"
+	val="$2"
+	# Remove the line
+	cmd="'/$var/d'"
+	eval sed -i $cmd $ZEDLET_DIR/zed.rc
+
+	# Add it at the end
+	echo "$var=$val" >> $ZEDLET_DIR/zed.rc
+}
+
+
 #
 # Check is provided device is being active used as a swap device.
 #
diff --git a/tests/zfs-tests/tests/functional/events/Makefile.am b/tests/zfs-tests/tests/functional/events/Makefile.am
index 7813c18be..415e44b6b 100644
--- a/tests/zfs-tests/tests/functional/events/Makefile.am
+++ b/tests/zfs-tests/tests/functional/events/Makefile.am
@@ -5,4 +5,5 @@ dist_pkgdata_SCRIPTS = \
 	events.cfg \
 	events_common.kshlib \
 	events_001_pos.ksh \
-	events_002_pos.ksh
+	events_002_pos.ksh \
+	zed_rc_filter.ksh
diff --git a/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh b/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh
new file mode 100755
index 000000000..44652ee4c
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh
@@ -0,0 +1,104 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+# DESCRIPTION:
+# Verify zed.rc ZED_SYSLOG_SUBCLASS_INCLUDE/EXCLUDE event filtering works.
+#
+# STRATEGY:
+# 1. Execute zpool sub-commands on a pool.
+# 2. Test different combinations of ZED_SYSLOG_SUBCLASS_INCLUDE filtering.
+# 3. Execute zpool sub-commands on a pool.
+# 4. Test different combinations of ZED_SYSLOG_SUBCLASS_EXCLUDE filtering.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+	log_must zed_stop
+	zed_rc_restore $zedrc_backup
+}
+
+log_assert "Verify zpool sub-commands generate expected events"
+log_onexit cleanup
+
+log_must zpool events -c
+log_must zed_start
+
+# Backup our zed.rc
+zedrc_backup=$(zed_rc_backup)
+
+log_note "Include a single event type"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE history_event
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.history_event" \
+    "zfs set compression=off $TESTPOOL/$TESTFS"
+
+log_note "Include a single event type with wildcards"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE '*history_event*'
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.history_event" \
+    "zfs set compression=off $TESTPOOL/$TESTFS"
+
+log_note "Test a filter of a non-match and a match"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'foobar|*history_event*'
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.history_event" \
+    "zfs set compression=off $TESTPOOL/$TESTFS"
+
+log_note "Include multiple events"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'scrub_start|scrub_finish'
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.scrub_start" \
+    -e "sysevent.fs.zfs.scrub_finish" \
+    "zpool scrub $TESTPOOL && wait_scrubbed $TESTPOOL"
+
+# We can't use run_and_verify() for exclusions, so run the rest of the tests
+# manually.
+log_note "Test one exclusion"
+zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE 'history_event'
+truncate -s 0 $ZED_DEBUG_LOG
+log_must zfs set compression=off $TESTPOOL/$TESTFS
+log_must file_wait $ZED_DEBUG_LOG 3
+log_mustnot grep -q history_event $ZED_DEBUG_LOG
+
+log_note "Test one exclusion with wildcards"
+zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE '*history_event*'
+truncate -s 0 $ZED_DEBUG_LOG
+log_must zfs set compression=off $TESTPOOL/$TESTFS
+log_must file_wait $ZED_DEBUG_LOG 3
+log_mustnot grep -q history_event $ZED_DEBUG_LOG
+
+log_note "Test one inclusion and one exclusion"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'scrub_start'
+zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE 'scrub_finish'
+truncate -s 0 $ZED_DEBUG_LOG
+zpool scrub $TESTPOOL
+wait_scrubbed $TESTPOOL
+log_must file_wait $ZED_DEBUG_LOG 3
+log_must grep -q scrub_start $ZED_DEBUG_LOG
+log_mustnot grep -q scrub_finish $ZED_DEBUG_LOG
+
+log_pass "zed.rc event filtering works correctly."
diff --git a/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh b/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh
index 558cb065f..3b124bdd5 100755
--- a/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh
+++ b/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh
@@ -32,16 +32,15 @@
 log_assert "Testing the scrub after resilver zedlet"
 
 # Backup our zed.rc
-zedrc_backup="$(mktemp)"
-log_must cp $ZEDLET_DIR/zed.rc $zedrc_backup
+zedrc_backup=$(zed_rc_backup)
 
-# Enable ZED_SCRUB_AFTER_RESILVER
-eval "sed -i 's/\#ZED_SCRUB_AFTER_RESILVER/ZED_SCRUB_AFTER_RESILVER/g' $ZEDLET_DIR/zed.rc"
+# Enable ZED_SCRUB_AFTER_RESILVER in zed.rc
+zed_rc_set ZED_SCRUB_AFTER_RESILVER 1
 
 function cleanup
 {
 	# Restore our zed.rc
-	log_must mv $zedrc_backup $ZEDLET_DIR/zed.rc
+	log_must zed_rc_restore $zedrc_backup
 	default_cleanup_noexit
 }