Replace "email" ZEDLETs with "notify" ZEDLETs

Several ZEDLETs already exist for sending email in reponse to a
particular zevent.  While email is ubiquitous, alternative methods may
be better suited for some configurations.  Instead of duplicating the
"email" ZEDLETs for every future notification method, it is preferable
to abstract the notification method into a function.  This has the
added benefit of reducing the amount of code duplicated between
ZEDLETs, and allowing related bugs to be fixed in a single location.

This commit replaces the existing "email" ZEDLETs with corresponding
"notify" ZEDLETs.  In addition, the ZEDLET code for sending an
email message has been moved into the zed_notify_email() function.
And this zed_notify_email() has been added to a generic zed_notify()
function for sending notifications via all available methods that
have been configured.

This commit also changes a couple of related zed.rc variables.
ZED_EMAIL_INTERVAL_SECS is changed to ZED_NOTIFY_INTERVAL_SECS,
and ZED_EMAIL_VERBOSE is changed to ZED_NOTIFY_VERBOSE.  Note that
ZED_EMAIL remains unchanged as its use is solely for the email
notification method.

Signed-off-by: Chris Dunlap <[email protected]>

14 files changed, 300 insertions, 247 deletions

diff --git a/cmd/zed/zed.d/checksum-email.sh b/cmd/zed/zed.d/checksum-email.sh
deleted file mode 120000
index f95bec215..000000000
--- a/cmd/zed/zed.d/checksum-email.sh
+++ /dev/null
@@ -1 +0,0 @@
-io-email.sh
\ No newline at end of file
diff --git a/cmd/zed/zed.d/checksum-notify.sh b/cmd/zed/zed.d/checksum-notify.sh
new file mode 120000
index 000000000..900873807
--- /dev/null
+++ b/cmd/zed/zed.d/checksum-notify.sh
@@ -0,0 +1 @@
+io-notify.sh
\ No newline at end of file
diff --git a/cmd/zed/zed.d/data-email.sh b/cmd/zed/zed.d/data-email.sh
deleted file mode 100755
index 2dae8ff6b..000000000
--- a/cmd/zed/zed.d/data-email.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/sh
-#
-# Send email to ZED_EMAIL in response to a DATA error.
-#
-# Only one email per ZED_EMAIL_INTERVAL_SECS will be sent for a given
-# class/pool combination.  This protects against spamming the recipient
-# should multiple events occur together in time for the same pool.
-#
-# Exit codes:
-#   0: email sent
-#   1: email failed
-#   2: email not configured
-#   3: email suppressed
-#   9: internal error
-
-[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
-. "${ZED_ZEDLET_DIR}/zed-functions.sh"
-
-[ -n "${ZED_EMAIL}" ] || exit 2
-
-[ -n "${ZEVENT_POOL}" ] || exit 9
-[ -n "${ZEVENT_SUBCLASS}" ] || exit 9
-
-if [ "${ZEVENT_SUBCLASS}" != "data" ]; then \
-    zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
-    exit 9
-fi
-
-zed_check_cmd "mail" || exit 9
-
-zed_rate_limit "${ZEVENT_POOL};${ZEVENT_SUBCLASS};email" || exit 3
-
-umask 077
-email_subject="ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on $(hostname)"
-email_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
-cat > "${email_pathname}" <<EOF
-A ZFS ${ZEVENT_SUBCLASS} error has been detected:
-
-   eid: ${ZEVENT_EID}
-  host: $(hostname)
-  time: ${ZEVENT_TIME_STRING}
-  pool: ${ZEVENT_POOL}
-EOF
-
-mail -s "${email_subject}" "${ZED_EMAIL}" < "${email_pathname}"
-mail_status=$?
-
-if [ "${mail_status}" -ne 0 ]; then
-    zed_log_msg "mail exit=${mail_status}"
-    exit 1
-fi
-rm -f "${email_pathname}"
-exit 0
diff --git a/cmd/zed/zed.d/data-notify.sh b/cmd/zed/zed.d/data-notify.sh
new file mode 100755
index 000000000..6b65c3815
--- /dev/null
+++ b/cmd/zed/zed.d/data-notify.sh
@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# Send notification in response to a DATA error.
+#
+# Only one notification per ZED_NOTIFY_INTERVAL_SECS will be sent for a given
+# class/pool combination.  This protects against spamming the recipient
+# should multiple events occur together in time for the same pool.
+#
+# Exit codes:
+#   0: notification sent
+#   1: notification failed
+#   2: notification not configured
+#   3: notification suppressed
+#   9: internal error
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ -n "${ZEVENT_POOL}" ] || exit 9
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 9
+
+if [ "${ZEVENT_SUBCLASS}" != "data" ]; then \
+    zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
+    exit 9
+fi
+
+zed_rate_limit "${ZEVENT_POOL};${ZEVENT_SUBCLASS};notify" || exit 3
+
+umask 077
+note_subject="ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on $(hostname)"
+note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
+{
+    echo "ZFS has detected a ${ZEVENT_SUBCLASS} error:"
+    echo
+    echo "   eid: ${ZEVENT_EID}"
+    echo " class: ${ZEVENT_SUBCLASS}"
+    echo "  host: $(hostname)"
+    echo "  time: ${ZEVENT_TIME_STRING}"
+    echo "  pool: ${ZEVENT_POOL}"
+
+} > "${note_pathname}"
+
+zed_notify "${note_subject}" "${note_pathname}"; rv=$?
+rm -f "${note_pathname}"
+exit "${rv}"
diff --git a/cmd/zed/zed.d/generic-email.sh b/cmd/zed/zed.d/generic-email.sh
deleted file mode 100755
index ad022e034..000000000
--- a/cmd/zed/zed.d/generic-email.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/sh
-#
-# Send email to ZED_EMAIL in response to a given zevent.
-#
-# This is a generic script than can be symlinked to a file in the
-# enabled-zedlets directory to have an email sent when a particular class of
-# zevents occurs.  The symlink filename must begin with the zevent (sub)class
-# string (e.g., "probe_failure-email.sh" for the "probe_failure" subclass).
-# Refer to the zed(8) manpage for details.
-#
-# Exit codes:
-#   0: email sent
-#   1: email failed
-#   2: email not configured
-#   3: email suppressed
-
-[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
-. "${ZED_ZEDLET_DIR}/zed-functions.sh"
-
-[ -n "${ZED_EMAIL}" ] || exit 2
-
-# Rate-limit the message based in part on the filename.
-#
-rate_limit_tag="${ZEVENT_POOL};${ZEVENT_SUBCLASS};$(basename -- "$0")"
-rate_limit_interval="${ZED_EMAIL_INTERVAL_SECS}"
-zed_rate_limit "${rate_limit_tag}" "${rate_limit_interval}" || exit 3
-
-umask 077
-pool_str="${ZEVENT_POOL:+" for ${ZEVENT_POOL}"}"
-host_str=" on $(hostname)"
-email_subject="ZFS ${ZEVENT_SUBCLASS} event${pool_str}${host_str}"
-email_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
-{
-    echo "ZFS has posted the following event:"
-    echo
-    echo "   eid: ${ZEVENT_EID}"
-    echo " class: ${ZEVENT_SUBCLASS}"
-    echo "  host: $(hostname)"
-    echo "  time: ${ZEVENT_TIME_STRING}"
-
-    if [ -n "${ZEVENT_VDEV_PATH}" ]; then
-        echo " vpath: ${ZEVENT_VDEV_PATH}"
-        [ -n "${ZEVENT_VDEV_TYPE}" ] && echo " vtype: ${ZEVENT_VDEV_TYPE}"
-    fi
-
-    [ -n "${ZEVENT_POOL}" ] && [ -x "${ZPOOL}" ] \
-        && "${ZPOOL}" status "${ZEVENT_POOL}"
-
-} > "${email_pathname}"
-
-mail -s "${email_subject}" "${ZED_EMAIL}" < "${email_pathname}"
-mail_status=$?
-
-if [ "${mail_status}" -ne 0 ]; then
-    zed_log_msg "mail exit=${mail_status}"
-    exit 1
-fi
-rm -f "${email_pathname}"
-exit 0
diff --git a/cmd/zed/zed.d/generic-notify.sh b/cmd/zed/zed.d/generic-notify.sh
new file mode 100755
index 000000000..e438031a0
--- /dev/null
+++ b/cmd/zed/zed.d/generic-notify.sh
@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# Send notification in response to a given zevent.
+#
+# This is a generic script than can be symlinked to a file in the
+# enabled-zedlets directory to have a notification sent when a particular
+# class of zevents occurs.  The symlink filename must begin with the zevent
+# (sub)class string (e.g., "probe_failure-notify.sh" for the "probe_failure"
+# subclass).  Refer to the zed(8) manpage for details.
+#
+# Only one notification per ZED_NOTIFY_INTERVAL_SECS will be sent for a given
+# class/pool combination.  This protects against spamming the recipient
+# should multiple events occur together in time for the same pool.
+#
+# Exit codes:
+#   0: notification sent
+#   1: notification failed
+#   2: notification not configured
+#   3: notification suppressed
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+# Rate-limit the notification based in part on the filename.
+#
+rate_limit_tag="${ZEVENT_POOL};${ZEVENT_SUBCLASS};$(basename -- "$0")"
+rate_limit_interval="${ZED_NOTIFY_INTERVAL_SECS}"
+zed_rate_limit "${rate_limit_tag}" "${rate_limit_interval}" || exit 3
+
+umask 077
+pool_str="${ZEVENT_POOL:+" for ${ZEVENT_POOL}"}"
+host_str=" on $(hostname)"
+note_subject="ZFS ${ZEVENT_SUBCLASS} event${pool_str}${host_str}"
+note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
+{
+    echo "ZFS has posted the following event:"
+    echo
+    echo "   eid: ${ZEVENT_EID}"
+    echo " class: ${ZEVENT_SUBCLASS}"
+    echo "  host: $(hostname)"
+    echo "  time: ${ZEVENT_TIME_STRING}"
+
+    [ -n "${ZEVENT_VDEV_TYPE}" ] && echo " vtype: ${ZEVENT_VDEV_TYPE}"
+    [ -n "${ZEVENT_VDEV_PATH}" ] && echo " vpath: ${ZEVENT_VDEV_PATH}"
+    [ -n "${ZEVENT_VDEV_GUID}" ] && echo " vguid: ${ZEVENT_VDEV_GUID}"
+
+    [ -n "${ZEVENT_POOL}" ] && [ -x "${ZPOOL}" ] \
+        && "${ZPOOL}" status "${ZEVENT_POOL}"
+
+} > "${note_pathname}"
+
+zed_notify "${note_subject}" "${note_pathname}"; rv=$?
+rm -f "${note_pathname}"
+exit "${rv}"
diff --git a/cmd/zed/zed.d/io-email.sh b/cmd/zed/zed.d/io-email.sh
deleted file mode 100755
index 1854b1593..000000000
--- a/cmd/zed/zed.d/io-email.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/bin/sh
-#
-# Send email to ZED_EMAIL in response to a CHECKSUM or IO error.
-#
-# Only one email per ZED_EMAIL_INTERVAL_SECS will be sent for a given
-# class/pool/vdev combination.  This protects against spamming the recipient
-# should multiple events occur together in time for the same pool/device.
-#
-# Exit codes:
-#   0: email sent
-#   1: email failed
-#   2: email not configured
-#   3: email suppressed
-#   9: internal error
-
-[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
-. "${ZED_ZEDLET_DIR}/zed-functions.sh"
-
-[ -n "${ZED_EMAIL}" ] || exit 2
-
-[ -n "${ZEVENT_POOL}" ] || exit 9
-[ -n "${ZEVENT_SUBCLASS}" ] || exit 9
-[ -n "${ZEVENT_VDEV_PATH}" ] || exit 9
-
-if [ "${ZEVENT_SUBCLASS}" != "checksum" ] \
-        && [ "${ZEVENT_SUBCLASS}" != "io" ]; then
-    zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
-    exit 9
-fi
-
-zed_check_cmd "mail" || exit 9
-
-zed_rate_limit "${ZEVENT_POOL};${ZEVENT_VDEV_PATH};${ZEVENT_SUBCLASS};email" \
-    || exit 3
-
-umask 077
-email_subject="ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on $(hostname)"
-email_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
-cat > "${email_pathname}" <<EOF
-A ZFS ${ZEVENT_SUBCLASS} error has been detected:
-
-   eid: ${ZEVENT_EID}
-  host: $(hostname)
-  time: ${ZEVENT_TIME_STRING}
-  pool: ${ZEVENT_POOL}
-  vdev: ${ZEVENT_VDEV_TYPE}:${ZEVENT_VDEV_PATH}
-EOF
-
-mail -s "${email_subject}" "${ZED_EMAIL}" < "${email_pathname}"
-mail_status=$?
-
-if [ "${mail_status}" -ne 0 ]; then
-    zed_log_msg "mail exit=${mail_status}"
-    exit 1
-fi
-rm -f "${email_pathname}"
-exit 0
diff --git a/cmd/zed/zed.d/io-notify.sh b/cmd/zed/zed.d/io-notify.sh
new file mode 100755
index 000000000..46b066945
--- /dev/null
+++ b/cmd/zed/zed.d/io-notify.sh
@@ -0,0 +1,55 @@
+#!/bin/sh
+#
+# Send notification in response to a CHECKSUM or IO error.
+#
+# Only one notification per ZED_NOTIFY_INTERVAL_SECS will be sent for a given
+# class/pool/vdev combination.  This protects against spamming the recipient
+# should multiple events occur together in time for the same pool/device.
+#
+# Exit codes:
+#   0: notification sent
+#   1: notification failed
+#   2: notification not configured
+#   3: notification suppressed
+#   9: internal error
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ -n "${ZEVENT_POOL}" ] || exit 9
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 9
+[ -n "${ZEVENT_VDEV_GUID}" ] || exit 9
+
+if [ "${ZEVENT_SUBCLASS}" != "checksum" ] \
+        && [ "${ZEVENT_SUBCLASS}" != "io" ]; then
+    zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
+    exit 9
+fi
+
+zed_rate_limit "${ZEVENT_POOL};${ZEVENT_VDEV_GUID};${ZEVENT_SUBCLASS};notify" \
+    || exit 3
+
+umask 077
+note_subject="ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on $(hostname)"
+note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
+{
+    [ "${ZEVENT_SUBCLASS}" = "io" ] && article="an" || article="a"
+
+    echo "ZFS has detected ${article} ${ZEVENT_SUBCLASS} error:"
+    echo
+    echo "   eid: ${ZEVENT_EID}"
+    echo " class: ${ZEVENT_SUBCLASS}"
+    echo "  host: $(hostname)"
+    echo "  time: ${ZEVENT_TIME_STRING}"
+
+    [ -n "${ZEVENT_VDEV_TYPE}" ] && echo " vtype: ${ZEVENT_VDEV_TYPE}"
+    [ -n "${ZEVENT_VDEV_PATH}" ] && echo " vpath: ${ZEVENT_VDEV_PATH}"
+    [ -n "${ZEVENT_VDEV_GUID}" ] && echo " vguid: ${ZEVENT_VDEV_GUID}"
+
+    echo "  pool: ${ZEVENT_POOL}"
+
+} > "${note_pathname}"
+
+zed_notify "${note_subject}" "${note_pathname}"; rv=$?
+rm -f "${note_pathname}"
+exit "${rv}"
diff --git a/cmd/zed/zed.d/resilver.finish-email.sh b/cmd/zed/zed.d/resilver.finish-email.sh
deleted file mode 120000
index 1afad3258..000000000
--- a/cmd/zed/zed.d/resilver.finish-email.sh
+++ /dev/null
@@ -1 +0,0 @@
-scrub.finish-email.sh
\ No newline at end of file
diff --git a/cmd/zed/zed.d/resilver.finish-notify.sh b/cmd/zed/zed.d/resilver.finish-notify.sh
new file mode 120000
index 000000000..2635dcce1
--- /dev/null
+++ b/cmd/zed/zed.d/resilver.finish-notify.sh
@@ -0,0 +1 @@
+scrub.finish-notify.sh
\ No newline at end of file
diff --git a/cmd/zed/zed.d/scrub.finish-email.sh b/cmd/zed/zed.d/scrub.finish-email.sh
deleted file mode 100755
index 4a8155caf..000000000
--- a/cmd/zed/zed.d/scrub.finish-email.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/bin/sh
-#
-# Send email to ZED_EMAIL in response to a RESILVER.FINISH or SCRUB.FINISH.
-#
-# By default, "zpool status" output will only be included for a scrub.finish
-# zevent if the pool is not healthy; to always include its output, set
-# ZED_EMAIL_VERBOSE=1.
-#
-# Exit codes:
-#   0: email sent
-#   1: email failed
-#   2: email not configured
-#   3: email suppressed
-#   9: internal error
-
-[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
-. "${ZED_ZEDLET_DIR}/zed-functions.sh"
-
-[ -n "${ZED_EMAIL}" ] || exit 2
-
-[ -n "${ZEVENT_POOL}" ] || exit 9
-[ -n "${ZEVENT_SUBCLASS}" ] || exit 9
-
-if   [ "${ZEVENT_SUBCLASS}" = "resilver.finish" ]; then
-    action="resilver"
-elif [ "${ZEVENT_SUBCLASS}" = "scrub.finish" ]; then
-    action="scrub"
-else
-    zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
-    exit 9
-fi
-
-zed_check_cmd "mail" "${ZPOOL}" || exit 9
-
-# For scrub, suppress email if pool is healthy and verbosity is not enabled.
-#
-if [ "${ZEVENT_SUBCLASS}" = "scrub.finish" ]; then
-    healthy="$("${ZPOOL}" status -x "${ZEVENT_POOL}" \
-        | grep "'${ZEVENT_POOL}' is healthy")"
-    [ -n "${healthy}" ] && [ "${ZED_EMAIL_VERBOSE}" -eq 0 ] && exit 3
-fi
-
-umask 077
-email_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)"
-email_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
-cat > "${email_pathname}" <<EOF
-ZFS has finished a ${action}:
-
-   eid: ${ZEVENT_EID}
-  host: $(hostname)
-  time: ${ZEVENT_TIME_STRING}
-$("${ZPOOL}" status "${ZEVENT_POOL}")
-EOF
-
-mail -s "${email_subject}" "${ZED_EMAIL}" < "${email_pathname}"
-mail_status=$?
-
-if [ "${mail_status}" -ne 0 ]; then
-    zed_log_msg "mail exit=${mail_status}"
-    exit 1
-fi
-rm -f "${email_pathname}"
-exit 0
diff --git a/cmd/zed/zed.d/scrub.finish-notify.sh b/cmd/zed/zed.d/scrub.finish-notify.sh
new file mode 100755
index 000000000..b4dd1dd43
--- /dev/null
+++ b/cmd/zed/zed.d/scrub.finish-notify.sh
@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# Send notification in response to a RESILVER.FINISH or SCRUB.FINISH.
+#
+# By default, "zpool status" output will only be included for a scrub.finish
+# zevent if the pool is not healthy; to always include its output, set
+# ZED_NOTIFY_VERBOSE=1.
+#
+# Exit codes:
+#   0: notification sent
+#   1: notification failed
+#   2: notification not configured
+#   3: notification suppressed
+#   9: internal error
+
+[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
+. "${ZED_ZEDLET_DIR}/zed-functions.sh"
+
+[ -n "${ZEVENT_POOL}" ] || exit 9
+[ -n "${ZEVENT_SUBCLASS}" ] || exit 9
+
+if   [ "${ZEVENT_SUBCLASS}" = "resilver.finish" ]; then
+    action="resilver"
+elif [ "${ZEVENT_SUBCLASS}" = "scrub.finish" ]; then
+    action="scrub"
+else
+    zed_log_err "unsupported event class \"${ZEVENT_SUBCLASS}\""
+    exit 9
+fi
+
+zed_check_cmd "${ZPOOL}" || exit 9
+
+# For scrub, suppress notification if the pool is healthy
+# and verbosity is not enabled.
+#
+if [ "${ZEVENT_SUBCLASS}" = "scrub.finish" ]; then
+    healthy="$("${ZPOOL}" status -x "${ZEVENT_POOL}" \
+        | grep "'${ZEVENT_POOL}' is healthy")"
+    [ -n "${healthy}" ] && [ "${ZED_NOTIFY_VERBOSE}" -eq 0 ] && exit 3
+fi
+
+umask 077
+note_subject="ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on $(hostname)"
+note_pathname="${TMPDIR:="/tmp"}/$(basename -- "$0").${ZEVENT_EID}.$$"
+{
+    echo "ZFS has finished a ${action}:"
+    echo
+    echo "   eid: ${ZEVENT_EID}"
+    echo " class: ${ZEVENT_SUBCLASS}"
+    echo "  host: $(hostname)"
+    echo "  time: ${ZEVENT_TIME_STRING}"
+
+    "${ZPOOL}" status "${ZEVENT_POOL}"
+
+} > "${note_pathname}"
+
+zed_notify "${note_subject}" "${note_pathname}"; rv=$?
+rm -f "${note_pathname}"
+exit "${rv}"
diff --git a/cmd/zed/zed.d/zed-functions.sh b/cmd/zed/zed.d/zed-functions.sh
index b42911b2e..14909d38c 100644
--- a/cmd/zed/zed.d/zed-functions.sh
+++ b/cmd/zed/zed.d/zed-functions.sh
@@ -5,9 +5,9 @@
 
 # Variable Defaults
 #
-: "${ZED_EMAIL_INTERVAL_SECS:=3600}"
-: "${ZED_EMAIL_VERBOSE:=0}"
 : "${ZED_LOCKDIR:="/var/lock"}"
+: "${ZED_NOTIFY_INTERVAL_SECS:=3600}"
+: "${ZED_NOTIFY_VERBOSE:=0}"
 : "${ZED_RUNDIR:="/var/run"}"
 : "${ZED_SYSLOG_PRIORITY:="daemon.notice"}"
 : "${ZED_SYSLOG_TAG:="zed"}"
@@ -171,6 +171,78 @@ zed_unlock()
 }
 
 
+# zed_notify (subject, pathname)
+#
+# Send a notification via all available methods.
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Return
+#   0: notification succeeded via at least one method
+#   1: notification failed
+#   2: no notification methods configured
+#
+zed_notify()
+{
+    local subject="$1"
+    local pathname="$2"
+    local num_success=0
+    local num_failure=0
+
+    zed_notify_email "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
+    [ "${num_success}" -gt 0 ] && return 0
+    [ "${num_failure}" -gt 0 ] && return 1
+    return 2
+}
+
+
+# zed_notify_email (subject, pathname)
+#
+# Send a notification via email to the address specified by ZED_EMAIL.
+#
+# Requires the mail executable to be installed in the standard PATH.
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_EMAIL
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_email()
+{
+    local subject="$1"
+    local pathname="${2:-"/dev/null"}"
+
+    [ -n "${ZED_EMAIL}" ] || return 2
+
+    [ -n "${subject}" ] || return 1
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err "mail cannot read \"${pathname}\""
+        return 1
+    fi
+
+    zed_check_cmd "mail" || return 1
+
+    mail -s "${subject}" "${ZED_EMAIL}" < "${pathname}" >/dev/null 2>&1; rv=$?
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "mail exit=${rv}"
+        return 1
+    fi
+    return 0
+}
+
+
 # zed_rate_limit (tag, [interval])
 #
 # Check whether an event of a given type [tag] has already occurred within the
@@ -183,7 +255,7 @@ zed_unlock()
 #   interval: time interval in seconds (OPTIONAL)
 #
 # Globals
-#   ZED_EMAIL_INTERVAL_SECS
+#   ZED_NOTIFY_INTERVAL_SECS
 #   ZED_RUNDIR
 #
 # Return
@@ -196,7 +268,7 @@ zed_unlock()
 zed_rate_limit()
 {
     local tag="$1"
-    local interval="${2:-${ZED_EMAIL_INTERVAL_SECS}}"
+    local interval="${2:-${ZED_NOTIFY_INTERVAL_SECS}}"
     local lockfile="zed.zedlet.state.lock"
     local lockfile_fd=9
     local statefile="${ZED_RUNDIR}/zed.zedlet.state"
diff --git a/cmd/zed/zed.d/zed.rc b/cmd/zed/zed.d/zed.rc
index 4c53207d7..05f84c877 100644
--- a/cmd/zed/zed.d/zed.rc
+++ b/cmd/zed/zed.d/zed.rc
@@ -8,28 +8,28 @@
 #ZED_DEBUG_LOG="/tmp/zed.debug.log"
 
 ##
-# Email address of the zpool administrator.
+# Email address of the zpool administrator for receipt of notifications.
 #   Email will only be sent if ZED_EMAIL is defined.
 # Disabled by default; uncomment to enable.
 #
 #ZED_EMAIL="root"
 
 ##
-# Minimum number of seconds between emails for a similar event.
+# Default directory for zed lock files.
 #
-#ZED_EMAIL_INTERVAL_SECS=3600
+#ZED_LOCKDIR="/var/lock"
 
 ##
-# Email verbosity.
-#   If set to 0, suppress email if the pool is healthy.
-#   If set to 1, send email regardless of pool health.
+# Minimum number of seconds between notifications for a similar event.
 #
-#ZED_EMAIL_VERBOSE=0
+#ZED_NOTIFY_INTERVAL_SECS=3600
 
 ##
-# Default directory for zed lock files.
+# Notification verbosity.
+#   If set to 0, suppress notification if the pool is healthy.
+#   If set to 1, send notification regardless of pool health.
 #
-#ZED_LOCKDIR="/var/lock"
+#ZED_NOTIFY_VERBOSE=0
 
 ##
 # Default directory for zed state files.