Initial implementation of zed (ZFS Event Daemon)

zed monitors ZFS events.  When a zevent is posted, zed will run any
scripts that have been enabled for the corresponding zevent class.
Multiple scripts may be invoked for a given zevent.  The zevent
nvpairs are passed to the scripts as environment variables.

Events are processed synchronously by the single thread, and there is
no maximum timeout for script execution.  Consequently, a misbehaving
script can delay (or forever block) the processing of subsequent
zevents.  Plans are to address this in future commits.

Initial scripts have been developed to log events to syslog
and send email in response to checksum/data/io errors and
resilver.finish/scrub.finish events.  By default, email will only
be sent if the ZED_EMAIL variable is configured in zed.rc (which is
serving as a config file of sorts until a proper configuration file
is implemented).

Signed-off-by: Chris Dunlap <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Issue #2

9 files changed, 357 insertions, 0 deletions

diff --git a/cmd/zed/zed.d/all-debug.sh b/cmd/zed/zed.d/all-debug.sh
new file mode 100755
index 000000000..ae64e0a79
--- /dev/null
+++ b/cmd/zed/zed.d/all-debug.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+#
+# Log all environment variables to ZED_DEBUG_LOG.
+#
+test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
+
+# Override the default umask to restrict access to a newly-created logfile.
+umask 077
+
+# Append stdout to the logfile after obtaining an advisory lock.
+exec >> "${ZED_DEBUG_LOG:=/tmp/zed.debug.log}"
+flock -x 1
+
+printenv | sort
+echo
+
+exit 0
diff --git a/cmd/zed/zed.d/all-syslog.sh b/cmd/zed/zed.d/all-syslog.sh
new file mode 100755
index 000000000..b8bd307a1
--- /dev/null
+++ b/cmd/zed/zed.d/all-syslog.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+#
+# Log the zevent via syslog.
+#
+test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
+
+logger -t "${ZED_SYSLOG_TAG:=zed}" -p "${ZED_SYSLOG_PRIORITY:=daemon.notice}" \
+  eid="${ZEVENT_EID}" class="${ZEVENT_SUBCLASS}" \
+  "${ZEVENT_POOL:+pool=$ZEVENT_POOL}"
+
+exit 0
diff --git a/cmd/zed/zed.d/checksum-email.sh b/cmd/zed/zed.d/checksum-email.sh
new file mode 120000
index 000000000..f95bec215
--- /dev/null
+++ b/cmd/zed/zed.d/checksum-email.sh
@@ -0,0 +1 @@
+io-email.sh
\ No newline at end of file
diff --git a/cmd/zed/zed.d/data-email.sh b/cmd/zed/zed.d/data-email.sh
new file mode 100755
index 000000000..9f8316149
--- /dev/null
+++ b/cmd/zed/zed.d/data-email.sh
@@ -0,0 +1,81 @@
+#!/bin/sh
+#
+# Send email to ZED_EMAIL in response to a DATA zevent.
+# Only one message per ZED_EMAIL_INTERVAL_SECS will be sent for a given
+#   class/pool combination.  This protects against spamming the recipient
+#   should multiple events occur together in time for the same pool.
+# Exit codes:
+#   0: email sent
+#   1: email failed
+#   2: email suppressed
+#   3: missing executable
+#   4: unsupported event class
+#   5: internal error
+# State File Format:
+#   POOL:TIME_OF_LAST_EMAIL
+#
+test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
+
+test -n "${ZEVENT_POOL}" || exit 5
+test -n "${ZEVENT_SUBCLASS}" || exit 5
+
+if test "${ZEVENT_SUBCLASS}" != "data"; then \
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: unsupported event class \"${ZEVENT_SUBCLASS}\"
+  exit 4
+fi
+
+# Only send email if ZED_EMAIL has been configured.
+test -n "${ZED_EMAIL}" || exit 2
+
+# Ensure requisite executables are installed.
+if ! command -v "${MAIL:=mail}" >/dev/null 2>&1; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: "${MAIL}" not installed
+  exit 3
+fi
+
+NAME="zed.${ZEVENT_SUBCLASS}.email"
+LOCKFILE="${ZED_LOCKDIR:=/var/lock}/${NAME}.lock"
+STATEFILE="${ZED_RUNDIR:=/var/run}/${NAME}.state"
+
+# Obtain lock to ensure mutual exclusion for accessing state.
+exec 8> "${LOCKFILE}"
+flock -x 8
+
+# Query state for last time email was sent for this pool.
+TIME_NOW=`date +%s`
+TIME_LAST=`egrep "^${ZEVENT_POOL}:" "${STATEFILE}" 2>/dev/null | cut -d: -f2`
+if test -n "${TIME_LAST}"; then
+  TIME_DELTA=`expr "${TIME_NOW}" - "${TIME_LAST}"`
+  if test "${TIME_DELTA}" -lt "${ZED_EMAIL_INTERVAL_SECS:=3600}"; then
+    exit 2
+  fi
+fi
+
+"${MAIL}" -s "ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on `hostname`" \
+  "${ZED_EMAIL}" <<EOF
+A ZFS ${ZEVENT_SUBCLASS} error has been detected:
+
+   eid: ${ZEVENT_EID}
+  host: `hostname`
+  time: ${ZEVENT_TIME_STRING}
+  pool: ${ZEVENT_POOL}
+EOF
+MAIL_STATUS=$?
+
+# Update state.
+egrep -v "^${ZEVENT_POOL}:" "${STATEFILE}" 2>/dev/null > "${STATEFILE}.$$"
+echo "${ZEVENT_POOL}:${TIME_NOW}" >> "${STATEFILE}.$$"
+mv -f "${STATEFILE}.$$" "${STATEFILE}"
+
+if test "${MAIL_STATUS}" -ne 0; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: "${MAIL}" exit="${MAIL_STATUS}"
+  exit 1
+fi
+
+exit 0
diff --git a/cmd/zed/zed.d/generic-email.sh b/cmd/zed/zed.d/generic-email.sh
new file mode 100755
index 000000000..16bbdb197
--- /dev/null
+++ b/cmd/zed/zed.d/generic-email.sh
@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# Send email to ZED_EMAIL in response to a given zevent.
+# This is a generic script than can be symlinked to a file in the zed
+#   enabled-scripts directory in order to have email sent when a particular
+#   class of zevents occurs.  The symlink filename must begin with the zevent
+#   (sub)class string (eg, "probe_failure-email.sh" for the "probe_failure"
+#   subclass).  Refer to the zed(8) manpage for details.
+# Exit codes:
+#   0: email sent
+#   1: email failed
+#   2: email suppressed
+#   3: missing executable
+#
+test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
+
+# Only send email if ZED_EMAIL has been configured.
+test -n "${ZED_EMAIL}" || exit 2
+
+# Ensure requisite executables are installed.
+if ! command -v "${MAIL:=mail}" >/dev/null 2>&1; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: "${MAIL}" not installed
+  exit 3
+fi
+
+# Override the default umask to restrict access to the msgbody tmpfile.
+umask 077
+
+SUBJECT="ZFS ${ZEVENT_SUBCLASS} event"
+test -n "${ZEVENT_POOL}" && SUBJECT="${SUBJECT} for ${ZEVENT_POOL}"
+SUBJECT="${SUBJECT} on `hostname`"
+
+MSGBODY="${TMPDIR:=/tmp}/`basename \"$0\"`.$$"
+{
+  echo "A ZFS ${ZEVENT_SUBCLASS} event has been posted:"
+  echo
+  echo "   eid: ${ZEVENT_EID}"
+  echo "  host: `hostname`"
+  echo "  time: ${ZEVENT_TIME_STRING}"
+  test -n "${ZEVENT_VDEV_TYPE}" -a -n "${ZEVENT_VDEV_PATH}" && \
+    echo "  vdev: ${ZEVENT_VDEV_TYPE}:${ZEVENT_VDEV_PATH}"
+  test -n "${ZEVENT_POOL}" -a -x "${ZPOOL}" && \
+    "${ZPOOL}" status "${ZEVENT_POOL}"
+} > "${MSGBODY}"
+
+test -f "${MSGBODY}" && "${MAIL}" -s "${SUBJECT}" "${ZED_EMAIL}" < "${MSGBODY}"
+MAIL_STATUS=$?
+rm -f "${MSGBODY}"
+
+if test "${MAIL_STATUS}" -ne 0; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: "${MAIL}" exit="${MAIL_STATUS}"
+  exit 1
+fi
+
+exit 0
diff --git a/cmd/zed/zed.d/io-email.sh b/cmd/zed/zed.d/io-email.sh
new file mode 100755
index 000000000..6cfe3c7f7
--- /dev/null
+++ b/cmd/zed/zed.d/io-email.sh
@@ -0,0 +1,86 @@
+#!/bin/sh
+#
+# Send email to ZED_EMAIL in response to a CHECKSUM or IO zevent.
+# Only one message per ZED_EMAIL_INTERVAL_SECS will be sent for a given
+#   class/pool/vdev combination.  This protects against spamming the recipient
+#   should multiple events occur together in time for the same pool/device.
+# Exit codes:
+#   0: email sent
+#   1: email failed
+#   2: email suppressed
+#   3: missing executable
+#   4: unsupported event class
+#   5: internal error
+# State File Format:
+#   POOL:VDEV_PATH:TIME_OF_LAST_EMAIL
+#
+test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
+
+test -n "${ZEVENT_POOL}" || exit 5
+test -n "${ZEVENT_SUBCLASS}" || exit 5
+test -n "${ZEVENT_VDEV_PATH}" || exit 5
+
+if test "${ZEVENT_SUBCLASS}" != "checksum" \
+    -a  "${ZEVENT_SUBCLASS}" != "io"; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: unsupported event class \"${ZEVENT_SUBCLASS}\"
+  exit 4
+fi
+
+# Only send email if ZED_EMAIL has been configured.
+test -n "${ZED_EMAIL}" || exit 2
+
+# Ensure requisite executables are installed.
+if ! command -v "${MAIL:=mail}" >/dev/null 2>&1; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: "${MAIL}" not installed
+  exit 3
+fi
+
+NAME="zed.${ZEVENT_SUBCLASS}.email"
+LOCKFILE="${ZED_LOCKDIR:=/var/lock}/${NAME}.lock"
+STATEFILE="${ZED_RUNDIR:=/var/run}/${NAME}.state"
+
+# Obtain lock to ensure mutual exclusion for accessing state.
+exec 8> "${LOCKFILE}"
+flock -x 8
+
+# Query state for last time email was sent for this pool/vdev.
+TIME_NOW=`date +%s`
+TIME_LAST=`egrep "^${ZEVENT_POOL}:${ZEVENT_VDEV_PATH}:" "${STATEFILE}" \
+  2>/dev/null | cut -d: -f3`
+if test -n "${TIME_LAST}"; then
+  TIME_DELTA=`expr "${TIME_NOW}" - "${TIME_LAST}"`
+  if test "${TIME_DELTA}" -lt "${ZED_EMAIL_INTERVAL_SECS:=3600}"; then
+    exit 2
+  fi
+fi
+
+"${MAIL}" -s "ZFS ${ZEVENT_SUBCLASS} error for ${ZEVENT_POOL} on `hostname`" \
+  "${ZED_EMAIL}" <<EOF
+A ZFS ${ZEVENT_SUBCLASS} error has been detected:
+
+   eid: ${ZEVENT_EID}
+  host: `hostname`
+  time: ${ZEVENT_TIME_STRING}
+  pool: ${ZEVENT_POOL}
+  vdev: ${ZEVENT_VDEV_TYPE}:${ZEVENT_VDEV_PATH}
+EOF
+MAIL_STATUS=$?
+
+# Update state.
+egrep -v "^${ZEVENT_POOL}:${ZEVENT_VDEV_PATH}:" "${STATEFILE}" \
+  2>/dev/null > "${STATEFILE}.$$"
+echo "${ZEVENT_POOL}:${ZEVENT_VDEV_PATH}:${TIME_NOW}" >> "${STATEFILE}.$$"
+mv -f "${STATEFILE}.$$" "${STATEFILE}"
+
+if test "${MAIL_STATUS}" -ne 0; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: "${MAIL}" exit="${MAIL_STATUS}"
+  exit 1
+fi
+
+exit 0
diff --git a/cmd/zed/zed.d/resilver.finish-email.sh b/cmd/zed/zed.d/resilver.finish-email.sh
new file mode 120000
index 000000000..1afad3258
--- /dev/null
+++ b/cmd/zed/zed.d/resilver.finish-email.sh
@@ -0,0 +1 @@
+scrub.finish-email.sh
\ No newline at end of file
diff --git a/cmd/zed/zed.d/scrub.finish-email.sh b/cmd/zed/zed.d/scrub.finish-email.sh
new file mode 100755
index 000000000..b5ce3f74d
--- /dev/null
+++ b/cmd/zed/zed.d/scrub.finish-email.sh
@@ -0,0 +1,73 @@
+#!/bin/sh
+#
+# Send email to ZED_EMAIL in response to a RESILVER.FINISH or SCRUB.FINISH.
+# By default, "zpool status" output will only be included in the email for
+#   a scrub.finish zevent if the pool is not healthy; to always include its
+#   output, set ZED_EMAIL_VERBOSE=1.
+# Exit codes:
+#   0: email sent
+#   1: email failed
+#   2: email suppressed
+#   3: missing executable
+#   4: unsupported event class
+#   5: internal error
+#
+test -f "${ZED_SCRIPT_DIR}/zed.rc" && . "${ZED_SCRIPT_DIR}/zed.rc"
+
+test -n "${ZEVENT_POOL}" || exit 5
+test -n "${ZEVENT_SUBCLASS}" || exit 5
+
+if   test "${ZEVENT_SUBCLASS}" = "resilver.finish"; then
+  ACTION="resilvering"
+elif test "${ZEVENT_SUBCLASS}" = "scrub.finish"; then
+  ACTION="scrubbing"
+else
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: unsupported event class \"${ZEVENT_SUBCLASS}\"
+  exit 4
+fi
+
+# Only send email if ZED_EMAIL has been configured.
+test -n "${ZED_EMAIL}" || exit 2
+
+# Ensure requisite executables are installed.
+if ! command -v "${MAIL:=mail}" >/dev/null 2>&1; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: "${MAIL}" not installed
+  exit 3
+fi
+if ! test -x "${ZPOOL}"; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: "${ZPOOL}" not installed
+  exit 3
+fi
+
+# For scrub, suppress email if pool is healthy and verbosity is not enabled.
+if test "${ZEVENT_SUBCLASS}" = "scrub.finish"; then
+  HEALTHY=`"${ZPOOL}" status -x "${ZEVENT_POOL}" | \
+    grep "'${ZEVENT_POOL}' is healthy"`
+  test -n "${HEALTHY}" -a "${ZED_EMAIL_VERBOSE:=0}" = 0 && exit 2
+fi
+
+"${MAIL}" -s "ZFS ${ZEVENT_SUBCLASS} event for ${ZEVENT_POOL} on `hostname`" \
+  "${ZED_EMAIL}" <<EOF
+A ZFS pool has finished ${ACTION}:
+
+   eid: ${ZEVENT_EID}
+  host: `hostname`
+  time: ${ZEVENT_TIME_STRING}
+`"${ZPOOL}" status "${ZEVENT_POOL}"`
+EOF
+MAIL_STATUS=$?
+
+if test "${MAIL_STATUS}" -ne 0; then
+  logger -t "${ZED_SYSLOG_TAG:=zed}" \
+    -p "${ZED_SYSLOG_PRIORITY:=daemon.warning}" \
+    `basename "$0"`: "${MAIL}" exit="${MAIL_STATUS}"
+  exit 1
+fi
+
+exit 0
diff --git a/cmd/zed/zed.d/zed.rc b/cmd/zed/zed.d/zed.rc
new file mode 100644
index 000000000..57c969c89
--- /dev/null
+++ b/cmd/zed/zed.d/zed.rc
@@ -0,0 +1,28 @@
+# zed.rc
+
+# Absolute path to the debug output file.
+#ZED_DEBUG_LOG="/tmp/zed.debug.log"
+
+# Email address of the zpool administrator.
+#   Email will only be sent if ZED_EMAIL is defined.
+#ZED_EMAIL="root"
+
+# Email verbosity.
+#   If set to 0, suppress email if the pool is healthy.
+#   If set to 1, send email regardless of pool health.
+#ZED_EMAIL_VERBOSE=0
+
+# Minimum number of seconds between emails sent for a similar event.
+#ZED_EMAIL_INTERVAL_SECS="3600"
+
+# Default directory for zed lock files.
+#ZED_LOCKDIR="/var/lock"
+
+# Default directory for zed state files.
+#ZED_RUNDIR="/var/run"
+
+# The syslog priority (eg, specified as a "facility.level" pair).
+#ZED_SYSLOG_PRIORITY="daemon.notice"
+
+# The syslog tag for marking zed events.
+#ZED_SYSLOG_TAG="zed"