Add zpool_influxdb command

A zpool_influxdb command is introduced to ease the collection
of zpool statistics into the InfluxDB time-series database.
Examples are given on how to integrate with the telegraf
statistics aggregator, a companion to influxdb.

Finally, a grafana dashboard template is included to show
how pool latency distributions can be visualized in a
ZFS + telegraf + influxdb  + grafana environment.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Ryan Moeller <[email protected]>
Signed-off-by: Richard Elling <[email protected]>
Closes #10786 

6 files changed, 137 insertions, 1 deletions

diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg
index 5a507b94a..21d1950e7 100644
--- a/tests/zfs-tests/include/commands.cfg
+++ b/tests/zfs-tests/include/commands.cfg
@@ -188,7 +188,8 @@ export ZFS_FILES='zdb
     zgenhostid
     zstream
     zstreamdump
-    zfs_ids_to_path'
+    zfs_ids_to_path
+    zpool_influxdb'
 
 export ZFSTEST_FILES='badsend
     btree_test
diff --git a/tests/zfs-tests/tests/functional/Makefile.am b/tests/zfs-tests/tests/functional/Makefile.am
index 6438898e4..3a5b7b0b9 100644
--- a/tests/zfs-tests/tests/functional/Makefile.am
+++ b/tests/zfs-tests/tests/functional/Makefile.am
@@ -82,6 +82,7 @@ SUBDIRS = \
 	vdev_zaps \
 	write_dirs \
 	xattr \
+	zpool_influxdb \
 	zvol
 
 if BUILD_LINUX
diff --git a/tests/zfs-tests/tests/functional/zpool_influxdb/Makefile.am b/tests/zfs-tests/tests/functional/zpool_influxdb/Makefile.am
new file mode 100644
index 000000000..36d08a41a
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/zpool_influxdb/Makefile.am
@@ -0,0 +1,5 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/zpool_influxdb
+dist_pkgdata_SCRIPTS = \
+	setup.ksh \
+	cleanup.ksh \
+	zpool_influxdb.ksh
diff --git a/tests/zfs-tests/tests/functional/zpool_influxdb/cleanup.ksh b/tests/zfs-tests/tests/functional/zpool_influxdb/cleanup.ksh
new file mode 100755
index 000000000..a8cd2e4b6
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/zpool_influxdb/cleanup.ksh
@@ -0,0 +1,29 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at
+# https://opensource.org/licenses/CDDL-1.0
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2020 Richard Elling
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+default_cleanup
diff --git a/tests/zfs-tests/tests/functional/zpool_influxdb/setup.ksh b/tests/zfs-tests/tests/functional/zpool_influxdb/setup.ksh
new file mode 100755
index 000000000..43f2c8c20
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/zpool_influxdb/setup.ksh
@@ -0,0 +1,29 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at
+# https://opensource.org/licenses/CDDL-1.0
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2020 Richard Elling
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+default_raidz_setup $DISKS
diff --git a/tests/zfs-tests/tests/functional/zpool_influxdb/zpool_influxdb.ksh b/tests/zfs-tests/tests/functional/zpool_influxdb/zpool_influxdb.ksh
new file mode 100755
index 000000000..495a4a38b
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/zpool_influxdb/zpool_influxdb.ksh
@@ -0,0 +1,71 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at
+# https://opensource.org/licenses/CDDL-1.0
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2020 Richard Elling
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+typeset tmpfile=$TEST_BASE_DIR/zpool_influxdb.out.$$
+function cleanup
+{
+	if [[ -f $tmpfile ]]; then
+		rm -f $tmpfile
+	fi
+}
+log_onexit cleanup
+
+log_assert "zpool_influxdb gathers statistics"
+
+if ! is_global_zone ; then
+	TESTPOOL=${TESTPOOL%%/*}
+fi
+
+function check_for
+{
+    grep "^${1}," $tmpfile >/dev/null 2>/dev/null
+    if [ $? -ne 0 ]; then
+        log_fail "cannot find stats for $1"
+    fi
+}
+
+# by default, all stats and histograms for all pools
+log_must zpool_influxdb > $tmpfile
+
+STATS="
+zpool_io_size
+zpool_latency
+zpool_stats
+zpool_vdev_queue
+zpool_vdev_stats
+"
+for stat in $STATS; do
+    check_for $stat
+done
+
+# scan stats aren't expected to be there until after a scan has started
+zpool scrub $TESTPOOL
+zpool_influxdb > $tmpfile
+check_for zpool_scan_stats
+
+log_pass "zpool_influxdb gathers statistics"