Add zpool status -s (slow I/Os) and -p (parseable)

This patch adds a new slow I/Os (-s) column to zpool status to show the
number of VDEV slow I/Os. This is the number of I/Os that didn't
complete in zio_slow_io_ms milliseconds. It also adds a new parsable
(-p) flag to display exact values.

 	NAME         STATE     READ WRITE CKSUM  SLOW
 	testpool     ONLINE       0     0     0     -
	  mirror-0   ONLINE       0     0     0     -
 	    loop0    ONLINE       0     0     0    20
 	    loop1    ONLINE       0     0     0     0

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed by: Matthew Ahrens <[email protected]>
Signed-off-by: Tony Hutter <[email protected]>
Closes #7756
Closes #6885

3 files changed, 80 insertions, 2 deletions

diff --git a/tests/zfs-tests/tests/functional/fault/Makefile.am b/tests/zfs-tests/tests/functional/fault/Makefile.am
index ec07db57d..5c68ea26f 100644
--- a/tests/zfs-tests/tests/functional/fault/Makefile.am
+++ b/tests/zfs-tests/tests/functional/fault/Makefile.am
@@ -11,7 +11,8 @@ dist_pkgdata_SCRIPTS = \
 	auto_spare_shared.ksh \
 	decrypt_fault.ksh \
 	decompress_fault.ksh \
-	scrub_after_resilver.ksh
+	scrub_after_resilver.ksh \
+	zpool_status_-s.ksh
 
 dist_pkgdata_DATA = \
 	fault.cfg
diff --git a/tests/zfs-tests/tests/functional/fault/zpool_status_-s.ksh b/tests/zfs-tests/tests/functional/fault/zpool_status_-s.ksh
new file mode 100755
index 000000000..b6a3e71fd
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/fault/zpool_status_-s.ksh
@@ -0,0 +1,77 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+#
+
+# DESCRIPTION:
+#	Verify zpool status -s (slow IOs) works
+#
+# STRATEGY:
+#	1. Create a file
+#	2. Inject slow IOs into the pool
+#	3. Verify we can see the slow IOs with "zpool status -s".
+#	4. Verify we can see delay events.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/include/zpool_script.shlib
+
+DISK=${DISKS%% *}
+
+verify_runnable "both"
+
+log_must zpool create $TESTPOOL mirror ${DISKS}
+
+function cleanup
+{
+	log_must zinject -c all
+	log_must set_tunable64 zio_slow_io_ms $OLD_SLOW_IO
+	log_must set_tunable64 zfs_slow_io_events_per_second $OLD_SLOW_IO_EVENTS
+	log_must destroy_pool $TESTPOOL
+}
+
+log_onexit cleanup
+
+log_must zpool events -c
+
+# Mark any IOs greater than 10ms as slow IOs
+OLD_SLOW_IO=$(get_tunable zio_slow_io_ms)
+OLD_SLOW_IO_EVENTS=$(get_tunable zfs_slow_io_events_per_second)
+log_must set_tunable64 zio_slow_io_ms 10
+log_must set_tunable64 zfs_slow_io_events_per_second 1000
+
+# Create 20ms IOs
+log_must zinject -d $DISK -D20:100 $TESTPOOL
+log_must mkfile 1048576 /$TESTPOOL/testfile
+log_must zpool sync $TESTPOOL
+
+log_must zinject -c all
+SLOW_IOS=$(zpool status -sp | grep "$DISK" | awk '{print $6}')
+DELAY_EVENTS=$(zpool events | grep delay | wc -l)
+
+if [ $SLOW_IOS -gt 0 ] && [ $DELAY_EVENTS -gt 0 ] ; then
+	log_pass "Correctly saw $SLOW_IOS slow IOs and $DELAY_EVENTS delay events"
+else
+	log_fail "Only saw $SLOW_IOS slow IOs and $DELAY_EVENTS delay events"
+fi
diff --git a/tests/zfs-tests/tests/perf/perf.shlib b/tests/zfs-tests/tests/perf/perf.shlib
index 3802c6aec..7165df759 100644
--- a/tests/zfs-tests/tests/perf/perf.shlib
+++ b/tests/zfs-tests/tests/perf/perf.shlib
@@ -410,7 +410,7 @@ function get_system_config
 		    zfs_vdev_async_write_max_active \
 		    zfs_vdev_sync_read_max_active \
 		    zfs_vdev_sync_write_max_active \
-		    zio_delay_max
+		    zio_slow_io_ms
 		do
 			if [ "$tunable" != "zfs_arc_max" ]
 			then