OpenZFS 6531 - Provide mechanism to artificially limit disk performance

Reviewed by: Paul Dagnelie <[email protected]>
Reviewed by: Matthew Ahrens <[email protected]>
Reviewed by: George Wilson <[email protected]>
Approved by: Dan McDonald <[email protected]>
Ported by: Tony Hutter <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>

OpenZFS-issue: https://www.illumos.org/issues/6531
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/97e8130

Porting notes:
- Added new IO delay tracepoints, and moved common ZIO tracepoint macros
  to a new trace_common.h file.
- Used zio_delay_taskq() in place of OpenZFS's timeout_generic() function.
- Updated zinject man page
- Updated zpool_scrub test files

6 files changed, 207 insertions, 81 deletions

diff --git a/include/sys/Makefile.am b/include/sys/Makefile.am
index 98d7ad6fb..89c314279 100644
--- a/include/sys/Makefile.am
+++ b/include/sys/Makefile.am
@@ -53,6 +53,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/trace.h \
 	$(top_srcdir)/include/sys/trace_acl.h \
 	$(top_srcdir)/include/sys/trace_arc.h \
+	$(top_srcdir)/include/sys/trace_common.h \
 	$(top_srcdir)/include/sys/trace_dbgmsg.h \
 	$(top_srcdir)/include/sys/trace_dbuf.h \
 	$(top_srcdir)/include/sys/trace_dmu.h \
@@ -60,6 +61,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/trace_multilist.h \
 	$(top_srcdir)/include/sys/trace_txg.h \
 	$(top_srcdir)/include/sys/trace_zil.h \
+	$(top_srcdir)/include/sys/trace_zio.h \
 	$(top_srcdir)/include/sys/trace_zrlock.h \
 	$(top_srcdir)/include/sys/txg.h \
 	$(top_srcdir)/include/sys/txg_impl.h \
diff --git a/include/sys/trace_arc.h b/include/sys/trace_arc.h
index 55dbdf19b..0fca639e1 100644
--- a/include/sys/trace_arc.h
+++ b/include/sys/trace_arc.h
@@ -34,6 +34,7 @@
 
 #include <linux/tracepoint.h>
 #include <sys/types.h>
+#include <sys/trace_common.h> /* For ZIO macros */
 
 /*
  * Generic support for one argument tracepoints of the form:
@@ -115,86 +116,6 @@ DEFINE_ARC_BUF_HDR_EVENT(zfs_l2arc__miss);
  *     zio_t *, ...);
  */
 
-#define	ZIO_TP_STRUCT_ENTRY						\
-		__field(zio_type_t,		zio_type)		\
-		__field(int,			zio_cmd)		\
-		__field(zio_priority_t,		zio_priority)		\
-		__field(uint64_t,		zio_size)		\
-		__field(uint64_t,		zio_orig_size)		\
-		__field(uint64_t,		zio_offset)		\
-		__field(hrtime_t,		zio_timestamp)		\
-		__field(hrtime_t,		zio_delta)		\
-		__field(uint64_t,		zio_delay)		\
-		__field(enum zio_flag,		zio_flags)		\
-		__field(enum zio_stage,		zio_stage)		\
-		__field(enum zio_stage,		zio_pipeline)		\
-		__field(enum zio_flag,		zio_orig_flags)		\
-		__field(enum zio_stage,		zio_orig_stage)		\
-		__field(enum zio_stage,		zio_orig_pipeline)	\
-		__field(uint8_t,		zio_reexecute)		\
-		__field(uint64_t,		zio_txg)		\
-		__field(int,			zio_error)		\
-		__field(uint64_t,		zio_ena)		\
-									\
-		__field(enum zio_checksum,	zp_checksum)		\
-		__field(enum zio_compress,	zp_compress)		\
-		__field(dmu_object_type_t,	zp_type)		\
-		__field(uint8_t,		zp_level)		\
-		__field(uint8_t,		zp_copies)		\
-		__field(boolean_t,		zp_dedup)		\
-		__field(boolean_t,		zp_dedup_verify)	\
-		__field(boolean_t,		zp_nopwrite)
-
-#define	ZIO_TP_FAST_ASSIGN						    \
-		__entry->zio_type		= zio->io_type;		    \
-		__entry->zio_cmd		= zio->io_cmd;		    \
-		__entry->zio_priority		= zio->io_priority;	    \
-		__entry->zio_size		= zio->io_size;		    \
-		__entry->zio_orig_size		= zio->io_orig_size;	    \
-		__entry->zio_offset		= zio->io_offset;	    \
-		__entry->zio_timestamp		= zio->io_timestamp;	    \
-		__entry->zio_delta		= zio->io_delta;	    \
-		__entry->zio_delay		= zio->io_delay;	    \
-		__entry->zio_flags		= zio->io_flags;	    \
-		__entry->zio_stage		= zio->io_stage;	    \
-		__entry->zio_pipeline		= zio->io_pipeline;	    \
-		__entry->zio_orig_flags		= zio->io_orig_flags;	    \
-		__entry->zio_orig_stage		= zio->io_orig_stage;	    \
-		__entry->zio_orig_pipeline	= zio->io_orig_pipeline;    \
-		__entry->zio_reexecute		= zio->io_reexecute;	    \
-		__entry->zio_txg		= zio->io_txg;		    \
-		__entry->zio_error		= zio->io_error;	    \
-		__entry->zio_ena		= zio->io_ena;		    \
-									    \
-		__entry->zp_checksum		= zio->io_prop.zp_checksum; \
-		__entry->zp_compress		= zio->io_prop.zp_compress; \
-		__entry->zp_type		= zio->io_prop.zp_type;	    \
-		__entry->zp_level		= zio->io_prop.zp_level;    \
-		__entry->zp_copies		= zio->io_prop.zp_copies;   \
-		__entry->zp_dedup		= zio->io_prop.zp_dedup;    \
-		__entry->zp_nopwrite		= zio->io_prop.zp_nopwrite; \
-		__entry->zp_dedup_verify	= zio->io_prop.zp_dedup_verify;
-
-#define	ZIO_TP_PRINTK_FMT						\
-	"zio { type %u cmd %i prio %u size %llu orig_size %llu "	\
-	"offset %llu timestamp %llu delta %llu delay %llu "		\
-	"flags 0x%x stage 0x%x pipeline 0x%x orig_flags 0x%x "		\
-	"orig_stage 0x%x orig_pipeline 0x%x reexecute %u "		\
-	"txg %llu error %d ena %llu prop { checksum %u compress %u "	\
-	"type %u level %u copies %u dedup %u dedup_verify %u nopwrite %u } }"
-
-#define	ZIO_TP_PRINTK_ARGS						\
-	__entry->zio_type, __entry->zio_cmd, __entry->zio_priority,	\
-	__entry->zio_size, __entry->zio_orig_size, __entry->zio_offset,	\
-	__entry->zio_timestamp, __entry->zio_delta, __entry->zio_delay,	\
-	__entry->zio_flags, __entry->zio_stage, __entry->zio_pipeline,	\
-	__entry->zio_orig_flags, __entry->zio_orig_stage,		\
-	__entry->zio_orig_pipeline, __entry->zio_reexecute,		\
-	__entry->zio_txg, __entry->zio_error, __entry->zio_ena,		\
-	__entry->zp_checksum, __entry->zp_compress, __entry->zp_type,	\
-	__entry->zp_level, __entry->zp_copies, __entry->zp_dedup,	\
-	__entry->zp_dedup_verify, __entry->zp_nopwrite
-
 DECLARE_EVENT_CLASS(zfs_l2arc_rw_class,
 	TP_PROTO(vdev_t *vd, zio_t *zio),
 	TP_ARGS(vd, zio),
diff --git a/include/sys/trace_common.h b/include/sys/trace_common.h
new file mode 100644
index 000000000..6922d1a18
--- /dev/null
+++ b/include/sys/trace_common.h
@@ -0,0 +1,112 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * This file contains commonly used trace macros.  Feel free to add and use
+ * them in your tracepoint headers.
+ */
+
+#ifndef	_SYS_TRACE_COMMON_H
+#define	_SYS_TRACE_COMMON_H
+#include <linux/tracepoint.h>
+
+/* ZIO macros */
+#define	ZIO_TP_STRUCT_ENTRY						\
+		__field(zio_type_t,		zio_type)		\
+		__field(int,			zio_cmd)		\
+		__field(zio_priority_t,		zio_priority)		\
+		__field(uint64_t,		zio_size)		\
+		__field(uint64_t,		zio_orig_size)		\
+		__field(uint64_t,		zio_offset)		\
+		__field(hrtime_t,		zio_timestamp)		\
+		__field(hrtime_t,		zio_delta)		\
+		__field(uint64_t,		zio_delay)		\
+		__field(enum zio_flag,		zio_flags)		\
+		__field(enum zio_stage,		zio_stage)		\
+		__field(enum zio_stage,		zio_pipeline)		\
+		__field(enum zio_flag,		zio_orig_flags)		\
+		__field(enum zio_stage,		zio_orig_stage)		\
+		__field(enum zio_stage,		zio_orig_pipeline)	\
+		__field(uint8_t,		zio_reexecute)		\
+		__field(uint64_t,		zio_txg)		\
+		__field(int,			zio_error)		\
+		__field(uint64_t,		zio_ena)		\
+									\
+		__field(enum zio_checksum,	zp_checksum)		\
+		__field(enum zio_compress,	zp_compress)		\
+		__field(dmu_object_type_t,	zp_type)		\
+		__field(uint8_t,		zp_level)		\
+		__field(uint8_t,		zp_copies)		\
+		__field(boolean_t,		zp_dedup)		\
+		__field(boolean_t,		zp_dedup_verify)	\
+		__field(boolean_t,		zp_nopwrite)
+
+#define	ZIO_TP_FAST_ASSIGN						    \
+		__entry->zio_type		= zio->io_type;		    \
+		__entry->zio_cmd		= zio->io_cmd;		    \
+		__entry->zio_priority		= zio->io_priority;	    \
+		__entry->zio_size		= zio->io_size;		    \
+		__entry->zio_orig_size		= zio->io_orig_size;	    \
+		__entry->zio_offset		= zio->io_offset;	    \
+		__entry->zio_timestamp		= zio->io_timestamp;	    \
+		__entry->zio_delta		= zio->io_delta;	    \
+		__entry->zio_delay		= zio->io_delay;	    \
+		__entry->zio_flags		= zio->io_flags;	    \
+		__entry->zio_stage		= zio->io_stage;	    \
+		__entry->zio_pipeline		= zio->io_pipeline;	    \
+		__entry->zio_orig_flags		= zio->io_orig_flags;	    \
+		__entry->zio_orig_stage		= zio->io_orig_stage;	    \
+		__entry->zio_orig_pipeline	= zio->io_orig_pipeline;    \
+		__entry->zio_reexecute		= zio->io_reexecute;	    \
+		__entry->zio_txg		= zio->io_txg;		    \
+		__entry->zio_error		= zio->io_error;	    \
+		__entry->zio_ena		= zio->io_ena;		    \
+									    \
+		__entry->zp_checksum		= zio->io_prop.zp_checksum; \
+		__entry->zp_compress		= zio->io_prop.zp_compress; \
+		__entry->zp_type		= zio->io_prop.zp_type;	    \
+		__entry->zp_level		= zio->io_prop.zp_level;    \
+		__entry->zp_copies		= zio->io_prop.zp_copies;   \
+		__entry->zp_dedup		= zio->io_prop.zp_dedup;    \
+		__entry->zp_nopwrite		= zio->io_prop.zp_nopwrite; \
+		__entry->zp_dedup_verify	= zio->io_prop.zp_dedup_verify;
+
+#define	ZIO_TP_PRINTK_FMT						\
+	"zio { type %u cmd %i prio %u size %llu orig_size %llu "	\
+	"offset %llu timestamp %llu delta %llu delay %llu "		\
+	"flags 0x%x stage 0x%x pipeline 0x%x orig_flags 0x%x "		\
+	"orig_stage 0x%x orig_pipeline 0x%x reexecute %u "		\
+	"txg %llu error %d ena %llu prop { checksum %u compress %u "	\
+	"type %u level %u copies %u dedup %u dedup_verify %u nopwrite %u } }"
+
+#define	ZIO_TP_PRINTK_ARGS						\
+	__entry->zio_type, __entry->zio_cmd, __entry->zio_priority,	\
+	__entry->zio_size, __entry->zio_orig_size, __entry->zio_offset,	\
+	__entry->zio_timestamp, __entry->zio_delta, __entry->zio_delay,	\
+	__entry->zio_flags, __entry->zio_stage, __entry->zio_pipeline,	\
+	__entry->zio_orig_flags, __entry->zio_orig_stage,		\
+	__entry->zio_orig_pipeline, __entry->zio_reexecute,		\
+	__entry->zio_txg, __entry->zio_error, __entry->zio_ena,		\
+	__entry->zp_checksum, __entry->zp_compress, __entry->zp_type,	\
+	__entry->zp_level, __entry->zp_copies, __entry->zp_dedup,	\
+	__entry->zp_dedup_verify, __entry->zp_nopwrite
+
+#endif /* _SYS_TRACE_COMMON_H */
diff --git a/include/sys/trace_zio.h b/include/sys/trace_zio.h
new file mode 100644
index 000000000..6db6cefe2
--- /dev/null
+++ b/include/sys/trace_zio.h
@@ -0,0 +1,87 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#include <sys/list.h>
+
+#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+
+#undef TRACE_SYSTEM
+#define	TRACE_SYSTEM zfs
+
+#undef TRACE_SYSTEM_VAR
+#define	TRACE_SYSTEM_VAR zfs_zio
+
+#if !defined(_TRACE_ZIO_H) || defined(TRACE_HEADER_MULTI_READ)
+#define	_TRACE_ZIO_H
+
+#include <linux/tracepoint.h>
+#include <sys/types.h>
+#include <sys/trace_common.h> /* For ZIO macros */
+
+TRACE_EVENT(zfs_zio__delay__miss,
+	TP_PROTO(zio_t *zio, hrtime_t now),
+	TP_ARGS(zio, now),
+	TP_STRUCT__entry(
+	    ZIO_TP_STRUCT_ENTRY
+	    __field(hrtime_t, now)
+	),
+	TP_fast_assign(
+	    ZIO_TP_FAST_ASSIGN
+	    __entry->now = now;
+	),
+	TP_printk("now %llu " ZIO_TP_PRINTK_FMT, __entry->now,
+	    ZIO_TP_PRINTK_ARGS)
+);
+
+TRACE_EVENT(zfs_zio__delay__hit,
+	TP_PROTO(zio_t *zio, hrtime_t now, hrtime_t diff),
+	TP_ARGS(zio, now, diff),
+	TP_STRUCT__entry(
+	    ZIO_TP_STRUCT_ENTRY
+	    __field(hrtime_t, now)
+	    __field(hrtime_t, diff)
+	),
+	TP_fast_assign(
+	    ZIO_TP_FAST_ASSIGN
+	    __entry->now = now;
+	    __entry->diff = diff;
+	),
+	TP_printk("now %llu diff %llu " ZIO_TP_PRINTK_FMT, __entry->now,
+	    __entry->diff, ZIO_TP_PRINTK_ARGS)
+);
+
+TRACE_EVENT(zfs_zio__delay__skip,
+	TP_PROTO(zio_t *zio),
+	TP_ARGS(zio),
+	TP_STRUCT__entry(ZIO_TP_STRUCT_ENTRY),
+	TP_fast_assign(ZIO_TP_FAST_ASSIGN),
+	TP_printk(ZIO_TP_PRINTK_FMT, ZIO_TP_PRINTK_ARGS)
+);
+
+#endif /* _TRACE_ZIO_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define	TRACE_INCLUDE_PATH sys
+#define	TRACE_INCLUDE_FILE trace_zio
+#include <trace/define_trace.h>
+
+#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
diff --git a/include/sys/zfs_ioctl.h b/include/sys/zfs_ioctl.h
index 601a9a70c..ac70f690e 100644
--- a/include/sys/zfs_ioctl.h
+++ b/include/sys/zfs_ioctl.h
@@ -287,6 +287,7 @@ typedef struct zinject_record {
 	uint32_t	zi_iotype;
 	int32_t		zi_duration;
 	uint64_t	zi_timer;
+	uint64_t	zi_nlanes;
 	uint32_t	zi_cmd;
 	uint32_t	zi_pad;
 } zinject_record_t;
diff --git a/include/sys/zio.h b/include/sys/zio.h
index 9790b4a90..300a5f5ab 100644
--- a/include/sys/zio.h
+++ b/include/sys/zio.h
@@ -420,6 +420,7 @@ struct zio {
 
 	uint64_t	io_offset;
 	hrtime_t	io_timestamp;	/* submitted at */
+	hrtime_t	io_target_timestamp;
 	hrtime_t	io_delta;	/* vdev queue service delta */
 	hrtime_t	io_delay;	/* Device access time (disk or */
 					/* file). */
@@ -510,6 +511,8 @@ extern int zio_wait(zio_t *zio);
 extern void zio_nowait(zio_t *zio);
 extern void zio_execute(zio_t *zio);
 extern void zio_interrupt(zio_t *zio);
+extern void zio_delay_init(zio_t *zio);
+extern void zio_delay_interrupt(zio_t *zio);
 
 extern zio_t *zio_walk_parents(zio_t *cio);
 extern zio_t *zio_walk_children(zio_t *pio);
@@ -572,7 +575,7 @@ extern int zio_handle_fault_injection(zio_t *zio, int error);
 extern int zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error);
 extern int zio_handle_label_injection(zio_t *zio, int error);
 extern void zio_handle_ignored_writes(zio_t *zio);
-extern uint64_t zio_handle_io_delay(zio_t *zio);
+extern hrtime_t zio_handle_io_delay(zio_t *zio);
 
 /*
  * Checksum ereport functions