8659 static dtrace probes unavailable on non-GPL modules

ZFS tracing efforts are hampered by the inability to access zfs static
probes(probes using DTRACE_PROBE macros). The probes are available via
tracepoints for GPL modules only.  The build could be modified to
generate a function for each unique DTRACE_PROBE invocation. These could
be then accessed via kprobes.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Matt Ahrens <[email protected]>
Reviewed-by: Richard Elling <[email protected]>
Signed-off-by: Brad Lewis <[email protected]>
Closes #8659 
Closes #8663 

18 files changed, 303 insertions, 57 deletions

diff --git a/include/spl/sys/sysmacros.h b/include/spl/sys/sysmacros.h
index e11eaece5..0753864d1 100644
--- a/include/spl/sys/sysmacros.h
+++ b/include/spl/sys/sysmacros.h
@@ -114,32 +114,6 @@
 #define	PAGESHIFT			PAGE_SHIFT
 #endif
 
-/* Dtrace probes do not exist in the linux kernel */
-#ifdef DTRACE_PROBE
-#undef  DTRACE_PROBE
-#endif  /* DTRACE_PROBE */
-#define	DTRACE_PROBE(a)					((void)0)
-
-#ifdef DTRACE_PROBE1
-#undef  DTRACE_PROBE1
-#endif  /* DTRACE_PROBE1 */
-#define	DTRACE_PROBE1(a, b, c)				((void)0)
-
-#ifdef DTRACE_PROBE2
-#undef  DTRACE_PROBE2
-#endif  /* DTRACE_PROBE2 */
-#define	DTRACE_PROBE2(a, b, c, d, e)			((void)0)
-
-#ifdef DTRACE_PROBE3
-#undef  DTRACE_PROBE3
-#endif  /* DTRACE_PROBE3 */
-#define	DTRACE_PROBE3(a, b, c, d, e, f, g)		((void)0)
-
-#ifdef DTRACE_PROBE4
-#undef  DTRACE_PROBE4
-#endif  /* DTRACE_PROBE4 */
-#define	DTRACE_PROBE4(a, b, c, d, e, f, g, h, i)	((void)0)
-
 /* Missing globals */
 extern char spl_gitrev[64];
 extern unsigned long spl_hostid;
diff --git a/include/sys/Makefile.am b/include/sys/Makefile.am
index 368f3fb2a..af45eb3bc 100644
--- a/include/sys/Makefile.am
+++ b/include/sys/Makefile.am
@@ -78,6 +78,7 @@ COMMON_H = \
 	$(top_srcdir)/include/sys/trace_dmu.h \
 	$(top_srcdir)/include/sys/trace_dnode.h \
 	$(top_srcdir)/include/sys/trace_multilist.h \
+	$(top_srcdir)/include/sys/trace_rrwlock.h \
 	$(top_srcdir)/include/sys/trace_txg.h \
 	$(top_srcdir)/include/sys/trace_vdev.h \
 	$(top_srcdir)/include/sys/trace_zil.h \
diff --git a/include/sys/trace.h b/include/sys/trace.h
index f32ba529e..48497cc35 100644
--- a/include/sys/trace.h
+++ b/include/sys/trace.h
@@ -19,7 +19,35 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+
+/*
+ * Calls to DTRACE_PROBE* are mapped to standard Linux kernel trace points
+ * when they are available(when HAVE_DECLARE_EVENT_CLASS is defined).  The
+ * tracepoint event class definitions are found in the general tracing
+ * header file: include/sys/trace_*.h.  See include/sys/trace_vdev.h for
+ * a good example.
+ *
+ * If tracepoints are not available, stub functions are generated which can
+ * be traced using kprobes.  In this case, the DEFINE_DTRACE_PROBE* macros
+ * are used to provide the stub functions and also the prototypes for
+ * those functions.  The mechanism to do this relies on DEFINE_DTRACE_PROBE
+ * macros defined in the general tracing headers(see trace_vdev.h) and
+ * CREATE_TRACE_POINTS being defined only in module/zfs/trace.c.  When ZFS
+ * source files include the general tracing headers, e.g.
+ * module/zfs/vdev_removal.c including trace_vdev.h, DTRACE_PROBE calls
+ * are mapped to stub functions calls and prototypes for those calls are
+ * declared via DEFINE_DTRACE_PROBE*.  Only module/zfs/trace.c defines
+ * CREATE_TRACE_POINTS.  That is follwed by includes of all the general
+ * tracing headers thereby defining all stub functions in one place via
+ * the DEFINE_DTRACE_PROBE macros.
+ *
+ * When adding new DTRACE_PROBEs to zfs source, both a tracepoint event
+ * class defintition and a DEFINE_DTRACE_PROBE definition are needed to
+ * avoid undefined function errors.
+ */
+
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -39,21 +67,21 @@
 #undef _SYS_TRACE_DBGMSG_INDIRECT
 
 /*
- * Redefine the DTRACE_PROBE* functions to use Linux tracepoints
+ * DTRACE_PROBE with 0 arguments is not currently available with
+ *  tracepoint events
  */
-#undef DTRACE_PROBE1
+#define	DTRACE_PROBE(name) \
+	((void)0)
+
 #define	DTRACE_PROBE1(name, t1, arg1) \
 	trace_zfs_##name((arg1))
 
-#undef DTRACE_PROBE2
 #define	DTRACE_PROBE2(name, t1, arg1, t2, arg2) \
 	trace_zfs_##name((arg1), (arg2))
 
-#undef DTRACE_PROBE3
 #define	DTRACE_PROBE3(name, t1, arg1, t2, arg2, t3, arg3) \
 	trace_zfs_##name((arg1), (arg2), (arg3))
 
-#undef DTRACE_PROBE4
 #define	DTRACE_PROBE4(name, t1, arg1, t2, arg2, t3, arg3, t4, arg4) \
 	trace_zfs_##name((arg1), (arg2), (arg3), (arg4))
 
@@ -65,4 +93,95 @@
 #define	TRACE_INCLUDE_FILE trace
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else /* HAVE_DECLARE_EVENT_CLASS */
+
+#define	DTRACE_PROBE(name) \
+	trace_zfs_##name()
+
+#define	DTRACE_PROBE1(name, t1, arg1) \
+	trace_zfs_##name((uintptr_t)(arg1))
+
+#define	DTRACE_PROBE2(name, t1, arg1, t2, arg2) \
+	trace_zfs_##name((uintptr_t)(arg1), (uintptr_t)(arg2))
+
+#define	DTRACE_PROBE3(name, t1, arg1, t2, arg2, t3, arg3) \
+	trace_zfs_##name((uintptr_t)(arg1), (uintptr_t)(arg2), \
+	(uintptr_t)(arg3))
+
+#define	DTRACE_PROBE4(name, t1, arg1, t2, arg2, t3, arg3, t4, arg4) \
+	trace_zfs_##name((uintptr_t)(arg1), (uintptr_t)(arg2), \
+	(uintptr_t)(arg3), (uintptr_t)(arg4))
+
+#if defined(CREATE_TRACE_POINTS)
+
+#define	FUNC_DTRACE_PROBE(name)					\
+noinline void trace_zfs_##name(void) { }			\
+EXPORT_SYMBOL(trace_zfs_##name)
+
+#define	FUNC_DTRACE_PROBE1(name)				\
+noinline void trace_zfs_##name(uintptr_t arg1) { }		\
+EXPORT_SYMBOL(trace_zfs_##name)
+
+#define	FUNC_DTRACE_PROBE2(name)				\
+noinline void trace_zfs_##name(uintptr_t arg1,			\
+    uintptr_t arg2) { }						\
+EXPORT_SYMBOL(trace_zfs_##name)
+
+#define	FUNC_DTRACE_PROBE3(name)				\
+noinline void trace_zfs_##name(uintptr_t arg1,			\
+    uintptr_t arg2, uintptr_t arg3) { }				\
+EXPORT_SYMBOL(trace_zfs_##name)
+
+#define	FUNC_DTRACE_PROBE4(name)				\
+noinline void trace_zfs_##name(uintptr_t arg1,			\
+    uintptr_t arg2, uintptr_t arg3, uintptr_t arg4) { }		\
+EXPORT_SYMBOL(trace_zfs_##name)
+
+#undef	DEFINE_DTRACE_PROBE
+#define	DEFINE_DTRACE_PROBE(name)	FUNC_DTRACE_PROBE(name)
+
+#undef	DEFINE_DTRACE_PROBE1
+#define	DEFINE_DTRACE_PROBE1(name)	FUNC_DTRACE_PROBE1(name)
+
+#undef	DEFINE_DTRACE_PROBE2
+#define	DEFINE_DTRACE_PROBE2(name)	FUNC_DTRACE_PROBE2(name)
+
+#undef	DEFINE_DTRACE_PROBE3
+#define	DEFINE_DTRACE_PROBE3(name)	FUNC_DTRACE_PROBE3(name)
+
+#undef	DEFINE_DTRACE_PROBE4
+#define	DEFINE_DTRACE_PROBE4(name)	FUNC_DTRACE_PROBE4(name)
+
+#else /* CREATE_TRACE_POINTS */
+
+#define	PROTO_DTRACE_PROBE(name)				\
+	noinline void trace_zfs_##name(void)
+#define	PROTO_DTRACE_PROBE1(name)				\
+	noinline void trace_zfs_##name(uintptr_t)
+#define	PROTO_DTRACE_PROBE2(name)				\
+	noinline void trace_zfs_##name(uintptr_t, uintptr_t)
+#define	PROTO_DTRACE_PROBE3(name)				\
+	noinline void trace_zfs_##name(uintptr_t, uintptr_t,	\
+	uintptr_t)
+#define	PROTO_DTRACE_PROBE4(name)				\
+	noinline void trace_zfs_##name(uintptr_t, uintptr_t,	\
+	uintptr_t, uintptr_t)
+
+#define	DEFINE_DTRACE_PROBE(name)	PROTO_DTRACE_PROBE(name)
+#define	DEFINE_DTRACE_PROBE1(name)	PROTO_DTRACE_PROBE1(name)
+#define	DEFINE_DTRACE_PROBE2(name)	PROTO_DTRACE_PROBE2(name)
+#define	DEFINE_DTRACE_PROBE3(name)	PROTO_DTRACE_PROBE3(name)
+#define	DEFINE_DTRACE_PROBE4(name)	PROTO_DTRACE_PROBE4(name)
+
+#endif /* CREATE_TRACE_POINTS */
+
+/*
+ * The sys/trace_dbgmsg.h header defines tracepoint events for
+ * dprintf(), dbgmsg(), and SET_ERROR().
+ */
+#define	_SYS_TRACE_DBGMSG_INDIRECT
+#include <sys/trace_dbgmsg.h>
+#undef _SYS_TRACE_DBGMSG_INDIRECT
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_acl.h b/include/sys/trace_acl.h
index 610bbe29c..083560952 100644
--- a/include/sys/trace_acl.h
+++ b/include/sys/trace_acl.h
@@ -19,7 +19,8 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -153,4 +154,11 @@ DEFINE_ACE_EVENT(zfs_zfs__ace__allows);
 #define	TRACE_INCLUDE_FILE trace_acl
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE3(zfs__ace__denies);
+DEFINE_DTRACE_PROBE3(zfs__ace__allows);
+DEFINE_DTRACE_PROBE(zfs__fastpath__execute__access__miss);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_arc.h b/include/sys/trace_arc.h
index c40b58e32..5ce5b38a3 100644
--- a/include/sys/trace_arc.h
+++ b/include/sys/trace_arc.h
@@ -21,7 +21,8 @@
 
 #include <sys/list.h>
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -361,4 +362,22 @@ DEFINE_L2ARC_EVICT_EVENT(zfs_l2arc__evict);
 #define	TRACE_INCLUDE_FILE trace_arc
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE1(arc__hit);
+DEFINE_DTRACE_PROBE1(arc__evict);
+DEFINE_DTRACE_PROBE1(arc__delete);
+DEFINE_DTRACE_PROBE1(new_state__mru);
+DEFINE_DTRACE_PROBE1(new_state__mfu);
+DEFINE_DTRACE_PROBE1(arc__async__upgrade__sync);
+DEFINE_DTRACE_PROBE1(arc__demand__hit__predictive__prefetch);
+DEFINE_DTRACE_PROBE1(l2arc__hit);
+DEFINE_DTRACE_PROBE1(l2arc__miss);
+DEFINE_DTRACE_PROBE2(l2arc__read);
+DEFINE_DTRACE_PROBE2(l2arc__write);
+DEFINE_DTRACE_PROBE2(l2arc__iodone);
+DEFINE_DTRACE_PROBE4(arc__miss);
+DEFINE_DTRACE_PROBE4(l2arc__evict);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_dbgmsg.h b/include/sys/trace_dbgmsg.h
index a4aab1e63..b2ef529ed 100644
--- a/include/sys/trace_dbgmsg.h
+++ b/include/sys/trace_dbgmsg.h
@@ -19,6 +19,9 @@
  * CDDL HEADER END
  */
 
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
+
 /* Do not include this file directly. Please use <sys/trace.h> instead. */
 #ifndef _SYS_TRACE_DBGMSG_INDIRECT
 #error "trace_dbgmsg.h included directly"
@@ -63,3 +66,10 @@ DEFINE_EVENT(zfs_dprintf_class, name, \
 	TP_ARGS(msg))
 /* END CSTYLED */
 DEFINE_DPRINTF_EVENT(zfs_zfs__dprintf);
+
+#else
+
+DEFINE_DTRACE_PROBE1(zfs__dprintf);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_dbuf.h b/include/sys/trace_dbuf.h
index e97b61137..fb12e2854 100644
--- a/include/sys/trace_dbuf.h
+++ b/include/sys/trace_dbuf.h
@@ -19,7 +19,8 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -142,4 +143,10 @@ DEFINE_DBUF_EVICT_ONE_EVENT(zfs_dbuf__evict__one);
 #define	TRACE_INCLUDE_FILE trace_dbuf
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE2(blocked__read);
+DEFINE_DTRACE_PROBE2(dbuf__evict__one);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_dmu.h b/include/sys/trace_dmu.h
index 24e57f514..3c64a370f 100644
--- a/include/sys/trace_dmu.h
+++ b/include/sys/trace_dmu.h
@@ -19,7 +19,8 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -126,4 +127,10 @@ DEFINE_FREE_LONG_RANGE_EVENT(zfs_free__long__range);
 #define	TRACE_INCLUDE_FILE trace_dmu
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE3(delay__mintime);
+DEFINE_DTRACE_PROBE3(free__long__range);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_dnode.h b/include/sys/trace_dnode.h
index 7196a497d..27ad6cba1 100644
--- a/include/sys/trace_dnode.h
+++ b/include/sys/trace_dnode.h
@@ -19,7 +19,8 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -120,4 +121,9 @@ DEFINE_DNODE_MOVE_EVENT(zfs_dnode__move);
 #define	TRACE_INCLUDE_FILE trace_dnode
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE3(dnode__move);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_multilist.h b/include/sys/trace_multilist.h
index ed0b38a3f..fe68d5296 100644
--- a/include/sys/trace_multilist.h
+++ b/include/sys/trace_multilist.h
@@ -19,7 +19,8 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -79,4 +80,10 @@ DEFINE_MULTILIST_INSERT_REMOVE_EVENT(zfs_multilist__remove);
 #define	TRACE_INCLUDE_FILE trace_multilist
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE3(multilist__insert);
+DEFINE_DTRACE_PROBE3(multilist__remove);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_rrwlock.h b/include/sys/trace_rrwlock.h
new file mode 100644
index 000000000..4c74d6257
--- /dev/null
+++ b/include/sys/trace_rrwlock.h
@@ -0,0 +1,31 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
+
+#else
+
+DEFINE_DTRACE_PROBE(zfs__rrwfastpath__rdmiss);
+DEFINE_DTRACE_PROBE(zfs__rrwfastpath__exitmiss);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_txg.h b/include/sys/trace_txg.h
index f85c3f9ef..23d5d358b 100644
--- a/include/sys/trace_txg.h
+++ b/include/sys/trace_txg.h
@@ -19,7 +19,8 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -75,4 +76,14 @@ DEFINE_TXG_EVENT(zfs_txg__quiesced);
 #define	TRACE_INCLUDE_FILE trace_txg
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE2(dsl_pool_sync__done);
+DEFINE_DTRACE_PROBE2(txg__quiescing);
+DEFINE_DTRACE_PROBE2(txg__opened);
+DEFINE_DTRACE_PROBE2(txg__syncing);
+DEFINE_DTRACE_PROBE2(txg__synced);
+DEFINE_DTRACE_PROBE2(txg__quiesced);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_vdev.h b/include/sys/trace_vdev.h
index d7af44c25..289aca69e 100644
--- a/include/sys/trace_vdev.h
+++ b/include/sys/trace_vdev.h
@@ -19,7 +19,14 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
+
+/*
+ * If tracepoints are available define dtrace_probe events for vdev
+ * related probes.  Definitions in usr/include/trace.h will map
+ * DTRACE_PROBE* calls to tracepoints.
+ */
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -116,4 +123,18 @@ DEFINE_REMOVE_FREE_EVENT_TXG(zfs_remove__free__inflight);
 #define	TRACE_INCLUDE_FILE trace_vdev
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+/*
+ * When tracepoints are not available, a DEFINE_DTRACE_PROBE* macro is
+ * needed for each DTRACE_PROBE.  These will be used to generate stub
+ * tracing functions and protoypes for those functions.  See
+ * include/sys/trace.h.
+ */
+
+DEFINE_DTRACE_PROBE3(remove__free__synced);
+DEFINE_DTRACE_PROBE3(remove__free__unvisited);
+DEFINE_DTRACE_PROBE4(remove__free__inflight);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_zil.h b/include/sys/trace_zil.h
index ff16c8686..526846e66 100644
--- a/include/sys/trace_zil.h
+++ b/include/sys/trace_zil.h
@@ -19,7 +19,8 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -218,4 +219,11 @@ DEFINE_ZIL_COMMIT_IO_ERROR_EVENT(zfs_zil__commit__io__error);
 #define	TRACE_INCLUDE_FILE trace_zil
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE2(zil__process__commit__itx);
+DEFINE_DTRACE_PROBE2(zil__process__normal__itx);
+DEFINE_DTRACE_PROBE2(zil__commit__io__error);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_zio.h b/include/sys/trace_zio.h
index af589b9df..8655e245c 100644
--- a/include/sys/trace_zio.h
+++ b/include/sys/trace_zio.h
@@ -21,7 +21,8 @@
 
 #include <sys/list.h>
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -86,4 +87,11 @@ TRACE_EVENT(zfs_zio__delay__skip,
 #define	TRACE_INCLUDE_FILE trace_zio
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE2(zio__delay__miss);
+DEFINE_DTRACE_PROBE3(zio__delay__hit);
+DEFINE_DTRACE_PROBE1(zio__delay__skip);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/include/sys/trace_zrlock.h b/include/sys/trace_zrlock.h
index fa330f2c1..23f9577ba 100644
--- a/include/sys/trace_zrlock.h
+++ b/include/sys/trace_zrlock.h
@@ -19,7 +19,8 @@
  * CDDL HEADER END
  */
 
-#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS)
+#if defined(_KERNEL)
+#if defined(HAVE_DECLARE_EVENT_CLASS)
 
 #undef TRACE_SYSTEM
 #define	TRACE_SYSTEM zfs
@@ -85,4 +86,9 @@ DEFINE_ZRLOCK_EVENT(zfs_zrlock__reentry);
 #define	TRACE_INCLUDE_FILE trace_zrlock
 #include <trace/define_trace.h>
 
-#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */
+#else
+
+DEFINE_DTRACE_PROBE3(zrlock__reentry);
+
+#endif /* HAVE_DECLARE_EVENT_CLASS */
+#endif /* _KERNEL */
diff --git a/module/zfs/rrwlock.c b/module/zfs/rrwlock.c
index 582b40a58..32d45f674 100644
--- a/module/zfs/rrwlock.c
+++ b/module/zfs/rrwlock.c
@@ -28,6 +28,7 @@
 
 #include <sys/refcount.h>
 #include <sys/rrwlock.h>
+#include <sys/trace_rrwlock.h>
 
 /*
  * This file contains the implementation of a re-entrant read
diff --git a/module/zfs/trace.c b/module/zfs/trace.c
index eb6efe841..fe503776b 100644
--- a/module/zfs/trace.c
+++ b/module/zfs/trace.c
@@ -18,9 +18,10 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Each Linux tracepoints subsystem must define CREATE_TRACE_POINTS in one
- * (and only one) C file, so this dummy file exists for that purpose.
+ * Each DTRACE_PROBE must define its trace point in one (and only one)
+ * source file, so this dummy file exists for that purpose.
  */
 
 #include <sys/multilist.h>
@@ -45,6 +46,7 @@
 #include <sys/trace_dmu.h>
 #include <sys/trace_dnode.h>
 #include <sys/trace_multilist.h>
+#include <sys/trace_rrwlock.h>
 #include <sys/trace_txg.h>
 #include <sys/trace_vdev.h>
 #include <sys/trace_zil.h>