allow callers to allocate and provide the abd_t struct

The `abd_get_offset_*()` routines create an abd_t that references
another abd_t, and doesn't allocate any pages/buffers of its own.  In
some workloads, these routines may be called frequently, to create many
abd_t's representing small pieces of a single large abd_t.  In
particular, the upcoming RAIDZ Expansion project makes heavy use of
these routines.

This commit adds the ability for the caller to allocate and provide the
abd_t struct to a variant of `abd_get_offset_*()`.  This eliminates the
cost of allocating the abd_t and performing the accounting associated
with it (`abdstat_struct_size`).  The RAIDZ/DRAID code uses this for
the `rc_abd`, which references the zio's abd.  The upcoming RAIDZ
Expansion project will leverage this infrastructure to increase
performance of reads post-expansion by around 50%.

Additionally, some of the interfaces around creating and destroying
abd_t's are cleaned up.  Most significantly, the distinction between
`abd_put()` and `abd_free()` is eliminated; all types of abd_t's are
now disposed of with `abd_free()`.

Reviewed-by: Brian Atkinson <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Matthew Ahrens <[email protected]>
Issue #8853 
Closes #11439 

3 files changed, 71 insertions, 51 deletions

diff --git a/include/sys/abd.h b/include/sys/abd.h
index 735a13147..4311076d8 100644
--- a/include/sys/abd.h
+++ b/include/sys/abd.h
@@ -35,8 +35,46 @@
 extern "C" {
 #endif
 
-struct abd; /* forward declaration */
-typedef struct abd abd_t;
+typedef enum abd_flags {
+	ABD_FLAG_LINEAR		= 1 << 0, /* is buffer linear (or scattered)? */
+	ABD_FLAG_OWNER		= 1 << 1, /* does it own its data buffers? */
+	ABD_FLAG_META		= 1 << 2, /* does this represent FS metadata? */
+	ABD_FLAG_MULTI_ZONE  	= 1 << 3, /* pages split over memory zones */
+	ABD_FLAG_MULTI_CHUNK 	= 1 << 4, /* pages split over multiple chunks */
+	ABD_FLAG_LINEAR_PAGE 	= 1 << 5, /* linear but allocd from page */
+	ABD_FLAG_GANG		= 1 << 6, /* mult ABDs chained together */
+	ABD_FLAG_GANG_FREE	= 1 << 7, /* gang ABD is responsible for mem */
+	ABD_FLAG_ZEROS		= 1 << 8, /* ABD for zero-filled buffer */
+	ABD_FLAG_ALLOCD		= 1 << 9, /* we allocated the abd_t */
+} abd_flags_t;
+
+typedef struct abd {
+	abd_flags_t	abd_flags;
+	uint_t		abd_size;	/* excludes scattered abd_offset */
+	list_node_t	abd_gang_link;
+	struct abd	*abd_parent;
+	zfs_refcount_t	abd_children;
+	kmutex_t	abd_mtx;
+	union {
+		struct abd_scatter {
+			uint_t		abd_offset;
+#if defined(__FreeBSD__) && defined(_KERNEL)
+			uint_t  abd_chunk_size;
+			void    *abd_chunks[1]; /* actually variable-length */
+#else
+			uint_t		abd_nents;
+			struct scatterlist *abd_sgl;
+#endif
+		} abd_scatter;
+		struct abd_linear {
+			void		*abd_buf;
+			struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
+		} abd_linear;
+		struct abd_gang {
+			list_t abd_gang_chain;
+		} abd_gang;
+	} abd_u;
+} abd_t;
 
 typedef int abd_iter_func_t(void *buf, size_t len, void *priv);
 typedef int abd_iter_func2_t(void *bufa, void *bufb, size_t len, void *priv);
@@ -49,14 +87,14 @@ extern int zfs_abd_scatter_enabled;
 
 abd_t *abd_alloc(size_t, boolean_t);
 abd_t *abd_alloc_linear(size_t, boolean_t);
-abd_t *abd_alloc_gang_abd(void);
+abd_t *abd_alloc_gang(void);
 abd_t *abd_alloc_for_io(size_t, boolean_t);
 abd_t *abd_alloc_sametype(abd_t *, size_t);
 void abd_gang_add(abd_t *, abd_t *, boolean_t);
 void abd_free(abd_t *);
-void abd_put(abd_t *);
 abd_t *abd_get_offset(abd_t *, size_t);
 abd_t *abd_get_offset_size(abd_t *, size_t, size_t);
+abd_t *abd_get_offset_struct(abd_t *, abd_t *, size_t, size_t);
 abd_t *abd_get_zeros(size_t);
 abd_t *abd_get_from_buf(void *, size_t);
 void abd_cache_reap_now(void);
@@ -87,7 +125,6 @@ int abd_cmp(abd_t *, abd_t *);
 int abd_cmp_buf_off(abd_t *, const void *, size_t, size_t);
 void abd_zero_off(abd_t *, size_t, size_t);
 void abd_verify(abd_t *);
-uint_t abd_get_size(abd_t *);
 
 void abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
 	ssize_t csize, ssize_t dsize, const unsigned parity,
@@ -135,9 +172,29 @@ abd_zero(abd_t *abd, size_t size)
 /*
  * ABD type check functions
  */
-boolean_t abd_is_linear(abd_t *);
-boolean_t abd_is_gang(abd_t *);
-boolean_t abd_is_linear_page(abd_t *);
+static inline boolean_t
+abd_is_linear(abd_t *abd)
+{
+	return ((abd->abd_flags & ABD_FLAG_LINEAR) != 0);
+}
+
+static inline boolean_t
+abd_is_linear_page(abd_t *abd)
+{
+	return ((abd->abd_flags & ABD_FLAG_LINEAR_PAGE) != 0);
+}
+
+static inline boolean_t
+abd_is_gang(abd_t *abd)
+{
+	return ((abd->abd_flags & ABD_FLAG_GANG) != 0);
+}
+
+static inline uint_t
+abd_get_size(abd_t *abd)
+{
+	return (abd->abd_size);
+}
 
 /*
  * Module lifecycle
diff --git a/include/sys/abd_impl.h b/include/sys/abd_impl.h
index b1fa87b42..435a8dc6d 100644
--- a/include/sys/abd_impl.h
+++ b/include/sys/abd_impl.h
@@ -32,51 +32,11 @@
 extern "C" {
 #endif
 
-typedef enum abd_flags {
-	ABD_FLAG_LINEAR		= 1 << 0, /* is buffer linear (or scattered)? */
-	ABD_FLAG_OWNER		= 1 << 1, /* does it own its data buffers? */
-	ABD_FLAG_META		= 1 << 2, /* does this represent FS metadata? */
-	ABD_FLAG_MULTI_ZONE  	= 1 << 3, /* pages split over memory zones */
-	ABD_FLAG_MULTI_CHUNK 	= 1 << 4, /* pages split over multiple chunks */
-	ABD_FLAG_LINEAR_PAGE 	= 1 << 5, /* linear but allocd from page */
-	ABD_FLAG_GANG		= 1 << 6, /* mult ABDs chained together */
-	ABD_FLAG_GANG_FREE	= 1 << 7, /* gang ABD is responsible for mem */
-	ABD_FLAG_ZEROS		= 1 << 8, /* ABD for zero-filled buffer */
-} abd_flags_t;
-
 typedef enum abd_stats_op {
 	ABDSTAT_INCR, /* Increase abdstat values */
 	ABDSTAT_DECR  /* Decrease abdstat values */
 } abd_stats_op_t;
 
-struct abd {
-	abd_flags_t	abd_flags;
-	uint_t		abd_size;	/* excludes scattered abd_offset */
-	list_node_t	abd_gang_link;
-	struct abd	*abd_parent;
-	zfs_refcount_t	abd_children;
-	kmutex_t	abd_mtx;
-	union {
-		struct abd_scatter {
-			uint_t		abd_offset;
-#if defined(__FreeBSD__) && defined(_KERNEL)
-			uint_t  abd_chunk_size;
-			void    *abd_chunks[];
-#else
-			uint_t		abd_nents;
-			struct scatterlist *abd_sgl;
-#endif
-		} abd_scatter;
-		struct abd_linear {
-			void		*abd_buf;
-			struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
-		} abd_linear;
-		struct abd_gang {
-			list_t abd_gang_chain;
-		} abd_gang;
-	} abd_u;
-};
-
 struct scatterlist; /* forward declaration */
 
 struct abd_iter {
@@ -95,14 +55,16 @@ struct abd_iter {
 extern abd_t *abd_zero_scatter;
 
 abd_t *abd_gang_get_offset(abd_t *, size_t *);
+abd_t *abd_alloc_struct(size_t);
+void abd_free_struct(abd_t *);
 
 /*
  * OS specific functions
  */
 
-abd_t *abd_alloc_struct(size_t);
-abd_t *abd_get_offset_scatter(abd_t *, size_t);
-void abd_free_struct(abd_t *);
+abd_t *abd_alloc_struct_impl(size_t);
+abd_t *abd_get_offset_scatter(abd_t *, abd_t *, size_t);
+void abd_free_struct_impl(abd_t *);
 void abd_alloc_chunks(abd_t *, size_t);
 void abd_free_chunks(abd_t *);
 boolean_t abd_size_alloc_linear(size_t);
diff --git a/include/sys/vdev_raidz_impl.h b/include/sys/vdev_raidz_impl.h
index 38d4f9e0b..c869b8b4d 100644
--- a/include/sys/vdev_raidz_impl.h
+++ b/include/sys/vdev_raidz_impl.h
@@ -106,6 +106,7 @@ typedef struct raidz_col {
 	uint64_t rc_devidx;		/* child device index for I/O */
 	uint64_t rc_offset;		/* device offset */
 	uint64_t rc_size;		/* I/O size */
+	abd_t rc_abdstruct;		/* rc_abd probably points here */
 	abd_t *rc_abd;			/* I/O data */
 	void *rc_orig_data;		/* pre-reconstruction */
 	abd_t *rc_gdata;		/* used to store the "good" version */