Gang ABD Type

Adding the gang ABD type, which allows for linear and scatter ABDs to
be chained together into a single ABD.

This can be used to avoid doing memory copies to/from ABDs. An example
of this can be found in vdev_queue.c in the vdev_queue_aggregate()
function.

Reviewed-by: Matthew Ahrens <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Co-authored-by: Brian <[email protected]>
Co-authored-by: Mark Maybee <[email protected]>
Signed-off-by: Brian Atkinson <[email protected]>
Closes #10069 

2 files changed, 18 insertions, 3 deletions

diff --git a/include/sys/abd.h b/include/sys/abd.h
index df4234f3c..be37b4a81 100644
--- a/include/sys/abd.h
+++ b/include/sys/abd.h
@@ -42,6 +42,7 @@ typedef int abd_iter_func_t(void *buf, size_t len, void *private);
 typedef int abd_iter_func2_t(void *bufa, void *bufb, size_t len, void *private);
 
 extern int zfs_abd_scatter_enabled;
+extern abd_t *abd_zero_scatter;
 
 /*
  * Allocations and deallocations
@@ -49,13 +50,16 @@ extern int zfs_abd_scatter_enabled;
 
 abd_t *abd_alloc(size_t, boolean_t);
 abd_t *abd_alloc_linear(size_t, boolean_t);
+abd_t *abd_alloc_gang_abd(void);
 abd_t *abd_alloc_for_io(size_t, boolean_t);
 abd_t *abd_alloc_sametype(abd_t *, size_t);
+void abd_gang_add(abd_t *, abd_t *, boolean_t);
 void abd_free(abd_t *);
+void abd_put(abd_t *);
 abd_t *abd_get_offset(abd_t *, size_t);
 abd_t *abd_get_offset_size(abd_t *, size_t, size_t);
+abd_t *abd_get_zeros(size_t);
 abd_t *abd_get_from_buf(void *, size_t);
-void abd_put(abd_t *);
 
 /*
  * Conversion to and from a normal buffer
@@ -132,6 +136,7 @@ abd_zero(abd_t *abd, size_t size)
  * ABD type check functions
  */
 boolean_t abd_is_linear(abd_t *);
+boolean_t abd_is_gang(abd_t *);
 boolean_t abd_is_linear_page(abd_t *);
 
 /*
@@ -146,8 +151,7 @@ void abd_fini(void);
  * Linux ABD bio functions
  */
 #if defined(__linux__) && defined(_KERNEL)
-unsigned int abd_scatter_bio_map_off(struct bio *, abd_t *, unsigned int,
-    size_t);
+unsigned int abd_bio_map_off(struct bio *, abd_t *, unsigned int, size_t);
 unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
 #endif
 
diff --git a/include/sys/abd_impl.h b/include/sys/abd_impl.h
index 5aee772b1..23eec9ad5 100644
--- a/include/sys/abd_impl.h
+++ b/include/sys/abd_impl.h
@@ -39,6 +39,9 @@ typedef enum abd_flags {
 	ABD_FLAG_MULTI_ZONE  	= 1 << 3, /* pages split over memory zones */
 	ABD_FLAG_MULTI_CHUNK 	= 1 << 4, /* pages split over multiple chunks */
 	ABD_FLAG_LINEAR_PAGE 	= 1 << 5, /* linear but allocd from page */
+	ABD_FLAG_GANG		= 1 << 6, /* mult ABDs chained together */
+	ABD_FLAG_GANG_FREE	= 1 << 7, /* gang ABD is responsible for mem */
+	ABD_FLAG_ZEROS		= 1 << 8, /* ABD for zero-filled buffer */
 } abd_flags_t;
 
 typedef enum abd_stats_op {
@@ -49,8 +52,10 @@ typedef enum abd_stats_op {
 struct abd {
 	abd_flags_t	abd_flags;
 	uint_t		abd_size;	/* excludes scattered abd_offset */
+	list_node_t	abd_gang_link;
 	struct abd	*abd_parent;
 	zfs_refcount_t	abd_children;
+	kmutex_t	abd_mtx;
 	union {
 		struct abd_scatter {
 			uint_t		abd_offset;
@@ -66,6 +71,9 @@ struct abd {
 			void		*abd_buf;
 			struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
 		} abd_linear;
+		struct abd_gang {
+			list_t abd_gang_chain;
+		} abd_gang;
 	} abd_u;
 };
 
@@ -84,6 +92,8 @@ struct abd_iter {
 	struct scatterlist *iter_sg;	/* current sg */
 };
 
+abd_t *abd_gang_get_offset(abd_t *, size_t *);
+
 /*
  * OS specific functions
  */
@@ -116,6 +126,7 @@ void abd_iter_unmap(struct abd_iter *);
 
 #define	ABD_SCATTER(abd)	(abd->abd_u.abd_scatter)
 #define	ABD_LINEAR_BUF(abd)	(abd->abd_u.abd_linear.abd_buf)
+#define	ABD_GANG(abd)		(abd->abd_u.abd_gang)
 
 #if defined(_KERNEL)
 #if defined(__FreeBSD__)