Add gang ABD child to parent gang ABD

By design a gang ABD can not have another gang ABD as a child. This is
to make sure the logical offset in a gang ABD is consistent with the
individual ABDS it contains as children. If a gang ABD is added as a
child of a gang ABD we will add the individual children of the gang ABD
to the parent gang ABD. This allows for a consistent view of offsets
within the parent gang ABD.

Reviewed-by: Mark Maybee <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Brian Atkinson <[email protected]>
Closes #10430 

1 files changed, 55 insertions, 1 deletions

diff --git a/module/zfs/abd.c b/module/zfs/abd.c
index 3d1ed63cf..6018a42ca 100644
--- a/module/zfs/abd.c
+++ b/module/zfs/abd.c
@@ -139,12 +139,15 @@ abd_verify(abd_t *abd)
 	if (abd_is_linear(abd)) {
 		ASSERT3P(ABD_LINEAR_BUF(abd), !=, NULL);
 	} else if (abd_is_gang(abd)) {
+		uint_t child_sizes = 0;
 		for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain);
 		    cabd != NULL;
 		    cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
 			ASSERT(list_link_active(&cabd->abd_gang_link));
+			child_sizes += cabd->abd_size;
 			abd_verify(cabd);
 		}
+		ASSERT3U(abd->abd_size, ==, child_sizes);
 	} else {
 		abd_verify_scatter(abd);
 	}
@@ -378,16 +381,67 @@ abd_alloc_gang_abd(void)
 }
 
 /*
+ * Add a child gang ABD to a parent gang ABDs chained list.
+ */
+static void
+abd_gang_add_gang(abd_t *pabd, abd_t *cabd, boolean_t free_on_free)
+{
+	ASSERT(abd_is_gang(pabd));
+	ASSERT(abd_is_gang(cabd));
+
+	if (free_on_free) {
+		/*
+		 * If the parent is responsible for freeing the child gang
+		 * ABD we will just splice the childs children ABD list to
+		 * the parents list and immediately free the child gang ABD
+		 * struct. The parent gang ABDs children from the child gang
+		 * will retain all the free_on_free settings after being
+		 * added to the parents list.
+		 */
+		pabd->abd_size += cabd->abd_size;
+		list_move_tail(&ABD_GANG(pabd).abd_gang_chain,
+		    &ABD_GANG(cabd).abd_gang_chain);
+		ASSERT(list_is_empty(&ABD_GANG(cabd).abd_gang_chain));
+		abd_verify(pabd);
+		abd_free_struct(cabd);
+	} else {
+		for (abd_t *child = list_head(&ABD_GANG(cabd).abd_gang_chain);
+		    child != NULL;
+		    child = list_next(&ABD_GANG(cabd).abd_gang_chain, child)) {
+			/*
+			 * We always pass B_FALSE for free_on_free as it is the
+			 * original child gang ABDs responsibilty to determine
+			 * if any of its child ABDs should be free'd on the call
+			 * to abd_free().
+			 */
+			abd_gang_add(pabd, child, B_FALSE);
+		}
+		abd_verify(pabd);
+	}
+}
+
+/*
  * Add a child ABD to a gang ABD's chained list.
  */
 void
 abd_gang_add(abd_t *pabd, abd_t *cabd, boolean_t free_on_free)
 {
 	ASSERT(abd_is_gang(pabd));
-	ASSERT(!abd_is_gang(cabd));
 	abd_t *child_abd = NULL;
 
 	/*
+	 * If the child being added is a gang ABD, we will add the
+	 * childs ABDs to the parent gang ABD. This alllows us to account
+	 * for the offset correctly in the parent gang ABD.
+	 */
+	if (abd_is_gang(cabd)) {
+		ASSERT(!list_link_active(&cabd->abd_gang_link));
+		ASSERT(!list_is_empty(&ABD_GANG(cabd).abd_gang_chain));
+		return (abd_gang_add_gang(pabd, cabd, free_on_free));
+	}
+	ASSERT(!abd_is_gang(cabd));
+
+	/*
 	 * In order to verify that an ABD is not already part of
 	 * another gang ABD, we must lock the child ABD's abd_mtx
 	 * to check its abd_gang_link status. We unlock the abd_mtx