Removing ZERO_PAGE abd_alloc_zero_scatter

For MIPS architectures on Linux the ZERO_PAGE macro references
empty_zero_page, which is exported as a GPL symbol. The call to
ZERO_PAGE in abd_alloc_zero_scatter has been removed and a single
zero'd page is now allocated for each of the pages in abd_zero_scatter
in the kernel ABD code path.

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Brian Atkinson <[email protected]>
Closes #10428 

3 files changed, 23 insertions, 13 deletions

diff --git a/include/sys/abd.h b/include/sys/abd.h
index 3b490005c..dad048af5 100644
--- a/include/sys/abd.h
+++ b/include/sys/abd.h
@@ -42,7 +42,6 @@ typedef int abd_iter_func_t(void *buf, size_t len, void *priv);
 typedef int abd_iter_func2_t(void *bufa, void *bufb, size_t len, void *priv);
 
 extern int zfs_abd_scatter_enabled;
-extern abd_t *abd_zero_scatter;
 
 /*
  * Allocations and deallocations
diff --git a/include/sys/abd_impl.h b/include/sys/abd_impl.h
index 23eec9ad5..b1fa87b42 100644
--- a/include/sys/abd_impl.h
+++ b/include/sys/abd_impl.h
@@ -92,6 +92,8 @@ struct abd_iter {
 	struct scatterlist *iter_sg;	/* current sg */
 };
 
+extern abd_t *abd_zero_scatter;
+
 abd_t *abd_gang_get_offset(abd_t *, size_t *);
 
 /*
diff --git a/module/os/linux/zfs/abd_os.c b/module/os/linux/zfs/abd_os.c
index 9ad40d69c..d39f0c6d5 100644
--- a/module/os/linux/zfs/abd_os.c
+++ b/module/os/linux/zfs/abd_os.c
@@ -167,6 +167,13 @@ int zfs_abd_scatter_min_size = 512 * 3;
  */
 abd_t *abd_zero_scatter = NULL;
 
+struct page;
+/*
+ * abd_zero_page we will be an allocated zero'd PAGESIZE buffer, which is
+ * assigned to set each of the pages of abd_zero_scatter.
+ */
+static struct page *abd_zero_page = NULL;
+
 static kmem_cache_t *abd_cache = NULL;
 static kstat_t *abd_ksp;
 
@@ -439,8 +446,7 @@ abd_free_chunks(abd_t *abd)
 
 /*
  * Allocate scatter ABD of size SPA_MAXBLOCKSIZE, where each page in
- * the scatterlist will be set to ZERO_PAGE(0). ZERO_PAGE(0) returns
- * a global shared page that is always zero'd out.
+ * the scatterlist will be set to the zero'd out buffer abd_zero_page.
  */
 static void
 abd_alloc_zero_scatter(void)
@@ -448,9 +454,16 @@ abd_alloc_zero_scatter(void)
 	struct scatterlist *sg = NULL;
 	struct sg_table table;
 	gfp_t gfp = __GFP_NOWARN | GFP_NOIO;
+	gfp_t gfp_zero_page = gfp | __GFP_ZERO;
 	int nr_pages = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
 	int i = 0;
 
+	while ((abd_zero_page = __page_cache_alloc(gfp_zero_page)) == NULL) {
+		ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
+		schedule_timeout_interruptible(1);
+	}
+	abd_mark_zfs_page(abd_zero_page);
+
 	while (sg_alloc_table(&table, nr_pages, gfp)) {
 		ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
 		schedule_timeout_interruptible(1);
@@ -468,7 +481,7 @@ abd_alloc_zero_scatter(void)
 	zfs_refcount_create(&abd_zero_scatter->abd_children);
 
 	abd_for_each_sg(abd_zero_scatter, sg, nr_pages, i) {
-		sg_set_page(sg, ZERO_PAGE(0), PAGESIZE, 0);
+		sg_set_page(sg, abd_zero_page, PAGESIZE, 0);
 	}
 
 	ABDSTAT_BUMP(abdstat_scatter_cnt);
@@ -478,14 +491,6 @@ abd_alloc_zero_scatter(void)
 
 #else /* _KERNEL */
 
-struct page;
-
-/*
- * In user space abd_zero_page we will be an allocated zero'd PAGESIZE
- * buffer, which is assigned to set each of the pages of abd_zero_scatter.
- */
-static struct page *abd_zero_page = NULL;
-
 #ifndef PAGE_SHIFT
 #define	PAGE_SHIFT (highbit64(PAGESIZE)-1)
 #endif
@@ -680,7 +685,11 @@ abd_free_zero_scatter(void)
 	abd_free_sg_table(abd_zero_scatter);
 	abd_free_struct(abd_zero_scatter);
 	abd_zero_scatter = NULL;
-#if !defined(_KERNEL)
+	ASSERT3P(abd_zero_page, !=, NULL);
+#if defined(_KERNEL)
+	abd_unmark_zfs_page(abd_zero_page);
+	__free_page(abd_zero_page);
+#else
 	umem_free(abd_zero_page, PAGESIZE);
 #endif /* _KERNEL */
 }