compress: change compression providers API to use ABDs

This commit changes the provider compress and decompress API to take ABD
pointers instead of buffer pointers for both data source and
destination. It then updates all providers to match.

This doesn't actually change the providers to do chunked compression,
just changes the API to allow such an update in the future. Helper
macros are added to easily adapt the ABD functions to their buffer-based
implementations.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Signed-off-by: Rob Norris <[email protected]>

11 files changed, 120 insertions, 54 deletions

diff --git a/cmd/zstream/zstream_decompress.c b/cmd/zstream/zstream_decompress.c
index f5f66080d..f8f439d46 100644
--- a/cmd/zstream/zstream_decompress.c
+++ b/cmd/zstream/zstream_decompress.c
@@ -288,10 +288,12 @@ zstream_do_decompress(int argc, char *argv[])
 			char *lzbuf = safe_calloc(payload_size);
 			(void) sfread(lzbuf, payload_size, stdin);
 
-			abd_t sabd;
+			abd_t sabd, dabd;
 			abd_get_from_buf_struct(&sabd, lzbuf, payload_size);
-			int err = zio_decompress_data(c, &sabd, buf,
+			abd_get_from_buf_struct(&dabd, buf, payload_size);
+			int err = zio_decompress_data(c, &sabd, &dabd,
 			    payload_size, payload_size, NULL);
+			abd_free(&dabd);
 			abd_free(&sabd);
 
 			if (err != 0) {
diff --git a/include/sys/zio_compress.h b/include/sys/zio_compress.h
index 56376fdd1..d41b5dfd4 100644
--- a/include/sys/zio_compress.h
+++ b/include/sys/zio_compress.h
@@ -22,7 +22,7 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2019, Allan Jude
- * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, 2024, Klara, Inc.
  * Use is subject to license terms.
  * Copyright (c) 2015, 2016 by Delphix. All rights reserved.
  */
@@ -122,13 +122,13 @@ enum zio_zstd_levels {
 struct zio_prop;
 
 /* Common signature for all zio compress functions. */
-typedef size_t zio_compress_func_t(void *src, void *dst,
+typedef size_t zio_compress_func_t(abd_t *src, abd_t *dst,
     size_t s_len, size_t d_len, int);
 /* Common signature for all zio decompress functions. */
-typedef int zio_decompress_func_t(void *src, void *dst,
+typedef int zio_decompress_func_t(abd_t *src, abd_t *dst,
     size_t s_len, size_t d_len, int);
 /* Common signature for all zio decompress and get level functions. */
-typedef int zio_decompresslevel_func_t(void *src, void *dst,
+typedef int zio_decompresslevel_func_t(abd_t *src, abd_t *dst,
     size_t s_len, size_t d_len, uint8_t *level);
 
 /*
@@ -153,21 +153,21 @@ extern void lz4_fini(void);
 /*
  * Compression routines.
  */
-extern size_t zfs_lzjb_compress(void *src, void *dst, size_t s_len,
+extern size_t zfs_lzjb_compress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int level);
-extern int zfs_lzjb_decompress(void *src, void *dst, size_t s_len,
+extern int zfs_lzjb_decompress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int level);
-extern size_t zfs_gzip_compress(void *src, void *dst, size_t s_len,
+extern size_t zfs_gzip_compress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int level);
-extern int zfs_gzip_decompress(void *src, void *dst, size_t s_len,
+extern int zfs_gzip_decompress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int level);
-extern size_t zfs_zle_compress(void *src, void *dst, size_t s_len,
+extern size_t zfs_zle_compress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int level);
-extern int zfs_zle_decompress(void *src, void *dst, size_t s_len,
+extern int zfs_zle_decompress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int level);
-extern size_t zfs_lz4_compress(void *src, void *dst, size_t s_len,
+extern size_t zfs_lz4_compress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int level);
-extern int zfs_lz4_decompress(void *src, void *dst, size_t s_len,
+extern int zfs_lz4_decompress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int level);
 
 /*
@@ -179,6 +179,40 @@ extern int zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
     size_t s_len, size_t d_len, uint8_t *level);
 extern int zio_compress_to_feature(enum zio_compress comp);
 
+#define	ZFS_COMPRESS_WRAP_DECL(name)					\
+size_t									\
+name(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int n)		\
+{									\
+	void *s_buf = abd_borrow_buf_copy(src, s_len);			\
+	void *d_buf = abd_borrow_buf(dst, d_len);			\
+	size_t c_len = name##_buf(s_buf, d_buf, s_len, d_len, n);	\
+	abd_return_buf(src, s_buf, s_len);				\
+	abd_return_buf_copy(dst, d_buf, d_len);				\
+	return (c_len);							\
+}
+#define	ZFS_DECOMPRESS_WRAP_DECL(name)					\
+int									\
+name(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int n)		\
+{									\
+	void *s_buf = abd_borrow_buf_copy(src, s_len);			\
+	void *d_buf = abd_borrow_buf(dst, d_len);			\
+	int err = name##_buf(s_buf, d_buf, s_len, d_len, n);		\
+	abd_return_buf(src, s_buf, s_len);				\
+	abd_return_buf_copy(dst, d_buf, d_len);				\
+	return (err);							\
+}
+#define	ZFS_DECOMPRESS_LEVEL_WRAP_DECL(name)				\
+int									\
+name(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, uint8_t *n)	\
+{									\
+	void *s_buf = abd_borrow_buf_copy(src, s_len);			\
+	void *d_buf = abd_borrow_buf(dst, d_len);			\
+	int err = name##_buf(s_buf, d_buf, s_len, d_len, n);		\
+	abd_return_buf(src, s_buf, s_len);				\
+	abd_return_buf_copy(dst, d_buf, d_len);				\
+	return (err);							\
+}
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/include/sys/zstd/zstd.h b/include/sys/zstd/zstd.h
index f9e7ac0b3..6d212b082 100644
--- a/include/sys/zstd/zstd.h
+++ b/include/sys/zstd/zstd.h
@@ -90,12 +90,12 @@ typedef struct zfs_zstd_meta {
 int zstd_init(void);
 void zstd_fini(void);
 
-size_t zfs_zstd_compress(void *s_start, void *d_start, size_t s_len,
+size_t zfs_zstd_compress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int level);
 int zfs_zstd_get_level(void *s_start, size_t s_len, uint8_t *level);
-int zfs_zstd_decompress_level(void *s_start, void *d_start, size_t s_len,
+int zfs_zstd_decompress_level(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, uint8_t *level);
-int zfs_zstd_decompress(void *s_start, void *d_start, size_t s_len,
+int zfs_zstd_decompress(abd_t *src, abd_t *dst, size_t s_len,
     size_t d_len, int n);
 void zfs_zstd_cache_reap_now(void);
 
diff --git a/module/zfs/ddt_zap.c b/module/zfs/ddt_zap.c
index 8e78ec327..e96984b86 100644
--- a/module/zfs/ddt_zap.c
+++ b/module/zfs/ddt_zap.c
@@ -53,8 +53,12 @@ ddt_zap_compress(const void *src, uchar_t *dst, size_t s_len, size_t d_len)
 	ASSERT3U(d_len, >=, s_len + 1);	/* no compression plus version byte */
 
 	/* Call compress function directly to avoid hole detection. */
-	c_len = ci->ci_compress((void *)src, dst, s_len, d_len - 1,
-	    ci->ci_level);
+	abd_t sabd, dabd;
+	abd_get_from_buf_struct(&sabd, (void *)src, s_len);
+	abd_get_from_buf_struct(&dabd, dst, d_len);
+	c_len = ci->ci_compress(&sabd, &dabd, s_len, d_len - 1, ci->ci_level);
+	abd_free(&dabd);
+	abd_free(&sabd);
 
 	if (c_len == s_len) {
 		cpfunc = ZIO_COMPRESS_OFF;
diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c
index e62ecdb25..042725b23 100644
--- a/module/zfs/dsl_dataset.c
+++ b/module/zfs/dsl_dataset.c
@@ -2426,8 +2426,13 @@ get_receive_resume_token_impl(dsl_dataset_t *ds)
 	compressed = kmem_alloc(packed_size, KM_SLEEP);
 
 	/* Call compress function directly to avoid hole detection. */
-	compressed_size = zfs_gzip_compress(packed, compressed,
+	abd_t pabd, cabd;
+	abd_get_from_buf_struct(&pabd, packed, packed_size);
+	abd_get_from_buf_struct(&cabd, compressed, packed_size);
+	compressed_size = zfs_gzip_compress(&pabd, &cabd,
 	    packed_size, packed_size, 6);
+	abd_free(&cabd);
+	abd_free(&pabd);
 
 	zio_cksum_t cksum;
 	fletcher_4_native_varsize(compressed, compressed_size, &cksum);
diff --git a/module/zfs/gzip.c b/module/zfs/gzip.c
index 0ca66c2bd..e7fd6f63c 100644
--- a/module/zfs/gzip.c
+++ b/module/zfs/gzip.c
@@ -47,8 +47,8 @@ typedef uLongf zlen_t;
 
 #endif
 
-size_t
-zfs_gzip_compress(void *s_start, void *d_start, size_t s_len,
+static size_t
+zfs_gzip_compress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	int ret;
@@ -83,8 +83,8 @@ zfs_gzip_compress(void *s_start, void *d_start, size_t s_len,
 	return ((size_t)dstlen);
 }
 
-int
-zfs_gzip_decompress(void *s_start, void *d_start, size_t s_len,
+static int
+zfs_gzip_decompress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	(void) n;
@@ -105,3 +105,6 @@ zfs_gzip_decompress(void *s_start, void *d_start, size_t s_len,
 
 	return (0);
 }
+
+ZFS_COMPRESS_WRAP_DECL(zfs_gzip_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_gzip_decompress)
diff --git a/module/zfs/lz4_zfs.c b/module/zfs/lz4_zfs.c
index 698ed6995..0033b5e50 100644
--- a/module/zfs/lz4_zfs.c
+++ b/module/zfs/lz4_zfs.c
@@ -52,8 +52,8 @@ int LZ4_uncompress_unknownOutputSize(const char *source, char *dest,
 
 static kmem_cache_t *lz4_cache;
 
-size_t
-zfs_lz4_compress(void *s_start, void *d_start, size_t s_len,
+static size_t
+zfs_lz4_compress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	(void) n;
@@ -80,8 +80,8 @@ zfs_lz4_compress(void *s_start, void *d_start, size_t s_len,
 	return (bufsiz + sizeof (bufsiz));
 }
 
-int
-zfs_lz4_decompress(void *s_start, void *d_start, size_t s_len,
+static int
+zfs_lz4_decompress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	(void) n;
@@ -100,6 +100,9 @@ zfs_lz4_decompress(void *s_start, void *d_start, size_t s_len,
 	    d_start, bufsiz, d_len) < 0);
 }
 
+ZFS_COMPRESS_WRAP_DECL(zfs_lz4_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_lz4_decompress)
+
 /*
  * LZ4 API Description:
  *
diff --git a/module/zfs/lzjb.c b/module/zfs/lzjb.c
index b24669312..2db549b16 100644
--- a/module/zfs/lzjb.c
+++ b/module/zfs/lzjb.c
@@ -45,8 +45,8 @@
 #define	OFFSET_MASK	((1 << (16 - MATCH_BITS)) - 1)
 #define	LEMPEL_SIZE	1024
 
-size_t
-zfs_lzjb_compress(void *s_start, void *d_start, size_t s_len,
+static size_t
+zfs_lzjb_compress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	(void) n;
@@ -101,8 +101,8 @@ zfs_lzjb_compress(void *s_start, void *d_start, size_t s_len,
 	return (dst - (uchar_t *)d_start);
 }
 
-int
-zfs_lzjb_decompress(void *s_start, void *d_start,
+static int
+zfs_lzjb_decompress_buf(void *s_start, void *d_start,
     size_t s_len, size_t d_len, int n)
 {
 	(void) s_len, (void) n;
@@ -132,3 +132,6 @@ zfs_lzjb_decompress(void *s_start, void *d_start,
 	}
 	return (0);
 }
+
+ZFS_COMPRESS_WRAP_DECL(zfs_lzjb_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_lzjb_decompress)
diff --git a/module/zfs/zio_compress.c b/module/zfs/zio_compress.c
index 9553a9377..118003bd2 100644
--- a/module/zfs/zio_compress.c
+++ b/module/zfs/zio_compress.c
@@ -29,7 +29,7 @@
 
 /*
  * Copyright (c) 2013, 2018 by Delphix. All rights reserved.
- * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, 2024, Klara, Inc.
  * Copyright (c) 2019, Allan Jude
  */
 
@@ -160,10 +160,10 @@ zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len,
 	if (*dst == NULL)
 		*dst = zio_buf_alloc(s_len);
 
-	/* No compression algorithms can read from ABDs directly */
-	void *tmp = abd_borrow_buf_copy(src, s_len);
-	c_len = ci->ci_compress(tmp, *dst, s_len, d_len, complevel);
-	abd_return_buf(src, tmp, s_len);
+	abd_t dabd;
+	abd_get_from_buf_struct(&dabd, dst, d_len);
+	c_len = ci->ci_compress(src, &dabd, s_len, d_len, complevel);
+	abd_free(&dabd);
 
 	if (c_len > d_len)
 		return (s_len);
@@ -180,15 +180,16 @@ zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
 	if ((uint_t)c >= ZIO_COMPRESS_FUNCTIONS || ci->ci_decompress == NULL)
 		return (SET_ERROR(EINVAL));
 
-	void *sbuf = abd_borrow_buf_copy(src, s_len);
+	abd_t dabd;
+	abd_get_from_buf_struct(&dabd, dst, d_len);
 
 	int err;
 	if (ci->ci_decompress_level != NULL && level != NULL)
-		err = ci->ci_decompress_level(sbuf, dst, s_len, d_len, level);
+		err = ci->ci_decompress_level(src, &dabd, s_len, d_len, level);
 	else
-		err = ci->ci_decompress(sbuf, dst, s_len, d_len, ci->ci_level);
+		err = ci->ci_decompress(src, &dabd, s_len, d_len, ci->ci_level);
 
-	abd_return_buf(src, sbuf, s_len);
+	abd_free(&dabd);
 
 	/*
 	 * Decompression shouldn't fail, because we've already verified
diff --git a/module/zfs/zle.c b/module/zfs/zle.c
index 32b5fe18c..781016196 100644
--- a/module/zfs/zle.c
+++ b/module/zfs/zle.c
@@ -34,8 +34,8 @@
 #include <sys/sysmacros.h>
 #include <sys/zio_compress.h>
 
-size_t
-zfs_zle_compress(void *s_start, void *d_start, size_t s_len,
+static size_t
+zfs_zle_compress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	uchar_t *src = s_start;
@@ -65,8 +65,8 @@ zfs_zle_compress(void *s_start, void *d_start, size_t s_len,
 	return (src == s_end ? dst - (uchar_t *)d_start : s_len);
 }
 
-int
-zfs_zle_decompress(void *s_start, void *d_start, size_t s_len,
+static int
+zfs_zle_decompress_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, int n)
 {
 	uchar_t *src = s_start;
@@ -91,3 +91,6 @@ zfs_zle_decompress(void *s_start, void *d_start, size_t s_len,
 	}
 	return (dst == d_end ? 0 : -1);
 }
+
+ZFS_COMPRESS_WRAP_DECL(zfs_zle_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_zle_decompress)
diff --git a/module/zstd/zfs_zstd.c b/module/zstd/zfs_zstd.c
index 34ab8fd8a..8d1d53d23 100644
--- a/module/zstd/zfs_zstd.c
+++ b/module/zstd/zfs_zstd.c
@@ -536,8 +536,8 @@ zfs_zstd_compress_impl(void *s_start, void *d_start, size_t s_len, size_t d_len,
 }
 
 
-size_t
-zfs_zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len,
+static size_t
+zfs_zstd_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len,
     int level)
 {
 	int16_t zstd_level;
@@ -569,7 +569,10 @@ zfs_zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len,
 	if (zstd_earlyabort_pass > 0 && zstd_level >= zstd_cutoff_level &&
 	    s_len >= actual_abort_size) {
 		int pass_len = 1;
-		pass_len = zfs_lz4_compress(s_start, d_start, s_len, d_len, 0);
+		abd_t sabd;
+		abd_get_from_buf_struct(&sabd, s_start, s_len);
+		pass_len = zfs_lz4_compress(&sabd, d_start, s_len, d_len, 0);
+		abd_free(&sabd);
 		if (pass_len < d_len) {
 			ZSTDSTAT_BUMP(zstd_stat_lz4pass_allowed);
 			goto keep_trying;
@@ -595,8 +598,8 @@ keep_trying:
 }
 
 /* Decompress block using zstd and return its stored level */
-int
-zfs_zstd_decompress_level(void *s_start, void *d_start, size_t s_len,
+static int
+zfs_zstd_decompress_level_buf(void *s_start, void *d_start, size_t s_len,
     size_t d_len, uint8_t *level)
 {
 	ZSTD_DCtx *dctx;
@@ -671,15 +674,20 @@ zfs_zstd_decompress_level(void *s_start, void *d_start, size_t s_len,
 }
 
 /* Decompress datablock using zstd */
-int
-zfs_zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len,
-    int level __maybe_unused)
+static int
+zfs_zstd_decompress_buf(void *s_start, void *d_start, size_t s_len,
+    size_t d_len, int level __maybe_unused)
 {
 
-	return (zfs_zstd_decompress_level(s_start, d_start, s_len, d_len,
+	return (zfs_zstd_decompress_level_buf(s_start, d_start, s_len, d_len,
 	    NULL));
 }
 
+ZFS_COMPRESS_WRAP_DECL(zfs_zstd_compress)
+ZFS_DECOMPRESS_WRAP_DECL(zfs_zstd_decompress)
+ZFS_DECOMPRESS_LEVEL_WRAP_DECL(zfs_zstd_decompress_level)
+
+
 /* Allocator for zstd compression context using mempool_allocator */
 static void *
 zstd_alloc(void *opaque __maybe_unused, size_t size)