DLPX-44812 integrate EP-220 large memory scalability

1 files changed, 51 insertions, 29 deletions

diff --git a/module/zfs/zio_compress.c b/module/zfs/zio_compress.c
index 6b8d6c39b..7e44d16e4 100644
--- a/module/zfs/zio_compress.c
+++ b/module/zfs/zio_compress.c
@@ -28,7 +28,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2013, 2016 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -41,24 +41,23 @@
 /*
  * Compression vectors.
  */
-
 zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS] = {
-	{NULL,			NULL,			0,	"inherit"},
-	{NULL,			NULL,			0,	"on"},
-	{NULL,			NULL,			0,	"uncompressed"},
-	{lzjb_compress,		lzjb_decompress,	0,	"lzjb"},
-	{NULL,			NULL,			0,	"empty"},
-	{gzip_compress,		gzip_decompress,	1,	"gzip-1"},
-	{gzip_compress,		gzip_decompress,	2,	"gzip-2"},
-	{gzip_compress,		gzip_decompress,	3,	"gzip-3"},
-	{gzip_compress,		gzip_decompress,	4,	"gzip-4"},
-	{gzip_compress,		gzip_decompress,	5,	"gzip-5"},
-	{gzip_compress,		gzip_decompress,	6,	"gzip-6"},
-	{gzip_compress,		gzip_decompress,	7,	"gzip-7"},
-	{gzip_compress,		gzip_decompress,	8,	"gzip-8"},
-	{gzip_compress,		gzip_decompress,	9,	"gzip-9"},
-	{zle_compress,		zle_decompress,		64,	"zle"},
-	{lz4_compress_zfs,	lz4_decompress_zfs,	0,	"lz4"},
+	{"inherit",		0,	NULL,		NULL},
+	{"on",			0,	NULL,		NULL},
+	{"uncompressed",	0,	NULL,		NULL},
+	{"lzjb",		0,	lzjb_compress,	lzjb_decompress},
+	{"empty",		0,	NULL,		NULL},
+	{"gzip-1",		1,	gzip_compress,	gzip_decompress},
+	{"gzip-2",		2,	gzip_compress,	gzip_decompress},
+	{"gzip-3",		3,	gzip_compress,	gzip_decompress},
+	{"gzip-4",		4,	gzip_compress,	gzip_decompress},
+	{"gzip-5",		5,	gzip_compress,	gzip_decompress},
+	{"gzip-6",		6,	gzip_compress,	gzip_decompress},
+	{"gzip-7",		7,	gzip_compress,	gzip_decompress},
+	{"gzip-8",		8,	gzip_compress,	gzip_decompress},
+	{"gzip-9",		9,	gzip_compress,	gzip_decompress},
+	{"zle",			64,	zle_compress,	zle_decompress},
+	{"lz4",			0,	lz4_compress_zfs, lz4_decompress_zfs}
 };
 
 enum zio_compress
@@ -85,12 +84,26 @@ zio_compress_select(spa_t *spa, enum zio_compress child,
 	return (result);
 }
 
+/*ARGSUSED*/
+static int
+zio_compress_zeroed_cb(void *data, size_t len, void *private)
+{
+	uint64_t *end = (uint64_t *)((char *)data + len);
+	uint64_t *word;
+
+	for (word = data; word < end; word++)
+		if (*word != 0)
+			return (1);
+
+	return (0);
+}
+
 size_t
-zio_compress_data(enum zio_compress c, void *src, void *dst, size_t s_len)
+zio_compress_data(enum zio_compress c, abd_t *src, void *dst, size_t s_len)
 {
-	uint64_t *word, *word_end;
 	size_t c_len, d_len;
 	zio_compress_info_t *ci = &zio_compress_table[c];
+	void *tmp;
 
 	ASSERT((uint_t)c < ZIO_COMPRESS_FUNCTIONS);
 	ASSERT((uint_t)c == ZIO_COMPRESS_EMPTY || ci->ci_compress != NULL);
@@ -99,12 +112,7 @@ zio_compress_data(enum zio_compress c, void *src, void *dst, size_t s_len)
 	 * If the data is all zeroes, we don't even need to allocate
 	 * a block for it.  We indicate this by returning zero size.
 	 */
-	word_end = (uint64_t *)((char *)src + s_len);
-	for (word = src; word < word_end; word++)
-		if (*word != 0)
-			break;
-
-	if (word == word_end)
+	if (abd_iterate_func(src, 0, s_len, zio_compress_zeroed_cb, NULL) == 0)
 		return (0);
 
 	if (c == ZIO_COMPRESS_EMPTY)
@@ -112,7 +120,11 @@ zio_compress_data(enum zio_compress c, void *src, void *dst, size_t s_len)
 
 	/* Compress at least 12.5% */
 	d_len = s_len - (s_len >> 3);
-	c_len = ci->ci_compress(src, dst, s_len, d_len, ci->ci_level);
+
+	/* No compression algorithms can read from ABDs directly */
+	tmp = abd_borrow_buf_copy(src, s_len);
+	c_len = ci->ci_compress(tmp, dst, s_len, d_len, ci->ci_level);
+	abd_return_buf(src, tmp, s_len);
 
 	if (c_len > d_len)
 		return (s_len);
@@ -122,13 +134,23 @@ zio_compress_data(enum zio_compress c, void *src, void *dst, size_t s_len)
 }
 
 int
-zio_decompress_data(enum zio_compress c, void *src, void *dst,
+zio_decompress_data_buf(enum zio_compress c, void *src, void *dst,
     size_t s_len, size_t d_len)
 {
 	zio_compress_info_t *ci = &zio_compress_table[c];
-
 	if ((uint_t)c >= ZIO_COMPRESS_FUNCTIONS || ci->ci_decompress == NULL)
 		return (SET_ERROR(EINVAL));
 
 	return (ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level));
 }
+
+int
+zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
+    size_t s_len, size_t d_len)
+{
+	void *tmp = abd_borrow_buf_copy(src, s_len);
+	int ret = zio_decompress_data_buf(c, tmp, dst, s_len, d_len);
+	abd_return_buf(src, tmp, s_len);
+
+	return (ret);
+}