QAT support for AES-GCM

This patch adds support for acceleration of AES-GCM encryption
with Intel Quick Assist Technology.

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Chengfeix Zhu <[email protected]>
Signed-off-by: Weigang Li <[email protected]>
Signed-off-by: Tom Caputi <[email protected]>
Closes #7282 

1 files changed, 57 insertions, 166 deletions

diff --git a/module/zfs/qat_compress.c b/module/zfs/qat_compress.c
index 62655f56d..3d756b53d 100644
--- a/module/zfs/qat_compress.c
+++ b/module/zfs/qat_compress.c
@@ -25,12 +25,7 @@
 #include <linux/pagemap.h>
 #include <linux/completion.h>
 #include <sys/zfs_context.h>
-#include "qat_compress.h"
-
-/*
- * Timeout - no response from hardware after 0.5 seconds
- */
-#define	TIMEOUT_MS		500
+#include "qat.h"
 
 /*
  * Max instances in QAT device, each instance is a channel to submit
@@ -38,7 +33,7 @@
  * and session arrays, the actual number of instances are defined in
  * the QAT driver's configure file.
  */
-#define	MAX_INSTANCES		48
+#define	QAT_DC_MAX_INSTANCES	48
 
 /*
  * ZLIB head and foot size
@@ -46,89 +41,20 @@
 #define	ZLIB_HEAD_SZ		2
 #define	ZLIB_FOOT_SZ		4
 
-/*
- * The minimal and maximal buffer size, which are not restricted
- * in the QAT hardware, but with the input buffer size between 4KB
- * and 128KB, the hardware can provide the optimal performance.
- */
-#define	QAT_MIN_BUF_SIZE	(4*1024)
-#define	QAT_MAX_BUF_SIZE	(128*1024)
-
-/*
- * Used for qat kstat.
- */
-typedef struct qat_stats {
-	/*
-	 * Number of jobs submitted to qat compression engine.
-	 */
-	kstat_named_t comp_requests;
-	/*
-	 * Total bytes sent to qat compression engine.
-	 */
-	kstat_named_t comp_total_in_bytes;
-	/*
-	 * Total bytes output from qat compression engine.
-	 */
-	kstat_named_t comp_total_out_bytes;
-	/*
-	 * Number of jobs submitted to qat de-compression engine.
-	 */
-	kstat_named_t decomp_requests;
-	/*
-	 * Total bytes sent to qat de-compression engine.
-	 */
-	kstat_named_t decomp_total_in_bytes;
-	/*
-	 * Total bytes output from qat de-compression engine.
-	 */
-	kstat_named_t decomp_total_out_bytes;
-	/*
-	 * Number of fails in qat engine.
-	 * Note: when qat fail happens, it doesn't mean a critical hardware
-	 * issue, sometimes it is because the output buffer is not big enough,
-	 * and the compression job will be transfered to gzip software again,
-	 * so the functionality of ZFS is not impacted.
-	 */
-	kstat_named_t dc_fails;
-} qat_stats_t;
-
-qat_stats_t qat_stats = {
-	{ "comp_reqests",			KSTAT_DATA_UINT64 },
-	{ "comp_total_in_bytes",		KSTAT_DATA_UINT64 },
-	{ "comp_total_out_bytes",		KSTAT_DATA_UINT64 },
-	{ "decomp_reqests",			KSTAT_DATA_UINT64 },
-	{ "decomp_total_in_bytes",		KSTAT_DATA_UINT64 },
-	{ "decomp_total_out_bytes",		KSTAT_DATA_UINT64 },
-	{ "dc_fails",				KSTAT_DATA_UINT64 },
-};
-
-static kstat_t *qat_ksp;
-static CpaInstanceHandle dc_inst_handles[MAX_INSTANCES];
-static CpaDcSessionHandle session_handles[MAX_INSTANCES];
-static CpaBufferList **buffer_array[MAX_INSTANCES];
+static CpaInstanceHandle dc_inst_handles[QAT_DC_MAX_INSTANCES];
+static CpaDcSessionHandle session_handles[QAT_DC_MAX_INSTANCES];
+static CpaBufferList **buffer_array[QAT_DC_MAX_INSTANCES];
 static Cpa16U num_inst = 0;
 static Cpa32U inst_num = 0;
-static boolean_t qat_init_done = B_FALSE;
-int zfs_qat_disable = 0;
-
-#define	QAT_STAT_INCR(stat, val) \
-	atomic_add_64(&qat_stats.stat.value.ui64, (val));
-#define	QAT_STAT_BUMP(stat) \
-	QAT_STAT_INCR(stat, 1);
+static boolean_t qat_dc_init_done = B_FALSE;
 
-#define	PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes)	\
-	mem_alloc_contig((void *)(pp_mem_addr), (size_bytes))
-
-#define	PHYS_CONTIG_FREE(p_mem_addr)	\
-	mem_free_contig((void *)&(p_mem_addr))
-
-static inline struct page *
-mem_to_page(void *addr)
+boolean_t
+qat_dc_use_accel(size_t s_len)
 {
-	if (!is_vmalloc_addr(addr))
-		return (virt_to_page(addr));
-
-	return (vmalloc_to_page(addr));
+	return (!zfs_qat_disable &&
+	    qat_dc_init_done &&
+	    s_len >= QAT_MIN_BUF_SIZE &&
+	    s_len <= QAT_MAX_BUF_SIZE);
 }
 
 static void
@@ -138,26 +64,8 @@ qat_dc_callback(void *p_callback, CpaStatus status)
 		complete((struct completion *)p_callback);
 }
 
-static inline CpaStatus
-mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes)
-{
-	*pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL);
-	if (*pp_mem_addr == NULL)
-		return (CPA_STATUS_RESOURCE);
-	return (CPA_STATUS_SUCCESS);
-}
-
-static inline void
-mem_free_contig(void **pp_mem_addr)
-{
-	if (*pp_mem_addr != NULL) {
-		kfree(*pp_mem_addr);
-		*pp_mem_addr = NULL;
-	}
-}
-
 static void
-qat_clean(void)
+qat_dc_clean(void)
 {
 	Cpa16U buff_num = 0;
 	Cpa16U num_inter_buff_lists = 0;
@@ -165,7 +73,7 @@ qat_clean(void)
 
 	for (i = 0; i < num_inst; i++) {
 		cpaDcStopInstance(dc_inst_handles[i]);
-		PHYS_CONTIG_FREE(session_handles[i]);
+		QAT_PHYS_CONTIG_FREE(session_handles[i]);
 		/* free intermediate buffers  */
 		if (buffer_array[i] != NULL) {
 			cpaDcGetNumIntermediateBuffers(
@@ -175,24 +83,24 @@ qat_clean(void)
 				CpaBufferList *buffer_inter =
 				    buffer_array[i][buff_num];
 				if (buffer_inter->pBuffers) {
-					PHYS_CONTIG_FREE(
+					QAT_PHYS_CONTIG_FREE(
 					    buffer_inter->pBuffers->pData);
-					PHYS_CONTIG_FREE(
+					QAT_PHYS_CONTIG_FREE(
 					    buffer_inter->pBuffers);
 				}
-				PHYS_CONTIG_FREE(
+				QAT_PHYS_CONTIG_FREE(
 				    buffer_inter->pPrivateMetaData);
-				PHYS_CONTIG_FREE(buffer_inter);
+				QAT_PHYS_CONTIG_FREE(buffer_inter);
 			}
 		}
 	}
 
 	num_inst = 0;
-	qat_init_done = B_FALSE;
+	qat_dc_init_done = B_FALSE;
 }
 
 int
-qat_init(void)
+qat_dc_init(void)
 {
 	CpaStatus status = CPA_STATUS_SUCCESS;
 	Cpa32U sess_size = 0;
@@ -204,11 +112,15 @@ qat_init(void)
 	Cpa16U i;
 
 	status = cpaDcGetNumInstances(&num_inst);
-	if (status != CPA_STATUS_SUCCESS || num_inst == 0)
+	if (status != CPA_STATUS_SUCCESS)
 		return (-1);
 
-	if (num_inst > MAX_INSTANCES)
-		num_inst = MAX_INSTANCES;
+	/* if the user has configured no QAT compression units just return */
+	if (num_inst == 0)
+		return (0);
+
+	if (num_inst > QAT_DC_MAX_INSTANCES)
+		num_inst = QAT_DC_MAX_INSTANCES;
 
 	status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
 	if (status != CPA_STATUS_SUCCESS)
@@ -226,25 +138,25 @@ qat_init(void)
 			    dc_inst_handles[i], &num_inter_buff_lists);
 
 		if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
-			status = PHYS_CONTIG_ALLOC(&buffer_array[i],
+			status = QAT_PHYS_CONTIG_ALLOC(&buffer_array[i],
 			    num_inter_buff_lists *
 			    sizeof (CpaBufferList *));
 
 		for (buff_num = 0; buff_num < num_inter_buff_lists;
 		    buff_num++) {
 			if (status == CPA_STATUS_SUCCESS)
-				status = PHYS_CONTIG_ALLOC(
+				status = QAT_PHYS_CONTIG_ALLOC(
 				    &buffer_array[i][buff_num],
 				    sizeof (CpaBufferList));
 
 			if (status == CPA_STATUS_SUCCESS)
-				status = PHYS_CONTIG_ALLOC(
+				status = QAT_PHYS_CONTIG_ALLOC(
 				    &buffer_array[i][buff_num]->
 				    pPrivateMetaData,
 				    buff_meta_size);
 
 			if (status == CPA_STATUS_SUCCESS)
-				status = PHYS_CONTIG_ALLOC(
+				status = QAT_PHYS_CONTIG_ALLOC(
 				    &buffer_array[i][buff_num]->pBuffers,
 				    sizeof (CpaFlatBuffer));
 
@@ -255,7 +167,7 @@ qat_init(void)
 				 *  output buffer, which is 2x max buffer
 				 *  size here.
 				 */
-				status = PHYS_CONTIG_ALLOC(
+				status = QAT_PHYS_CONTIG_ALLOC(
 				    &buffer_array[i][buff_num]->pBuffers->
 				    pData, 2 * QAT_MAX_BUF_SIZE);
 				if (status != CPA_STATUS_SUCCESS)
@@ -284,7 +196,7 @@ qat_init(void)
 		if (status != CPA_STATUS_SUCCESS)
 			goto fail;
 
-		PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
+		QAT_PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
 		if (session_handles[i] == NULL)
 			goto fail;
 
@@ -295,39 +207,20 @@ qat_init(void)
 			goto fail;
 	}
 
-	qat_ksp = kstat_create("zfs", 0, "qat", "misc",
-	    KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
-	    KSTAT_FLAG_VIRTUAL);
-	if (qat_ksp != NULL) {
-		qat_ksp->ks_data = &qat_stats;
-		kstat_install(qat_ksp);
-	}
-
-	qat_init_done = B_TRUE;
+	qat_dc_init_done = B_TRUE;
 	return (0);
 fail:
-	qat_clean();
+	qat_dc_clean();
 	return (-1);
 }
 
 void
-qat_fini(void)
+qat_dc_fini(void)
 {
-	qat_clean();
+	if (!qat_dc_init_done)
+		return;
 
-	if (qat_ksp != NULL) {
-		kstat_delete(qat_ksp);
-		qat_ksp = NULL;
-	}
-}
-
-boolean_t
-qat_use_accel(size_t s_len)
-{
-	return (!zfs_qat_disable &&
-	    qat_init_done &&
-	    s_len >= QAT_MIN_BUF_SIZE &&
-	    s_len <= QAT_MAX_BUF_SIZE);
+	qat_dc_clean();
 }
 
 int
@@ -364,11 +257,11 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 	Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
 	    (num_dst_buf * sizeof (CpaFlatBuffer));
 
-	if (PHYS_CONTIG_ALLOC(&in_pages,
+	if (QAT_PHYS_CONTIG_ALLOC(&in_pages,
 	    num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
 		goto fail;
 
-	if (PHYS_CONTIG_ALLOC(&out_pages,
+	if (QAT_PHYS_CONTIG_ALLOC(&out_pages,
 	    num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
 		goto fail;
 
@@ -378,18 +271,18 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 
 	cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
 	    &buffer_meta_size);
-	if (PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
+	if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
 	    CPA_STATUS_SUCCESS)
 		goto fail;
 
 	cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf,
 	    &buffer_meta_size);
-	if (PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
+	if (QAT_PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
 	    CPA_STATUS_SUCCESS)
 		goto fail;
 
 	/* build source buffer list */
-	if (PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
+	if (QAT_PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
 	    CPA_STATUS_SUCCESS)
 		goto fail;
 
@@ -398,7 +291,7 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 	buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
 
 	/* build destination buffer list */
-	if (PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
+	if (QAT_PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
 	    CPA_STATUS_SUCCESS)
 		goto fail;
 
@@ -412,7 +305,7 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 	data = src;
 	page_num = 0;
 	while (bytes_left > 0) {
-		in_page = mem_to_page(data);
+		in_page = qat_mem_to_page(data);
 		in_pages[page_num] = in_page;
 		flat_buf_src->pData = kmap(in_page);
 		flat_buf_src->dataLenInBytes =
@@ -431,7 +324,7 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 	data = dst;
 	page_num = 0;
 	while (bytes_left > 0) {
-		out_page = mem_to_page(data);
+		out_page = qat_mem_to_page(data);
 		flat_buf_dst->pData = kmap(out_page);
 		out_pages[page_num] = out_page;
 		flat_buf_dst->dataLenInBytes =
@@ -465,7 +358,7 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 
 		/* we now wait until the completion of the operation. */
 		if (!wait_for_completion_interruptible_timeout(&complete,
-		    TIMEOUT_MS)) {
+		    QAT_TIMEOUT_MS)) {
 			status = CPA_STATUS_FAIL;
 			goto fail;
 		}
@@ -508,7 +401,8 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 
 		ret = 0;
 
-	} else if (dir == QAT_DECOMPRESS) {
+	} else {
+		ASSERT3U(dir, ==, QAT_DECOMPRESS);
 		QAT_STAT_BUMP(decomp_requests);
 		QAT_STAT_INCR(decomp_total_in_bytes, src_len);
 
@@ -529,7 +423,7 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 
 		/* we now wait until the completion of the operation. */
 		if (!wait_for_completion_interruptible_timeout(&complete,
-		    TIMEOUT_MS)) {
+		    QAT_TIMEOUT_MS)) {
 			status = CPA_STATUS_FAIL;
 			goto fail;
 		}
@@ -557,7 +451,7 @@ fail:
 		    page_num++) {
 			kunmap(in_pages[page_num]);
 		}
-		PHYS_CONTIG_FREE(in_pages);
+		QAT_PHYS_CONTIG_FREE(in_pages);
 	}
 
 	if (out_pages) {
@@ -566,18 +460,15 @@ fail:
 		    page_num++) {
 			kunmap(out_pages[page_num]);
 		}
-		PHYS_CONTIG_FREE(out_pages);
+		QAT_PHYS_CONTIG_FREE(out_pages);
 	}
 
-	PHYS_CONTIG_FREE(buffer_meta_src);
-	PHYS_CONTIG_FREE(buffer_meta_dst);
-	PHYS_CONTIG_FREE(buf_list_src);
-	PHYS_CONTIG_FREE(buf_list_dst);
+	QAT_PHYS_CONTIG_FREE(buffer_meta_src);
+	QAT_PHYS_CONTIG_FREE(buffer_meta_dst);
+	QAT_PHYS_CONTIG_FREE(buf_list_src);
+	QAT_PHYS_CONTIG_FREE(buf_list_dst);
 
 	return (ret);
 }
 
-module_param(zfs_qat_disable, int, 0644);
-MODULE_PARM_DESC(zfs_qat_disable, "Disable QAT compression");
-
 #endif