QAT related bug fixes

1. Fix issue:  Kernel BUG with QAT during decompression  #9276.
   Now it is uninterruptible for a specific given QAT request,
   but Ctrl-C interrupt still works in user-space process.

2. Copy the digest result to the buffer only when doing encryption,
   and vise-versa for decryption.

Reviewed-by: Tom Caputi <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Chengfei Zhu <[email protected]>
Closes #9276 
Closes #9303 

4 files changed, 18 insertions, 32 deletions

diff --git a/include/sys/qat.h b/include/sys/qat.h
index fdd608139..9ae8eb173 100644
--- a/include/sys/qat.h
+++ b/include/sys/qat.h
@@ -41,11 +41,6 @@ typedef enum qat_encrypt_dir {
 #include "lac/cpa_cy_sym.h"
 
 /*
- * Timeout - no response from hardware after 0.5 seconds
- */
-#define	QAT_TIMEOUT_MS		500
-
-/*
  * The minimal and maximal buffer size which are not restricted
  * in the QAT hardware, but with the input buffer size between 4KB
  * and 128KB the hardware can provide the optimal performance.
diff --git a/module/os/linux/zfs/qat.c b/module/os/linux/zfs/qat.c
index a6f024cb4..08613b3a2 100644
--- a/module/os/linux/zfs/qat.c
+++ b/module/os/linux/zfs/qat.c
@@ -21,7 +21,7 @@
 
 #if defined(_KERNEL) && defined(HAVE_QAT)
 #include <sys/zfs_context.h>
-#include "qat.h"
+#include <sys/qat.h>
 
 qat_stats_t qat_stats = {
 	{ "comp_requests",			KSTAT_DATA_UINT64 },
diff --git a/module/os/linux/zfs/qat_compress.c b/module/os/linux/zfs/qat_compress.c
index 4136b6555..79457a1ed 100644
--- a/module/os/linux/zfs/qat_compress.c
+++ b/module/os/linux/zfs/qat_compress.c
@@ -27,7 +27,7 @@
 #include <sys/zfs_context.h>
 #include <sys/byteorder.h>
 #include <sys/zio.h>
-#include "qat.h"
+#include <sys/qat.h>
 
 /*
  * Max instances in a QAT device, each instance is a channel to submit
@@ -403,11 +403,7 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
 		}
 
 		/* we now wait until the completion of the operation. */
-		if (!wait_for_completion_interruptible_timeout(&complete,
-		    QAT_TIMEOUT_MS)) {
-			status = CPA_STATUS_FAIL;
-			goto fail;
-		}
+		wait_for_completion(&complete);
 
 		if (dc_results.status != CPA_STATUS_SUCCESS) {
 			status = CPA_STATUS_FAIL;
@@ -462,11 +458,7 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
 		}
 
 		/* we now wait until the completion of the operation. */
-		if (!wait_for_completion_interruptible_timeout(&complete,
-		    QAT_TIMEOUT_MS)) {
-			status = CPA_STATUS_FAIL;
-			goto fail;
-		}
+		wait_for_completion(&complete);
 
 		if (dc_results.status != CPA_STATUS_SUCCESS) {
 			status = CPA_STATUS_FAIL;
diff --git a/module/os/linux/zfs/qat_crypt.c b/module/os/linux/zfs/qat_crypt.c
index 02e19d21d..4771b2f3b 100644
--- a/module/os/linux/zfs/qat_crypt.c
+++ b/module/os/linux/zfs/qat_crypt.c
@@ -35,7 +35,7 @@
 #include <sys/zio_crypt.h>
 #include "lac/cpa_cy_im.h"
 #include "lac/cpa_cy_common.h"
-#include "qat.h"
+#include <sys/qat.h>
 
 /*
  * Max instances in a QAT device, each instance is a channel to submit
@@ -414,6 +414,9 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 	op_data.messageLenToCipherInBytes = enc_len;
 	op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
 	bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);
+	/* if dir is QAT_DECRYPT, copy digest_buf to pDigestResult */
+	if (dir == QAT_DECRYPT)
+		bcopy(digest_buf, op_data.pDigestResult, ZIO_DATA_MAC_LEN);
 
 	cb.verify_result = CPA_FALSE;
 	init_completion(&cb.complete);
@@ -422,23 +425,21 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
 	if (status != CPA_STATUS_SUCCESS)
 		goto fail;
 
-	if (!wait_for_completion_interruptible_timeout(&cb.complete,
-	    QAT_TIMEOUT_MS)) {
-		status = CPA_STATUS_FAIL;
-		goto fail;
-	}
+	/* we now wait until the completion of the operation. */
+	wait_for_completion(&cb.complete);
 
 	if (cb.verify_result == CPA_FALSE) {
 		status = CPA_STATUS_FAIL;
 		goto fail;
 	}
 
-	/* save digest result to digest_buf */
-	bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
-	if (dir == QAT_ENCRYPT)
+	if (dir == QAT_ENCRYPT) {
+		/* if dir is QAT_ENCRYPT, save pDigestResult to digest_buf */
+		bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
 		QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
-	else
+	} else {
 		QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
+	}
 
 fail:
 	if (status != CPA_STATUS_SUCCESS)
@@ -548,11 +549,9 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
 	if (status != CPA_STATUS_SUCCESS)
 		goto fail;
 
-	if (!wait_for_completion_interruptible_timeout(&cb.complete,
-	    QAT_TIMEOUT_MS)) {
-		status = CPA_STATUS_FAIL;
-		goto fail;
-	}
+	/* we now wait until the completion of the operation. */
+	wait_for_completion(&cb.complete);
+
 	if (cb.verify_result == CPA_FALSE) {
 		status = CPA_STATUS_FAIL;
 		goto fail;