aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--module/zfs/qat_compress.c22
-rw-r--r--module/zfs/qat_crypt.c72
2 files changed, 60 insertions, 34 deletions
diff --git a/module/zfs/qat_compress.c b/module/zfs/qat_compress.c
index 2116c9790..2f7dad4b4 100644
--- a/module/zfs/qat_compress.c
+++ b/module/zfs/qat_compress.c
@@ -241,18 +241,24 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
CpaStatus status = CPA_STATUS_SUCCESS;
Cpa32U hdr_sz = 0;
Cpa32U compressed_sz;
- Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 1;
- Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 1;
+ Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2;
+ Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2;
Cpa32U bytes_left;
char *data;
struct page *in_page, *out_page;
struct page **in_pages = NULL;
struct page **out_pages = NULL;
+ Cpa32U page_off = 0;
struct completion complete;
size_t ret = -1;
- Cpa16U page_num = 0;
+ Cpa32U page_num = 0;
Cpa16U i;
+ /*
+ * We increment num_src_buf and num_dst_buf by 2 to allow
+ * us to handle non page-aligned buffer addresses and buffers
+ * whose sizes are not divisible by PAGE_SIZE.
+ */
Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
(num_src_buf * sizeof (CpaFlatBuffer));
Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
@@ -306,11 +312,12 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
data = src;
page_num = 0;
while (bytes_left > 0) {
+ page_off = ((long)data & ~PAGE_MASK);
in_page = qat_mem_to_page(data);
in_pages[page_num] = in_page;
- flat_buf_src->pData = kmap(in_page);
+ flat_buf_src->pData = kmap(in_page) + page_off;
flat_buf_src->dataLenInBytes =
- min((long)bytes_left, (long)PAGE_SIZE);
+ min((long)PAGE_SIZE - page_off, (long)bytes_left);
bytes_left -= flat_buf_src->dataLenInBytes;
data += flat_buf_src->dataLenInBytes;
@@ -325,11 +332,12 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
data = dst;
page_num = 0;
while (bytes_left > 0) {
+ page_off = ((long)data & ~PAGE_MASK);
out_page = qat_mem_to_page(data);
- flat_buf_dst->pData = kmap(out_page);
+ flat_buf_dst->pData = kmap(out_page) + page_off;
out_pages[page_num] = out_page;
flat_buf_dst->dataLenInBytes =
- min((long)bytes_left, (long)PAGE_SIZE);
+ min((long)PAGE_SIZE - page_off, (long)bytes_left);
bytes_left -= flat_buf_dst->dataLenInBytes;
data += flat_buf_dst->dataLenInBytes;
diff --git a/module/zfs/qat_crypt.c b/module/zfs/qat_crypt.c
index 3e2a76ab2..b9931e404 100644
--- a/module/zfs/qat_crypt.c
+++ b/module/zfs/qat_crypt.c
@@ -266,13 +266,11 @@ qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
if (status != CPA_STATUS_SUCCESS)
return (status);
- src->numBuffers = nr_bufs;
status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
if (status != CPA_STATUS_SUCCESS)
goto error;
if (src != dst) {
- dst->numBuffers = nr_bufs;
status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
meta_size);
if (status != CPA_STATUS_SUCCESS)
@@ -297,10 +295,9 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
CpaStatus status = CPA_STATUS_SUCCESS;
Cpa16U i;
CpaInstanceHandle cy_inst_handle;
- Cpa16U nr_bufs = (enc_len + PAGE_CACHE_SIZE - 1) / PAGE_CACHE_SIZE;
+ Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
Cpa32U bytes_left = 0;
- Cpa8S *in = NULL;
- Cpa8S *out = NULL;
+ Cpa8S *data = NULL;
CpaCySymSessionCtx *cy_session_ctx = NULL;
cy_callback_t cb;
CpaCySymOpData op_data = { 0 };
@@ -312,7 +309,8 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
CpaFlatBuffer *flat_dst_buf = NULL;
struct page *in_pages[MAX_PAGE_NUM];
struct page *out_pages[MAX_PAGE_NUM];
- Cpa32S page_num = 0;
+ Cpa32U in_page_num = 0;
+ Cpa32U out_page_num = 0;
Cpa32U in_page_off = 0;
Cpa32U out_page_off = 0;
@@ -336,6 +334,11 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
return (status);
}
+ /*
+ * We increment nr_bufs by 2 to allow us to handle non
+ * page-aligned buffer addresses and buffers whose sizes
+ * are not divisible by PAGE_SIZE.
+ */
status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
&src_buffer_list, &dst_buffer_list);
if (status != CPA_STATUS_SUCCESS)
@@ -351,30 +354,39 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
goto fail;
bytes_left = enc_len;
- in = src_buf;
- out = dst_buf;
+ data = src_buf;
flat_src_buf = flat_src_buf_array;
- flat_dst_buf = flat_dst_buf_array;
while (bytes_left > 0) {
- in_page_off = ((long)in & ~PAGE_MASK);
- out_page_off = ((long)out & ~PAGE_MASK);
- in_pages[page_num] = qat_mem_to_page(in);
- out_pages[page_num] = qat_mem_to_page(out);
- flat_src_buf->pData = kmap(in_pages[page_num]) + in_page_off;
- flat_dst_buf->pData = kmap(out_pages[page_num]) + out_page_off;
+ in_page_off = ((long)data & ~PAGE_MASK);
+ in_pages[in_page_num] = qat_mem_to_page(data);
+ flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
flat_src_buf->dataLenInBytes =
- min((long)PAGE_CACHE_SIZE - in_page_off, (long)bytes_left);
- flat_dst_buf->dataLenInBytes =
- min((long)PAGE_CACHE_SIZE - out_page_off, (long)bytes_left);
- in += flat_src_buf->dataLenInBytes;
- out += flat_dst_buf->dataLenInBytes;
+ min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
+ data += flat_src_buf->dataLenInBytes;
bytes_left -= flat_src_buf->dataLenInBytes;
flat_src_buf++;
- flat_dst_buf++;
- page_num++;
+ in_page_num++;
}
src_buffer_list.pBuffers = flat_src_buf_array;
+ src_buffer_list.numBuffers = in_page_num;
+
+ bytes_left = enc_len;
+ data = dst_buf;
+ flat_dst_buf = flat_dst_buf_array;
+ while (bytes_left > 0) {
+ out_page_off = ((long)data & ~PAGE_MASK);
+ out_pages[out_page_num] = qat_mem_to_page(data);
+ flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
+ out_page_off;
+ flat_dst_buf->dataLenInBytes =
+ min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
+ data += flat_dst_buf->dataLenInBytes;
+ bytes_left -= flat_dst_buf->dataLenInBytes;
+ flat_dst_buf++;
+ out_page_num++;
+ }
dst_buffer_list.pBuffers = flat_dst_buf_array;
+ dst_buffer_list.numBuffers = out_page_num;
op_data.sessionCtx = cy_session_ctx;
op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
@@ -418,10 +430,10 @@ fail:
if (status != CPA_STATUS_SUCCESS)
QAT_STAT_BUMP(crypt_fails);
- for (i = 0; i < page_num; i++) {
+ for (i = 0; i < in_page_num; i++)
kunmap(in_pages[i]);
+ for (i = 0; i < out_page_num; i++)
kunmap(out_pages[i]);
- }
cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
@@ -439,7 +451,7 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
CpaStatus status;
Cpa16U i;
CpaInstanceHandle cy_inst_handle;
- Cpa16U nr_bufs = (size + PAGE_CACHE_SIZE - 1) / PAGE_CACHE_SIZE;
+ Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
Cpa32U bytes_left = 0;
Cpa8S *data = NULL;
CpaCySymSessionCtx *cy_session_ctx = NULL;
@@ -450,7 +462,7 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
CpaFlatBuffer *flat_src_buf_array = NULL;
CpaFlatBuffer *flat_src_buf = NULL;
struct page *in_pages[MAX_PAGE_NUM];
- Cpa32S page_num = 0;
+ Cpa32U page_num = 0;
Cpa32U page_off = 0;
QAT_STAT_BUMP(cksum_requests);
@@ -469,6 +481,11 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
return (status);
}
+ /*
+ * We increment nr_bufs by 2 to allow us to handle non
+ * page-aligned buffer addresses and buffers whose sizes
+ * are not divisible by PAGE_SIZE.
+ */
status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
&src_buffer_list, &src_buffer_list);
if (status != CPA_STATUS_SUCCESS)
@@ -487,13 +504,14 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
in_pages[page_num] = qat_mem_to_page(data);
flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
flat_src_buf->dataLenInBytes =
- min((long)PAGE_CACHE_SIZE - page_off, (long)bytes_left);
+ min((long)PAGE_SIZE - page_off, (long)bytes_left);
data += flat_src_buf->dataLenInBytes;
bytes_left -= flat_src_buf->dataLenInBytes;
flat_src_buf++;
page_num++;
}
src_buffer_list.pBuffers = flat_src_buf_array;
+ src_buffer_list.numBuffers = page_num;
op_data.sessionCtx = cy_session_ctx;
op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-12 07:14:18 +0000