diff options
| -rw-r--r-- | config/kernel.m4 | 90 | ||||
| -rw-r--r-- | config/zfs-build.m4 | 3 | ||||
| -rw-r--r-- | module/Makefile.in | 2 | ||||
| -rw-r--r-- | module/zfs/Makefile.in | 1 | ||||
| -rw-r--r-- | module/zfs/gzip.c | 17 | ||||
| -rw-r--r-- | module/zfs/qat_compress.c | 585 | ||||
| -rw-r--r-- | module/zfs/qat_compress.h | 48 | ||||
| -rw-r--r-- | module/zfs/spa_misc.c | 3 |
8 files changed, 749 insertions, 0 deletions
diff --git a/config/kernel.m4 b/config/kernel.m4 index 25271cedc..71c88aa36 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -4,6 +4,7 @@ dnl # AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [ ZFS_AC_KERNEL ZFS_AC_SPL + ZFS_AC_QAT ZFS_AC_TEST_MODULE ZFS_AC_KERNEL_OBJTOOL ZFS_AC_KERNEL_CONFIG @@ -470,6 +471,95 @@ AC_DEFUN([ZFS_AC_SPL], [ ]) dnl # +dnl # Detect the QAT module to be built against +dnl # QAT provides hardware acceleration for data compression: +dnl # https://01.org/intel-quickassist-technology +dnl # * Download and install QAT driver from the above link +dnl # * Start QAT driver in your system: +dnl # service qat_service start +dnl # * Enable QAT in ZFS, e.g.: +dnl # ./configure --with-qat=<qat-driver-path>/QAT1.6 +dnl # make +dnl # * Set GZIP compression in ZFS dataset: +dnl # zfs set compression = gzip <dataset> +dnl # Then the data written to this ZFS pool is compressed +dnl # by QAT accelerator automatically, and de-compressed by +dnl # QAT when read from the pool. +dnl # * Get QAT hardware statistics by: +dnl # cat /proc/icp_dh895xcc_dev/qat +dnl # * To disable QAT: +dnl # insmod zfs.ko zfs_qat_disable=1 +dnl # +AC_DEFUN([ZFS_AC_QAT], [ + AC_ARG_WITH([qat], + AS_HELP_STRING([--with-qat=PATH], + [Path to qat source]), + AS_IF([test "$withval" = "yes"], + AC_MSG_ERROR([--with-qat=PATH requires a PATH]), + [qatsrc="$withval"])) + + AC_ARG_WITH([qat-obj], + AS_HELP_STRING([--with-qat-obj=PATH], + [Path to qat build objects]), + [qatbuild="$withval"]) + + AS_IF([test ! -z "${qatsrc}"], [ + AC_MSG_CHECKING([qat source directory]) + AC_MSG_RESULT([$qatsrc]) + QAT_SRC="${qatsrc}/quickassist" + AS_IF([ test ! -e "$QAT_SRC/include/cpa.h"], [ + AC_MSG_ERROR([ + *** Please make sure the qat driver package is installed + *** and specify the location of the qat source with the + *** '--with-qat=PATH' option then try again. Failed to + *** find cpa.h in: + ${QAT_SRC}/include]) + ]) + ]) + + AS_IF([test ! -z "${qatsrc}"], [ + AC_MSG_CHECKING([qat build directory]) + AS_IF([test -z "$qatbuild"], [ + qatbuild="${qatsrc}/build" + ]) + + AC_MSG_RESULT([$qatbuild]) + QAT_OBJ=${qatbuild} + AS_IF([ ! test -e "$QAT_OBJ/icp_qa_al.ko"], [ + AC_MSG_ERROR([ + *** Please make sure the qat driver is installed then try again. + *** Failed to find icp_qa_al.ko in: + $QAT_OBJ]) + ]) + + AC_SUBST(QAT_SRC) + AC_SUBST(QAT_OBJ) + + AC_DEFINE(HAVE_QAT, 1, + [qat is enabled and existed]) + ]) + + dnl # + dnl # Detect the name used for the QAT Module.symvers file. + dnl # + AS_IF([test ! -z "${qatsrc}"], [ + AC_MSG_CHECKING([qat file for module symbols]) + QAT_SYMBOLS=$QAT_SRC/lookaside/access_layer/src/Module.symvers + + AS_IF([test -r $QAT_SYMBOLS], [ + AC_MSG_RESULT([$QAT_SYMBOLS]) + AC_SUBST(QAT_SYMBOLS) + ],[ + AC_MSG_ERROR([ + *** Please make sure the qat driver is installed then try again. + *** Failed to find Module.symvers in: + $QAT_SYMBOLS]) + ]) + ]) + ]) +]) + +dnl # dnl # Basic toolchain sanity check. dnl # AC_DEFUN([ZFS_AC_TEST_MODULE], [ diff --git a/config/zfs-build.m4 b/config/zfs-build.m4 index 6c5f13240..7651dc2c1 100644 --- a/config/zfs-build.m4 +++ b/config/zfs-build.m4 @@ -81,6 +81,9 @@ AC_DEFUN([ZFS_AC_CONFIG], [ [test "x$enable_linux_builtin" != xyes ]) AM_CONDITIONAL([WANT_DEVNAME2DEVID], [test "x$user_libudev" = xyes ]) + AM_CONDITIONAL([CONFIG_QAT], + [test "$ZFS_CONFIG" = kernel -o "$ZFS_CONFIG" = all] && + [test "x$qatsrc" != x ]) ]) dnl # diff --git a/module/Makefile.in b/module/Makefile.in index 1c1ea032e..093cf2286 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -11,6 +11,8 @@ INSTALL_MOD_DIR ?= extra ZFS_MODULE_CFLAGS += -include @SPL_OBJ@/spl_config.h ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include -I@SPL@/include -I@SPL@ +@CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include +@CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@ export ZFS_MODULE_CFLAGS SUBDIR_TARGETS = icp diff --git a/module/zfs/Makefile.in b/module/zfs/Makefile.in index 6712b9b3c..f8d54f4dd 100644 --- a/module/zfs/Makefile.in +++ b/module/zfs/Makefile.in @@ -114,6 +114,7 @@ $(MODULE)-objs += zrlock.o $(MODULE)-objs += zvol.o $(MODULE)-objs += dsl_destroy.o $(MODULE)-objs += dsl_userhold.o +$(MODULE)-objs += qat_compress.o $(MODULE)-$(CONFIG_X86) += vdev_raidz_math_sse2.o $(MODULE)-$(CONFIG_X86) += vdev_raidz_math_ssse3.o diff --git a/module/zfs/gzip.c b/module/zfs/gzip.c index 6e5c859fe..6c8fdd308 100644 --- a/module/zfs/gzip.c +++ b/module/zfs/gzip.c @@ -28,6 +28,7 @@ #include <sys/debug.h> #include <sys/types.h> +#include "qat_compress.h" #ifdef _KERNEL @@ -56,6 +57,14 @@ gzip_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) ASSERT(d_len <= s_len); + /* check if hardware accelerator can be used */ + if (qat_use_accel(s_len)) { + if (qat_compress(QAT_COMPRESS, s_start, + s_len, d_start, d_len, &dstlen) == CPA_STATUS_SUCCESS) + return ((size_t)dstlen); + /* if hardware compress fail, do it again with software */ + } + if (compress_func(d_start, &dstlen, s_start, s_len, n) != Z_OK) { if (d_len != s_len) return (s_len); @@ -75,6 +84,14 @@ gzip_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) ASSERT(d_len >= s_len); + /* check if hardware accelerator can be used */ + if (qat_use_accel(d_len)) { + if (qat_compress(QAT_DECOMPRESS, s_start, s_len, + d_start, d_len, &dstlen) == CPA_STATUS_SUCCESS) + return (0); + /* if hardware de-compress fail, do it again with software */ + } + if (uncompress_func(d_start, &dstlen, s_start, s_len) != Z_OK) return (-1); diff --git a/module/zfs/qat_compress.c b/module/zfs/qat_compress.c new file mode 100644 index 000000000..a350c0f3e --- /dev/null +++ b/module/zfs/qat_compress.c @@ -0,0 +1,585 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#if defined(_KERNEL) && defined(HAVE_QAT) +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/pagemap.h> +#include <linux/completion.h> +#include <sys/zfs_context.h> +#include "qat_compress.h" + +/* + * Timeout - no response from hardware after 0.5 seconds + */ +#define TIMEOUT_MS 500 + +/* + * Max instances in QAT device, each instance is a channel to submit + * jobs to QAT hardware + */ +#define MAX_INSTANCES 6 + +/* + * ZLIB head and foot size + */ +#define ZLIB_HEAD_SZ 2 +#define ZLIB_FOOT_SZ 4 + +/* + * The minimal and maximal buffer size, which are not restricted + * in the QAT hardware, but with the input buffer size between 4KB + * and 128KB, the hardware can provide the optimal performance. + */ +#define QAT_MIN_BUF_SIZE (4*1024) +#define QAT_MAX_BUF_SIZE (128*1024) + +/* + * Used for qat kstat. + */ +typedef struct qat_stats { + /* + * Number of jobs submitted to qat compression engine. + */ + kstat_named_t comp_requests; + /* + * Total bytes sent to qat compression engine. + */ + kstat_named_t comp_total_in_bytes; + /* + * Total bytes output from qat compression engine. + */ + kstat_named_t comp_total_out_bytes; + /* + * Number of jobs submitted to qat de-compression engine. + */ + kstat_named_t decomp_requests; + /* + * Total bytes sent to qat de-compression engine. + */ + kstat_named_t decomp_total_in_bytes; + /* + * Total bytes output from qat de-compression engine. + */ + kstat_named_t decomp_total_out_bytes; + /* + * Number of fails in qat engine. + * Note: when qat fail happens, it doesn't mean a critical hardware + * issue, sometimes it is because the output buffer is not big enough, + * and the compression job will be transfered to gzip software again, + * so the functionality of ZFS is not impacted. + */ + kstat_named_t dc_fails; +} qat_stats_t; + +qat_stats_t qat_stats = { + { "comp_reqests", KSTAT_DATA_UINT64 }, + { "comp_total_in_bytes", KSTAT_DATA_UINT64 }, + { "comp_total_out_bytes", KSTAT_DATA_UINT64 }, + { "decomp_reqests", KSTAT_DATA_UINT64 }, + { "decomp_total_in_bytes", KSTAT_DATA_UINT64 }, + { "decomp_total_out_bytes", KSTAT_DATA_UINT64 }, + { "dc_fails", KSTAT_DATA_UINT64 }, +}; + +static kstat_t *qat_ksp; +static CpaInstanceHandle dc_inst_handles[MAX_INSTANCES]; +static CpaDcSessionHandle session_handles[MAX_INSTANCES]; +static CpaBufferList **buffer_array[MAX_INSTANCES]; +static Cpa16U num_inst = 0; +static Cpa16U inst_num = 0; +static boolean_t qat_init_done = B_FALSE; +int zfs_qat_disable = 0; + +#define QAT_STAT_INCR(stat, val) \ + atomic_add_64(&qat_stats.stat.value.ui64, (val)); +#define QAT_STAT_BUMP(stat) \ + QAT_STAT_INCR(stat, 1); + +#define PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes) \ + mem_alloc_contig((void *)(pp_mem_addr), (size_bytes)) + +#define PHYS_CONTIG_FREE(p_mem_addr) \ + mem_free_contig((void *)&(p_mem_addr)) + +static inline struct page * +mem_to_page(void *addr) +{ + if (!is_vmalloc_addr(addr)) + return (virt_to_page(addr)); + + return (vmalloc_to_page(addr)); +} + +static void +qat_dc_callback(void *p_callback, CpaStatus status) +{ + if (p_callback != NULL) + complete((struct completion *)p_callback); +} + +static inline CpaStatus +mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes) +{ + *pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL); + if (*pp_mem_addr == NULL) + return (CPA_STATUS_RESOURCE); + return (CPA_STATUS_SUCCESS); +} + +static inline void +mem_free_contig(void **pp_mem_addr) +{ + if (*pp_mem_addr != NULL) { + kfree(*pp_mem_addr); + *pp_mem_addr = NULL; + } +} + +static void +qat_clean(void) +{ + Cpa16U buff_num = 0; + Cpa16U num_inter_buff_lists = 0; + Cpa16U i = 0; + + for (i = 0; i < num_inst; i++) { + cpaDcStopInstance(dc_inst_handles[i]); + PHYS_CONTIG_FREE(session_handles[i]); + /* free intermediate buffers */ + if (buffer_array[i] != NULL) { + cpaDcGetNumIntermediateBuffers( + dc_inst_handles[i], &num_inter_buff_lists); + for (buff_num = 0; buff_num < num_inter_buff_lists; + buff_num++) { + CpaBufferList *buffer_inter = + buffer_array[i][buff_num]; + if (buffer_inter->pBuffers) { + PHYS_CONTIG_FREE( + buffer_inter->pBuffers->pData); + PHYS_CONTIG_FREE( + buffer_inter->pBuffers); + } + PHYS_CONTIG_FREE( + buffer_inter->pPrivateMetaData); + PHYS_CONTIG_FREE(buffer_inter); + } + } + } + + num_inst = 0; + qat_init_done = B_FALSE; +} + +int +qat_init(void) +{ + CpaStatus status = CPA_STATUS_SUCCESS; + Cpa32U sess_size = 0; + Cpa32U ctx_size = 0; + Cpa16U num_inter_buff_lists = 0; + Cpa16U buff_num = 0; + Cpa32U buff_meta_size = 0; + CpaDcSessionSetupData sd = {0}; + Cpa16U i; + + status = cpaDcGetNumInstances(&num_inst); + if (status != CPA_STATUS_SUCCESS || num_inst == 0) + return (-1); + + if (num_inst > MAX_INSTANCES) + num_inst = MAX_INSTANCES; + + status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]); + if (status != CPA_STATUS_SUCCESS) + return (-1); + + for (i = 0; i < num_inst; i++) { + cpaDcSetAddressTranslation(dc_inst_handles[i], + (void*)virt_to_phys); + + status = cpaDcBufferListGetMetaSize(dc_inst_handles[i], + 1, &buff_meta_size); + + if (status == CPA_STATUS_SUCCESS) + status = cpaDcGetNumIntermediateBuffers( + dc_inst_handles[i], &num_inter_buff_lists); + + if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0) + status = PHYS_CONTIG_ALLOC(&buffer_array[i], + num_inter_buff_lists * + sizeof (CpaBufferList *)); + + for (buff_num = 0; buff_num < num_inter_buff_lists; + buff_num++) { + if (status == CPA_STATUS_SUCCESS) + status = PHYS_CONTIG_ALLOC( + &buffer_array[i][buff_num], + sizeof (CpaBufferList)); + + if (status == CPA_STATUS_SUCCESS) + status = PHYS_CONTIG_ALLOC( + &buffer_array[i][buff_num]-> + pPrivateMetaData, + buff_meta_size); + + if (status == CPA_STATUS_SUCCESS) + status = PHYS_CONTIG_ALLOC( + &buffer_array[i][buff_num]->pBuffers, + sizeof (CpaFlatBuffer)); + + if (status == CPA_STATUS_SUCCESS) { + /* + * implementation requires an intermediate + * buffer approximately twice the size of + * output buffer, which is 2x max buffer + * size here. + */ + status = PHYS_CONTIG_ALLOC( + &buffer_array[i][buff_num]->pBuffers-> + pData, 2 * QAT_MAX_BUF_SIZE); + if (status != CPA_STATUS_SUCCESS) + goto fail; + + buffer_array[i][buff_num]->numBuffers = 1; + buffer_array[i][buff_num]->pBuffers-> + dataLenInBytes = 2 * QAT_MAX_BUF_SIZE; + } + } + + status = cpaDcStartInstance(dc_inst_handles[i], + num_inter_buff_lists, buffer_array[i]); + if (status != CPA_STATUS_SUCCESS) + goto fail; + + sd.compLevel = CPA_DC_L1; + sd.compType = CPA_DC_DEFLATE; + sd.huffType = CPA_DC_HT_FULL_DYNAMIC; + sd.sessDirection = CPA_DC_DIR_COMBINED; + sd.sessState = CPA_DC_STATELESS; + sd.deflateWindowSize = 7; + sd.checksum = CPA_DC_ADLER32; + status = cpaDcGetSessionSize(dc_inst_handles[i], + &sd, &sess_size, &ctx_size); + if (status != CPA_STATUS_SUCCESS) + goto fail; + + PHYS_CONTIG_ALLOC(&session_handles[i], sess_size); + if (session_handles[i] == NULL) + goto fail; + + status = cpaDcInitSession(dc_inst_handles[i], + session_handles[i], + &sd, NULL, qat_dc_callback); + if (status != CPA_STATUS_SUCCESS) + goto fail; + } + + qat_ksp = kstat_create("zfs", 0, "qat", "misc", + KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t), + KSTAT_FLAG_VIRTUAL); + if (qat_ksp != NULL) { + qat_ksp->ks_data = &qat_stats; + kstat_install(qat_ksp); + } + + qat_init_done = B_TRUE; + return (0); +fail: + qat_clean(); + return (-1); +} + +void +qat_fini(void) +{ + qat_clean(); + + if (qat_ksp != NULL) { + kstat_delete(qat_ksp); + qat_ksp = NULL; + } +} + +boolean_t +qat_use_accel(size_t s_len) +{ + return (!zfs_qat_disable && + qat_init_done && + s_len >= QAT_MIN_BUF_SIZE && + s_len <= QAT_MAX_BUF_SIZE); +} + +int +qat_compress(qat_compress_dir_t dir, char *src, int src_len, + char *dst, int dst_len, size_t *c_len) +{ + CpaInstanceHandle dc_inst_handle; + CpaDcSessionHandle session_handle; + CpaBufferList *buf_list_src = NULL; + CpaBufferList *buf_list_dst = NULL; + CpaFlatBuffer *flat_buf_src = NULL; + CpaFlatBuffer *flat_buf_dst = NULL; + Cpa8U *buffer_meta_src = NULL; + Cpa8U *buffer_meta_dst = NULL; + Cpa32U buffer_meta_size = 0; + CpaDcRqResults dc_results; + CpaStatus status = CPA_STATUS_SUCCESS; + Cpa32U hdr_sz = 0; + Cpa32U compressed_sz; + Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 1; + Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 1; + Cpa32U bytes_left; + char *data; + struct page *in_page, *out_page; + struct page **in_pages = NULL; + struct page **out_pages = NULL; + struct completion complete; + size_t ret = -1; + Cpa16U page_num = 0; + Cpa16U i; + + Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) + + (num_src_buf * sizeof (CpaFlatBuffer)); + Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) + + (num_dst_buf * sizeof (CpaFlatBuffer)); + + if (!is_vmalloc_addr(src) || !is_vmalloc_addr(src + src_len - 1) || + !is_vmalloc_addr(dst) || !is_vmalloc_addr(dst + dst_len - 1)) + return (-1); + + if (PHYS_CONTIG_ALLOC(&in_pages, + num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS) + goto fail; + + if (PHYS_CONTIG_ALLOC(&out_pages, + num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS) + goto fail; + + i = atomic_inc_32_nv(&inst_num) % num_inst; + dc_inst_handle = dc_inst_handles[i]; + session_handle = session_handles[i]; + + cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf, + &buffer_meta_size); + if (PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) != + CPA_STATUS_SUCCESS) + goto fail; + + cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf, + &buffer_meta_size); + if (PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) != + CPA_STATUS_SUCCESS) + goto fail; + + /* build source buffer list */ + if (PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) != + CPA_STATUS_SUCCESS) + goto fail; + + flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1); + + buf_list_src->pBuffers = flat_buf_src; /* always point to first one */ + + /* build destination buffer list */ + if (PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) != + CPA_STATUS_SUCCESS) + goto fail; + + flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); + + buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */ + + buf_list_src->numBuffers = 0; + buf_list_src->pPrivateMetaData = buffer_meta_src; + bytes_left = src_len; + data = src; + page_num = 0; + while (bytes_left > 0) { + in_page = mem_to_page(data); + in_pages[page_num] = in_page; + flat_buf_src->pData = kmap(in_page); + flat_buf_src->dataLenInBytes = + min((long)bytes_left, (long)PAGE_SIZE); + + bytes_left -= flat_buf_src->dataLenInBytes; + data += flat_buf_src->dataLenInBytes; + flat_buf_src++; + buf_list_src->numBuffers++; + page_num++; + } + + buf_list_dst->numBuffers = 0; + buf_list_dst->pPrivateMetaData = buffer_meta_dst; + bytes_left = dst_len; + data = dst; + page_num = 0; + while (bytes_left > 0) { + out_page = mem_to_page(data); + flat_buf_dst->pData = kmap(out_page); + out_pages[page_num] = out_page; + flat_buf_dst->dataLenInBytes = + min((long)bytes_left, (long)PAGE_SIZE); + + bytes_left -= flat_buf_dst->dataLenInBytes; + data += flat_buf_dst->dataLenInBytes; + flat_buf_dst++; + buf_list_dst->numBuffers++; + page_num++; + } + + init_completion(&complete); + + if (dir == QAT_COMPRESS) { + QAT_STAT_BUMP(comp_requests); + QAT_STAT_INCR(comp_total_in_bytes, src_len); + + cpaDcGenerateHeader(session_handle, + buf_list_dst->pBuffers, &hdr_sz); + buf_list_dst->pBuffers->pData += hdr_sz; + buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz; + status = cpaDcCompressData( + dc_inst_handle, session_handle, + buf_list_src, buf_list_dst, + &dc_results, CPA_DC_FLUSH_FINAL, + &complete); + if (status != CPA_STATUS_SUCCESS) { + goto fail; + } + + /* we now wait until the completion of the operation. */ + if (!wait_for_completion_interruptible_timeout(&complete, + TIMEOUT_MS)) { + status = CPA_STATUS_FAIL; + goto fail; + } + + if (dc_results.status != CPA_STATUS_SUCCESS) { + status = CPA_STATUS_FAIL; + goto fail; + } + + compressed_sz = dc_results.produced; + if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) { + goto fail; + } + + flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1); + /* move to the last page */ + flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT; + + /* no space for gzip foot in the last page */ + if (((compressed_sz + hdr_sz) % PAGE_SIZE) + + ZLIB_FOOT_SZ > PAGE_SIZE) + goto fail; + + flat_buf_dst->pData += (compressed_sz + hdr_sz) % PAGE_SIZE; + flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ; + + dc_results.produced = 0; + status = cpaDcGenerateFooter(session_handle, + flat_buf_dst, &dc_results); + if (status != CPA_STATUS_SUCCESS) { + goto fail; + } + + *c_len = compressed_sz + dc_results.produced + hdr_sz; + + if (*c_len < PAGE_SIZE) + *c_len = 8 * PAGE_SIZE; + + QAT_STAT_INCR(comp_total_out_bytes, *c_len); + + ret = 0; + + } else if (dir == QAT_DECOMPRESS) { + QAT_STAT_BUMP(decomp_requests); + QAT_STAT_INCR(decomp_total_in_bytes, src_len); + + buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ; + buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ; + status = cpaDcDecompressData(dc_inst_handle, + session_handle, + buf_list_src, + buf_list_dst, + &dc_results, + CPA_DC_FLUSH_FINAL, + &complete); + + if (CPA_STATUS_SUCCESS != status) { + status = CPA_STATUS_FAIL; + goto fail; + } + + /* we now wait until the completion of the operation. */ + if (!wait_for_completion_interruptible_timeout(&complete, + TIMEOUT_MS)) { + status = CPA_STATUS_FAIL; + goto fail; + } + + if (dc_results.status != CPA_STATUS_SUCCESS) { + status = CPA_STATUS_FAIL; + goto fail; + } + + *c_len = dc_results.produced; + + QAT_STAT_INCR(decomp_total_out_bytes, *c_len); + + ret = 0; + } + +fail: + if (status != CPA_STATUS_SUCCESS) { + QAT_STAT_BUMP(dc_fails); + } + + if (in_pages) { + for (page_num = 0; + page_num < buf_list_src->numBuffers; + page_num++) { + kunmap(in_pages[page_num]); + } + PHYS_CONTIG_FREE(in_pages); + } + + if (out_pages) { + for (page_num = 0; + page_num < buf_list_dst->numBuffers; + page_num++) { + kunmap(out_pages[page_num]); + } + PHYS_CONTIG_FREE(out_pages); + } + + PHYS_CONTIG_FREE(buffer_meta_src); + PHYS_CONTIG_FREE(buffer_meta_dst); + PHYS_CONTIG_FREE(buf_list_src); + PHYS_CONTIG_FREE(buf_list_dst); + + return (ret); +} + +module_param(zfs_qat_disable, int, 0644); +MODULE_PARM_DESC(zfs_qat_disable, "Disable QAT compression"); + +#endif diff --git a/module/zfs/qat_compress.h b/module/zfs/qat_compress.h new file mode 100644 index 000000000..ff074646f --- /dev/null +++ b/module/zfs/qat_compress.h @@ -0,0 +1,48 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#ifndef _SYS_QAT_COMPRESS_H +#define _SYS_QAT_COMPRESS_H + +#if defined(_KERNEL) && defined(HAVE_QAT) +#include <sys/zio.h> +#include "cpa.h" +#include "dc/cpa_dc.h" + +typedef enum qat_compress_dir { + QAT_COMPRESS = 0, + QAT_DECOMPRESS = 1, +} qat_compress_dir_t; + +extern int qat_init(void); +extern void qat_fini(void); +extern boolean_t qat_use_accel(size_t s_len); +extern int qat_compress(qat_compress_dir_t dir, char *src, int src_len, + char *dst, int dst_len, size_t *c_len); +#else +#define CPA_STATUS_SUCCESS 0 +#define qat_init() +#define qat_fini() +#define qat_use_accel(s_len) 0 +#define qat_compress(dir, s, sl, d, dl, cl) 0 +#endif + +#endif /* _SYS_QAT_COMPRESS_H */ diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index 230c52156..ea16e69f0 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -55,6 +55,7 @@ #include <sys/kstat.h> #include "zfs_prop.h" #include <sys/zfeature.h> +#include "qat_compress.h" /* * SPA locking @@ -1863,6 +1864,7 @@ spa_init(int mode) zpool_feature_init(); spa_config_load(); l2arc_start(); + qat_init(); } void @@ -1884,6 +1886,7 @@ spa_fini(void) unique_fini(); refcount_fini(); fm_fini(); + qat_fini(); avl_destroy(&spa_namespace_avl); avl_destroy(&spa_spare_avl); |