diff options
| -rw-r--r-- | include/zfs_fletcher.h | 26 | ||||
| -rw-r--r-- | lib/libzfs/libzfs_sendrecv.c | 2 | ||||
| -rw-r--r-- | module/zcommon/zfs_fletcher.c | 482 | ||||
| -rw-r--r-- | module/zcommon/zfs_fletcher_avx512.c | 10 | ||||
| -rw-r--r-- | module/zcommon/zfs_fletcher_intel.c | 10 | ||||
| -rw-r--r-- | module/zcommon/zfs_fletcher_sse.c | 18 | ||||
| -rw-r--r-- | module/zfs/dsl_dataset.c | 2 |
7 files changed, 372 insertions, 178 deletions
diff --git a/include/zfs_fletcher.h b/include/zfs_fletcher.h index ecba4ada7..f0cfbd573 100644 --- a/include/zfs_fletcher.h +++ b/include/zfs_fletcher.h @@ -35,11 +35,20 @@ extern "C" { /* * fletcher checksum functions + * + * Note: Fletcher checksum methods expect buffer size to be 4B aligned. This + * limitation stems from the algorithm design. Performing incremental checksum + * without said alignment would yield different results. Therefore, the code + * includes assertions for the size alignment. + * For compatibility, it is required that some code paths calculate checksum of + * non-aligned buffer sizes. For this purpose, `fletcher_4_native_varsize()` + * checksum method is added. This method will ignore last (size % 4) bytes of + * the data buffer. */ - void fletcher_2_native(const void *, uint64_t, zio_cksum_t *); void fletcher_2_byteswap(const void *, uint64_t, zio_cksum_t *); void fletcher_4_native(const void *, uint64_t, zio_cksum_t *); +void fletcher_4_native_varsize(const void *, uint64_t, zio_cksum_t *); void fletcher_4_byteswap(const void *, uint64_t, zio_cksum_t *); void fletcher_4_incremental_native(const void *, uint64_t, zio_cksum_t *); @@ -49,14 +58,21 @@ int fletcher_4_impl_set(const char *selector); void fletcher_4_init(void); void fletcher_4_fini(void); + /* * fletcher checksum struct */ +typedef void (*fletcher_4_init_f)(zio_cksum_t *); +typedef void (*fletcher_4_fini_f)(zio_cksum_t *); +typedef void (*fletcher_4_compute_f)(const void *, uint64_t, zio_cksum_t *); + typedef struct fletcher_4_func { - void (*init)(zio_cksum_t *); - void (*fini)(zio_cksum_t *); - void (*compute)(const void *, uint64_t, zio_cksum_t *); - void (*compute_byteswap)(const void *, uint64_t, zio_cksum_t *); + fletcher_4_init_f init_native; + fletcher_4_fini_f fini_native; + fletcher_4_compute_f compute_native; + fletcher_4_init_f init_byteswap; + fletcher_4_fini_f fini_byteswap; + fletcher_4_compute_f compute_byteswap; boolean_t (*valid)(void); const char *name; } fletcher_4_ops_t; diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 6adcc7a54..0d19c3bf4 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -1442,7 +1442,7 @@ zfs_send_resume_token_to_nvlist(libzfs_handle_t *hdl, const char *token) /* verify checksum */ zio_cksum_t cksum; - fletcher_4_native(compressed, len, &cksum); + fletcher_4_native_varsize(compressed, len, &cksum); if (cksum.zc_word[0] != checksum) { free(compressed); zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, diff --git a/module/zcommon/zfs_fletcher.c b/module/zcommon/zfs_fletcher.c index e8ba93433..221e586d8 100644 --- a/module/zcommon/zfs_fletcher.c +++ b/module/zcommon/zfs_fletcher.c @@ -21,6 +21,7 @@ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. */ /* @@ -129,25 +130,33 @@ #include <sys/sysmacros.h> #include <sys/byteorder.h> #include <sys/spa.h> +#include <sys/zio_checksum.h> #include <sys/zfs_context.h> #include <zfs_fletcher.h> + static void fletcher_4_scalar_init(zio_cksum_t *zcp); -static void fletcher_4_scalar(const void *buf, uint64_t size, +static void fletcher_4_scalar_native(const void *buf, uint64_t size, zio_cksum_t *zcp); static void fletcher_4_scalar_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp); static boolean_t fletcher_4_scalar_valid(void); static const fletcher_4_ops_t fletcher_4_scalar_ops = { - .init = fletcher_4_scalar_init, - .compute = fletcher_4_scalar, + .init_native = fletcher_4_scalar_init, + .compute_native = fletcher_4_scalar_native, + .init_byteswap = fletcher_4_scalar_init, .compute_byteswap = fletcher_4_scalar_byteswap, .valid = fletcher_4_scalar_valid, .name = "scalar" }; -static const fletcher_4_ops_t *fletcher_4_algos[] = { +static fletcher_4_ops_t fletcher_4_fastest_impl = { + .name = "fastest", + .valid = fletcher_4_scalar_valid +}; + +static const fletcher_4_ops_t *fletcher_4_impls[] = { &fletcher_4_scalar_ops, #if defined(HAVE_SSE2) &fletcher_4_sse2_ops, @@ -163,52 +172,39 @@ static const fletcher_4_ops_t *fletcher_4_algos[] = { #endif }; -static enum fletcher_selector { - FLETCHER_FASTEST = 0, - FLETCHER_SCALAR, -#if defined(HAVE_SSE2) - FLETCHER_SSE2, -#endif -#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) - FLETCHER_SSSE3, -#endif -#if defined(HAVE_AVX) && defined(HAVE_AVX2) - FLETCHER_AVX2, -#endif -#if defined(__x86_64) && defined(HAVE_AVX512F) - FLETCHER_AVX512F, -#endif - FLETCHER_CYCLE -} fletcher_4_impl_chosen = FLETCHER_SCALAR; +/* Hold all supported implementations */ +static uint32_t fletcher_4_supp_impls_cnt = 0; +static fletcher_4_ops_t *fletcher_4_supp_impls[ARRAY_SIZE(fletcher_4_impls)]; + +/* Select fletcher4 implementation */ +#define IMPL_FASTEST (UINT32_MAX) +#define IMPL_CYCLE (UINT32_MAX - 1) +#define IMPL_SCALAR (0) + +static uint32_t fletcher_4_impl_chosen = IMPL_FASTEST; + +#define IMPL_READ(i) (*(volatile uint32_t *) &(i)) static struct fletcher_4_impl_selector { - const char *fis_name; - const fletcher_4_ops_t *fis_ops; + const char *fis_name; + uint32_t fis_sel; } fletcher_4_impl_selectors[] = { - [ FLETCHER_FASTEST ] = { "fastest", NULL }, - [ FLETCHER_SCALAR ] = { "scalar", &fletcher_4_scalar_ops }, -#if defined(HAVE_SSE2) - [ FLETCHER_SSE2 ] = { "sse2", &fletcher_4_sse2_ops }, -#endif -#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) - [ FLETCHER_SSSE3 ] = { "ssse3", &fletcher_4_ssse3_ops }, -#endif -#if defined(HAVE_AVX) && defined(HAVE_AVX2) - [ FLETCHER_AVX2 ] = { "avx2", &fletcher_4_avx2_ops }, -#endif -#if defined(__x86_64) && defined(HAVE_AVX512F) - [ FLETCHER_AVX512F ] = { "avx512f", &fletcher_4_avx512f_ops }, -#endif #if !defined(_KERNEL) - [ FLETCHER_CYCLE ] = { "cycle", &fletcher_4_scalar_ops } + { "cycle", IMPL_CYCLE }, #endif + { "fastest", IMPL_FASTEST }, + { "scalar", IMPL_SCALAR } }; -static kmutex_t fletcher_4_impl_lock; - static kstat_t *fletcher_4_kstat; -static kstat_named_t fletcher_4_kstat_data[ARRAY_SIZE(fletcher_4_algos)]; +static struct fletcher_4_kstat { + uint64_t native; + uint64_t byteswap; +} fletcher_4_stat_data[ARRAY_SIZE(fletcher_4_impls) + 1]; + +/* Indicate that benchmark has been completed */ +static boolean_t fletcher_4_initialized = B_FALSE; void fletcher_2_native(const void *buf, uint64_t size, zio_cksum_t *zcp) @@ -244,13 +240,14 @@ fletcher_2_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp) ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1); } -static void fletcher_4_scalar_init(zio_cksum_t *zcp) +static void +fletcher_4_scalar_init(zio_cksum_t *zcp) { ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); } static void -fletcher_4_scalar(const void *buf, uint64_t size, zio_cksum_t *zcp) +fletcher_4_scalar_native(const void *buf, uint64_t size, zio_cksum_t *zcp) { const uint32_t *ip = buf; const uint32_t *ipend = ip + (size / sizeof (uint32_t)); @@ -302,180 +299,353 @@ fletcher_4_scalar_valid(void) int fletcher_4_impl_set(const char *val) { - const fletcher_4_ops_t *ops; - enum fletcher_selector idx = FLETCHER_FASTEST; - size_t val_len; - unsigned i; + int err = -EINVAL; + uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); + size_t i, val_len; val_len = strlen(val); while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */ val_len--; + /* check mandatory implementations */ for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) { const char *name = fletcher_4_impl_selectors[i].fis_name; if (val_len == strlen(name) && strncmp(val, name, val_len) == 0) { - idx = i; + impl = fletcher_4_impl_selectors[i].fis_sel; + err = 0; break; } } - if (i >= ARRAY_SIZE(fletcher_4_impl_selectors)) - return (-EINVAL); - ops = fletcher_4_impl_selectors[idx].fis_ops; - if (ops == NULL || !ops->valid()) - return (-ENOTSUP); + if (err != 0 && fletcher_4_initialized) { + /* check all supported implementations */ + for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { + const char *name = fletcher_4_supp_impls[i]->name; - mutex_enter(&fletcher_4_impl_lock); - if (fletcher_4_impl_chosen != idx) - fletcher_4_impl_chosen = idx; - mutex_exit(&fletcher_4_impl_lock); + if (val_len == strlen(name) && + strncmp(val, name, val_len) == 0) { + impl = i; + err = 0; + break; + } + } + } - return (0); + if (err == 0) { + atomic_swap_32(&fletcher_4_impl_chosen, impl); + membar_producer(); + } + + return (err); } static inline const fletcher_4_ops_t * fletcher_4_impl_get(void) { + fletcher_4_ops_t *ops = NULL; + const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); + + switch (impl) { + case IMPL_FASTEST: + ASSERT(fletcher_4_initialized); + ops = &fletcher_4_fastest_impl; + break; #if !defined(_KERNEL) - if (fletcher_4_impl_chosen == FLETCHER_CYCLE) { - static volatile unsigned int cycle_count = 0; - const fletcher_4_ops_t *ops = NULL; - unsigned int index; - - while (1) { - index = atomic_inc_uint_nv(&cycle_count); - ops = fletcher_4_algos[ - index % ARRAY_SIZE(fletcher_4_algos)]; - if (ops->valid()) - break; - } - return (ops); + case IMPL_CYCLE: { + ASSERT(fletcher_4_initialized); + ASSERT3U(fletcher_4_supp_impls_cnt, >, 0); + + static uint32_t cycle_count = 0; + uint32_t idx = (++cycle_count) % fletcher_4_supp_impls_cnt; + ops = fletcher_4_supp_impls[idx]; } + break; #endif - membar_producer(); - return (fletcher_4_impl_selectors[fletcher_4_impl_chosen].fis_ops); + default: + ASSERT3U(fletcher_4_supp_impls_cnt, >, 0); + ASSERT3U(impl, <, fletcher_4_supp_impls_cnt); + + ops = fletcher_4_supp_impls[impl]; + break; + } + + ASSERT3P(ops, !=, NULL); + + return (ops); +} + +void +fletcher_4_incremental_native(const void *buf, uint64_t size, + zio_cksum_t *zcp) +{ + ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); + + fletcher_4_scalar_native(buf, size, zcp); +} + +void +fletcher_4_incremental_byteswap(const void *buf, uint64_t size, + zio_cksum_t *zcp) +{ + ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); + + fletcher_4_scalar_byteswap(buf, size, zcp); +} + +static inline void +fletcher_4_native_impl(const fletcher_4_ops_t *ops, const void *buf, + uint64_t size, zio_cksum_t *zcp) +{ + ops->init_native(zcp); + ops->compute_native(buf, size, zcp); + if (ops->fini_native != NULL) + ops->fini_native(zcp); } void fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp) { const fletcher_4_ops_t *ops; + uint64_t p2size = P2ALIGN(size, 64); - if (IS_P2ALIGNED(size, 8 * sizeof (uint32_t))) - ops = fletcher_4_impl_get(); - else + ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); + + if (size == 0) { + ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); + } else if (p2size == 0) { ops = &fletcher_4_scalar_ops; + fletcher_4_native_impl(ops, buf, size, zcp); + } else { + ops = fletcher_4_impl_get(); + fletcher_4_native_impl(ops, buf, p2size, zcp); - ops->init(zcp); - ops->compute(buf, size, zcp); - if (ops->fini != NULL) - ops->fini(zcp); + if (p2size < size) + fletcher_4_incremental_native((char *)buf + p2size, + size - p2size, zcp); + } +} + +void +fletcher_4_native_varsize(const void *buf, uint64_t size, zio_cksum_t *zcp) +{ + fletcher_4_native_impl(&fletcher_4_scalar_ops, buf, size, zcp); +} + +static inline void +fletcher_4_byteswap_impl(const fletcher_4_ops_t *ops, const void *buf, + uint64_t size, zio_cksum_t *zcp) +{ + ops->init_byteswap(zcp); + ops->compute_byteswap(buf, size, zcp); + if (ops->fini_byteswap != NULL) + ops->fini_byteswap(zcp); } void fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp) { const fletcher_4_ops_t *ops; + uint64_t p2size = P2ALIGN(size, 64); - if (IS_P2ALIGNED(size, 8 * sizeof (uint32_t))) - ops = fletcher_4_impl_get(); - else + ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); + + if (size == 0) { + ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); + } else if (p2size == 0) { ops = &fletcher_4_scalar_ops; + fletcher_4_byteswap_impl(ops, buf, size, zcp); + } else { + ops = fletcher_4_impl_get(); + fletcher_4_byteswap_impl(ops, buf, p2size, zcp); - ops->init(zcp); - ops->compute_byteswap(buf, size, zcp); - if (ops->fini != NULL) - ops->fini(zcp); + if (p2size < size) + fletcher_4_incremental_byteswap((char *)buf + p2size, + size - p2size, zcp); + } } -void -fletcher_4_incremental_native(const void *buf, uint64_t size, - zio_cksum_t *zcp) +static int +fletcher_4_kstat_headers(char *buf, size_t size) { - fletcher_4_scalar(buf, size, zcp); + ssize_t off = 0; + + off += snprintf(buf + off, size, "%-17s", "implementation"); + off += snprintf(buf + off, size - off, "%-15s", "native"); + (void) snprintf(buf + off, size - off, "%-15s\n", "byteswap"); + + return (0); } -void -fletcher_4_incremental_byteswap(const void *buf, uint64_t size, - zio_cksum_t *zcp) +static int +fletcher_4_kstat_data(char *buf, size_t size, void *data) { - fletcher_4_scalar_byteswap(buf, size, zcp); + struct fletcher_4_kstat *fastest_stat = + &fletcher_4_stat_data[fletcher_4_supp_impls_cnt]; + struct fletcher_4_kstat *curr_stat = (struct fletcher_4_kstat *) data; + ssize_t off = 0; + + if (curr_stat == fastest_stat) { + off += snprintf(buf + off, size - off, "%-17s", "fastest"); + off += snprintf(buf + off, size - off, "%-15s", + fletcher_4_supp_impls[fastest_stat->native]->name); + off += snprintf(buf + off, size - off, "%-15s\n", + fletcher_4_supp_impls[fastest_stat->byteswap]->name); + } else { + ptrdiff_t id = curr_stat - fletcher_4_stat_data; + + off += snprintf(buf + off, size - off, "%-17s", + fletcher_4_supp_impls[id]->name); + off += snprintf(buf + off, size - off, "%-15llu", + (u_longlong_t) curr_stat->native); + off += snprintf(buf + off, size - off, "%-15llu\n", + (u_longlong_t) curr_stat->byteswap); + } + + return (0); } -void -fletcher_4_init(void) +static void * +fletcher_4_kstat_addr(kstat_t *ksp, loff_t n) { - const uint64_t const bench_ns = (50 * MICROSEC); /* 50ms */ - unsigned long best_run_count = 0; - unsigned long best_run_index = 0; - const unsigned data_size = 4096; - char *databuf; - int i; + if (n <= fletcher_4_supp_impls_cnt) + ksp->ks_private = (void *) (fletcher_4_stat_data + n); + else + ksp->ks_private = NULL; + + return (ksp->ks_private); +} + +#define FLETCHER_4_FASTEST_FN_COPY(type, src) \ +{ \ + fletcher_4_fastest_impl.init_ ## type = src->init_ ## type; \ + fletcher_4_fastest_impl.fini_ ## type = src->fini_ ## type; \ + fletcher_4_fastest_impl.compute_ ## type = src->compute_ ## type; \ +} + +#define FLETCHER_4_BENCH_NS (MSEC2NSEC(50)) /* 50ms */ - databuf = kmem_alloc(data_size, KM_SLEEP); - for (i = 0; i < ARRAY_SIZE(fletcher_4_algos); i++) { - const fletcher_4_ops_t *ops = fletcher_4_algos[i]; - kstat_named_t *stat = &fletcher_4_kstat_data[i]; - unsigned long run_count = 0; - hrtime_t start; - zio_cksum_t zc; +static void +fletcher_4_benchmark_impl(boolean_t native, char *data, uint64_t data_size) +{ + + struct fletcher_4_kstat *fastest_stat = + &fletcher_4_stat_data[fletcher_4_supp_impls_cnt]; + hrtime_t start; + uint64_t run_bw, run_time_ns, best_run = 0; + zio_cksum_t zc; + uint32_t i, l, sel_save = IMPL_READ(fletcher_4_impl_chosen); + + zio_checksum_func_t *fletcher_4_test = native ? fletcher_4_native : + fletcher_4_byteswap; - strncpy(stat->name, ops->name, sizeof (stat->name) - 1); - stat->data_type = KSTAT_DATA_UINT64; - stat->value.ui64 = 0; + for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { + struct fletcher_4_kstat *stat = &fletcher_4_stat_data[i]; + uint64_t run_count = 0; - if (!ops->valid()) - continue; + /* temporary set an implementation */ + fletcher_4_impl_chosen = i; kpreempt_disable(); start = gethrtime(); - ops->init(&zc); do { - ops->compute(databuf, data_size, &zc); - ops->compute_byteswap(databuf, data_size, &zc); - run_count++; - } while (gethrtime() < start + bench_ns); - if (ops->fini != NULL) - ops->fini(&zc); + for (l = 0; l < 32; l++, run_count++) + fletcher_4_test(data, data_size, &zc); + + run_time_ns = gethrtime() - start; + } while (run_time_ns < FLETCHER_4_BENCH_NS); kpreempt_enable(); - if (run_count > best_run_count) { - best_run_count = run_count; - best_run_index = i; + run_bw = data_size * run_count * NANOSEC; + run_bw /= run_time_ns; /* B/s */ + + if (native) + stat->native = run_bw; + else + stat->byteswap = run_bw; + + if (run_bw > best_run) { + best_run = run_bw; + + if (native) { + fastest_stat->native = i; + FLETCHER_4_FASTEST_FN_COPY(native, + fletcher_4_supp_impls[i]); + } else { + fastest_stat->byteswap = i; + FLETCHER_4_FASTEST_FN_COPY(byteswap, + fletcher_4_supp_impls[i]); + } } + } + + /* restore original selection */ + atomic_swap_32(&fletcher_4_impl_chosen, sel_save); +} - /* - * Due to high overhead of gethrtime(), the performance data - * here is inaccurate and much slower than it could be. - * It's fine for our use though because only relative speed - * is important. - */ - stat->value.ui64 = data_size * run_count * - (NANOSEC / bench_ns) >> 20; /* by MB/s */ +void +fletcher_4_init(void) +{ + static const size_t data_size = 1 << SPA_OLD_MAXBLOCKSHIFT; /* 128kiB */ + fletcher_4_ops_t *curr_impl; + char *databuf; + int i, c; + + /* move supported impl into fletcher_4_supp_impls */ + for (i = 0, c = 0; i < ARRAY_SIZE(fletcher_4_impls); i++) { + curr_impl = (fletcher_4_ops_t *) fletcher_4_impls[i]; + + if (curr_impl->valid && curr_impl->valid()) + fletcher_4_supp_impls[c++] = curr_impl; } - kmem_free(databuf, data_size); + membar_producer(); /* complete fletcher_4_supp_impls[] init */ + fletcher_4_supp_impls_cnt = c; /* number of supported impl */ - fletcher_4_impl_selectors[FLETCHER_FASTEST].fis_ops = - fletcher_4_algos[best_run_index]; +#if !defined(_KERNEL) + /* Skip benchmarking and use last implementation as fastest */ + memcpy(&fletcher_4_fastest_impl, + fletcher_4_supp_impls[fletcher_4_supp_impls_cnt-1], + sizeof (fletcher_4_fastest_impl)); + fletcher_4_fastest_impl.name = "fastest"; + membar_producer(); - mutex_init(&fletcher_4_impl_lock, NULL, MUTEX_DEFAULT, NULL); - fletcher_4_impl_set("fastest"); + fletcher_4_initialized = B_TRUE; - fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench", - "misc", KSTAT_TYPE_NAMED, ARRAY_SIZE(fletcher_4_algos), - KSTAT_FLAG_VIRTUAL); + /* Use 'cycle' math selection method for userspace */ + VERIFY0(fletcher_4_impl_set("cycle")); + return; +#endif + /* Benchmark all supported implementations */ + databuf = vmem_alloc(data_size, KM_SLEEP); + for (i = 0; i < data_size / sizeof (uint64_t); i++) + ((uint64_t *)databuf)[i] = (uintptr_t)(databuf+i); /* warm-up */ + + fletcher_4_benchmark_impl(B_FALSE, databuf, data_size); + fletcher_4_benchmark_impl(B_TRUE, databuf, data_size); + + vmem_free(databuf, data_size); + + /* install kstats for all implementations */ + fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench", "misc", + KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); if (fletcher_4_kstat != NULL) { - fletcher_4_kstat->ks_data = fletcher_4_kstat_data; + fletcher_4_kstat->ks_data = NULL; + fletcher_4_kstat->ks_ndata = UINT32_MAX; + kstat_set_raw_ops(fletcher_4_kstat, + fletcher_4_kstat_headers, + fletcher_4_kstat_data, + fletcher_4_kstat_addr); kstat_install(fletcher_4_kstat); } + + /* Finish initialization */ + fletcher_4_initialized = B_TRUE; } void fletcher_4_fini(void) { - mutex_destroy(&fletcher_4_impl_lock); if (fletcher_4_kstat != NULL) { kstat_delete(fletcher_4_kstat); fletcher_4_kstat = NULL; @@ -487,18 +657,19 @@ fletcher_4_fini(void) static int fletcher_4_param_get(char *buffer, struct kernel_param *unused) { + const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); + char *fmt; int i, cnt = 0; - for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) { - const fletcher_4_ops_t *ops; - - ops = fletcher_4_impl_selectors[i].fis_ops; - if (!ops->valid()) - continue; + /* list fastest */ + fmt = (impl == IMPL_FASTEST) ? "[%s] " : "%s "; + cnt += sprintf(buffer + cnt, fmt, "fastest"); - cnt += sprintf(buffer + cnt, - fletcher_4_impl_chosen == i ? "[%s] " : "%s ", - fletcher_4_impl_selectors[i].fis_name); + /* list all supported implementations */ + for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { + fmt = (i == impl) ? "[%s] " : "%s "; + cnt += sprintf(buffer + cnt, fmt, + fletcher_4_supp_impls[i]->name); } return (cnt); @@ -512,20 +683,19 @@ fletcher_4_param_set(const char *val, struct kernel_param *unused) /* * Choose a fletcher 4 implementation in ZFS. - * Users can choose the "fastest" algorithm, or "scalar" and "avx2" which means - * to compute fletcher 4 by CPU or vector instructions respectively. - * Users can also choose "cycle" to exercise all implementions, but this is + * Users can choose "cycle" to exercise all implementations, but this is * for testing purpose therefore it can only be set in user space. */ module_param_call(zfs_fletcher_4_impl, fletcher_4_param_set, fletcher_4_param_get, NULL, 0644); -MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 algorithm"); +MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 implementation."); EXPORT_SYMBOL(fletcher_4_init); EXPORT_SYMBOL(fletcher_4_fini); EXPORT_SYMBOL(fletcher_2_native); EXPORT_SYMBOL(fletcher_2_byteswap); EXPORT_SYMBOL(fletcher_4_native); +EXPORT_SYMBOL(fletcher_4_native_varsize); EXPORT_SYMBOL(fletcher_4_byteswap); EXPORT_SYMBOL(fletcher_4_incremental_native); EXPORT_SYMBOL(fletcher_4_incremental_byteswap); diff --git a/module/zcommon/zfs_fletcher_avx512.c b/module/zcommon/zfs_fletcher_avx512.c index 488b9f3cf..22e1f410f 100644 --- a/module/zcommon/zfs_fletcher_avx512.c +++ b/module/zcommon/zfs_fletcher_avx512.c @@ -48,7 +48,7 @@ fletcher_4_avx512f_init(zio_cksum_t *zcp) } static void -fletcher_4_avx512f(const void *buf, uint64_t size, zio_cksum_t *unused) +fletcher_4_avx512f_native(const void *buf, uint64_t size, zio_cksum_t *unused) { const uint32_t *ip = buf; const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); @@ -146,9 +146,11 @@ fletcher_4_avx512f_valid(void) } const fletcher_4_ops_t fletcher_4_avx512f_ops = { - .init = fletcher_4_avx512f_init, - .fini = fletcher_4_avx512f_fini, - .compute = fletcher_4_avx512f, + .init_native = fletcher_4_avx512f_init, + .fini_native = fletcher_4_avx512f_fini, + .compute_native = fletcher_4_avx512f_native, + .init_byteswap = fletcher_4_avx512f_init, + .fini_byteswap = fletcher_4_avx512f_fini, .compute_byteswap = fletcher_4_avx512f_byteswap, .valid = fletcher_4_avx512f_valid, .name = "avx512f" diff --git a/module/zcommon/zfs_fletcher_intel.c b/module/zcommon/zfs_fletcher_intel.c index 38a40e234..adc4151c5 100644 --- a/module/zcommon/zfs_fletcher_intel.c +++ b/module/zcommon/zfs_fletcher_intel.c @@ -92,7 +92,7 @@ fletcher_4_avx2_fini(zio_cksum_t *zcp) } static void -fletcher_4_avx2(const void *buf, uint64_t size, zio_cksum_t *unused) +fletcher_4_avx2_native(const void *buf, uint64_t size, zio_cksum_t *unused) { const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); @@ -137,9 +137,11 @@ static boolean_t fletcher_4_avx2_valid(void) } const fletcher_4_ops_t fletcher_4_avx2_ops = { - .init = fletcher_4_avx2_init, - .fini = fletcher_4_avx2_fini, - .compute = fletcher_4_avx2, + .init_native = fletcher_4_avx2_init, + .fini_native = fletcher_4_avx2_fini, + .compute_native = fletcher_4_avx2_native, + .init_byteswap = fletcher_4_avx2_init, + .fini_byteswap = fletcher_4_avx2_fini, .compute_byteswap = fletcher_4_avx2_byteswap, .valid = fletcher_4_avx2_valid, .name = "avx2" diff --git a/module/zcommon/zfs_fletcher_sse.c b/module/zcommon/zfs_fletcher_sse.c index 2a4e6a3f2..9bc5f7ab6 100644 --- a/module/zcommon/zfs_fletcher_sse.c +++ b/module/zcommon/zfs_fletcher_sse.c @@ -97,7 +97,7 @@ fletcher_4_sse2_fini(zio_cksum_t *zcp) } static void -fletcher_4_sse2(const void *buf, uint64_t size, zio_cksum_t *unused) +fletcher_4_sse2_native(const void *buf, uint64_t size, zio_cksum_t *unused) { const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); @@ -147,9 +147,11 @@ static boolean_t fletcher_4_sse2_valid(void) } const fletcher_4_ops_t fletcher_4_sse2_ops = { - .init = fletcher_4_sse2_init, - .fini = fletcher_4_sse2_fini, - .compute = fletcher_4_sse2, + .init_native = fletcher_4_sse2_init, + .fini_native = fletcher_4_sse2_fini, + .compute_native = fletcher_4_sse2_native, + .init_byteswap = fletcher_4_sse2_init, + .fini_byteswap = fletcher_4_sse2_fini, .compute_byteswap = fletcher_4_sse2_byteswap, .valid = fletcher_4_sse2_valid, .name = "sse2" @@ -194,9 +196,11 @@ static boolean_t fletcher_4_ssse3_valid(void) } const fletcher_4_ops_t fletcher_4_ssse3_ops = { - .init = fletcher_4_sse2_init, - .fini = fletcher_4_sse2_fini, - .compute = fletcher_4_sse2, + .init_native = fletcher_4_sse2_init, + .fini_native = fletcher_4_sse2_fini, + .compute_native = fletcher_4_sse2_native, + .init_byteswap = fletcher_4_sse2_init, + .fini_byteswap = fletcher_4_sse2_fini, .compute_byteswap = fletcher_4_ssse3_byteswap, .valid = fletcher_4_ssse3_valid, .name = "ssse3" diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c index 5d7847d46..dd390d49a 100644 --- a/module/zfs/dsl_dataset.c +++ b/module/zfs/dsl_dataset.c @@ -1770,7 +1770,7 @@ get_receive_resume_stats(dsl_dataset_t *ds, nvlist_t *nv) compressed_size = gzip_compress(packed, compressed, packed_size, packed_size, 6); - fletcher_4_native(compressed, compressed_size, &cksum); + fletcher_4_native_varsize(compressed, compressed_size, &cksum); str = kmem_alloc(compressed_size * 2 + 1, KM_SLEEP); for (i = 0; i < compressed_size; i++) { |