diff options
| -rw-r--r-- | cmd/ztest/ztest.c | 28 | ||||
| -rw-r--r-- | include/sys/zio_checksum.h | 27 | ||||
| -rw-r--r-- | module/zcommon/zfs_fletcher.c | 87 | ||||
| -rw-r--r-- | module/zfs/zio_checksum.c | 33 |
4 files changed, 167 insertions, 8 deletions
diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c index d1c6a783a..0ef834403 100644 --- a/cmd/ztest/ztest.c +++ b/cmd/ztest/ztest.c @@ -5680,6 +5680,7 @@ ztest_fletcher(ztest_ds_t *zd, uint64_t id) while (gethrtime() <= end) { int run_count = 100; void *buf; + struct abd *abd_data, *abd_meta; uint32_t size; int *ptr; int i; @@ -5687,11 +5688,17 @@ ztest_fletcher(ztest_ds_t *zd, uint64_t id) zio_cksum_t zc_ref_byteswap; size = ztest_random_blocksize(); + buf = umem_alloc(size, UMEM_NOFAIL); + abd_data = abd_alloc(size, B_FALSE); + abd_meta = abd_alloc(size, B_TRUE); for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++) *ptr = ztest_random(UINT_MAX); + abd_copy_from_buf_off(abd_data, buf, 0, size); + abd_copy_from_buf_off(abd_meta, buf, 0, size); + VERIFY0(fletcher_4_impl_set("scalar")); fletcher_4_native(buf, size, NULL, &zc_ref); fletcher_4_byteswap(buf, size, NULL, &zc_ref_byteswap); @@ -5707,9 +5714,30 @@ ztest_fletcher(ztest_ds_t *zd, uint64_t id) VERIFY0(bcmp(&zc, &zc_ref, sizeof (zc))); VERIFY0(bcmp(&zc_byteswap, &zc_ref_byteswap, sizeof (zc_byteswap))); + + /* Test ABD - data */ + abd_fletcher_4_byteswap(abd_data, size, NULL, + &zc_byteswap); + abd_fletcher_4_native(abd_data, size, NULL, &zc); + + VERIFY0(bcmp(&zc, &zc_ref, sizeof (zc))); + VERIFY0(bcmp(&zc_byteswap, &zc_ref_byteswap, + sizeof (zc_byteswap))); + + /* Test ABD - metadata */ + abd_fletcher_4_byteswap(abd_meta, size, NULL, + &zc_byteswap); + abd_fletcher_4_native(abd_meta, size, NULL, &zc); + + VERIFY0(bcmp(&zc, &zc_ref, sizeof (zc))); + VERIFY0(bcmp(&zc_byteswap, &zc_ref_byteswap, + sizeof (zc_byteswap))); + } umem_free(buf, size); + abd_free(abd_data); + abd_free(abd_meta); } } diff --git a/include/sys/zio_checksum.h b/include/sys/zio_checksum.h index a6cafc9b2..60fd12b06 100644 --- a/include/sys/zio_checksum.h +++ b/include/sys/zio_checksum.h @@ -29,6 +29,7 @@ #include <sys/zio.h> #include <zfeature_common.h> +#include <zfs_fletcher.h> #ifdef __cplusplus extern "C" { @@ -57,6 +58,28 @@ typedef enum zio_checksum_flags { ZCHECKSUM_FLAG_NOPWRITE = (1 << 5) } zio_checksum_flags_t; +typedef enum { + ZIO_CHECKSUM_NATIVE, + ZIO_CHECKSUM_BYTESWAP +} zio_byteorder_t; + +typedef struct zio_abd_checksum_data { + zio_byteorder_t acd_byteorder; + fletcher_4_ctx_t *acd_ctx; + zio_cksum_t *acd_zcp; + void *acd_private; +} zio_abd_checksum_data_t; + +typedef void zio_abd_checksum_init_t(zio_abd_checksum_data_t *); +typedef void zio_abd_checksum_fini_t(zio_abd_checksum_data_t *); +typedef int zio_abd_checksum_iter_t(void *, size_t, void *); + +typedef const struct zio_abd_checksum_func { + zio_abd_checksum_init_t *acf_init; + zio_abd_checksum_fini_t *acf_fini; + zio_abd_checksum_iter_t *acf_iter; +} zio_abd_checksum_func_t; + /* * Information about each checksum function. */ @@ -99,6 +122,10 @@ extern zio_checksum_t abd_checksum_edonr_byteswap; extern zio_checksum_tmpl_init_t abd_checksum_edonr_tmpl_init; extern zio_checksum_tmpl_free_t abd_checksum_edonr_tmpl_free; +extern zio_abd_checksum_func_t fletcher_4_abd_ops; +extern zio_checksum_t abd_fletcher_4_native; +extern zio_checksum_t abd_fletcher_4_byteswap; + extern int zio_checksum_equal(spa_t *, blkptr_t *, enum zio_checksum, void *, uint64_t, uint64_t, zio_bad_cksum_t *); extern void zio_checksum_compute(zio_t *, enum zio_checksum, diff --git a/module/zcommon/zfs_fletcher.c b/module/zcommon/zfs_fletcher.c index 1d1ea2e24..9cdf36d44 100644 --- a/module/zcommon/zfs_fletcher.c +++ b/module/zcommon/zfs_fletcher.c @@ -141,6 +141,7 @@ #include <sys/zfs_context.h> #include <zfs_fletcher.h> +#define FLETCHER_MIN_SIMD_SIZE 64 static void fletcher_4_scalar_init(fletcher_4_ctx_t *ctx); static void fletcher_4_scalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp); @@ -456,7 +457,7 @@ void fletcher_4_native(const void *buf, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { - const uint64_t p2size = P2ALIGN(size, 64); + const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE); ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); @@ -498,7 +499,7 @@ void fletcher_4_byteswap(const void *buf, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { - const uint64_t p2size = P2ALIGN(size, 64); + const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE); ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); @@ -778,6 +779,87 @@ fletcher_4_fini(void) } } +/* ABD adapters */ + +static void +abd_fletcher_4_init(zio_abd_checksum_data_t *cdp) +{ + const fletcher_4_ops_t *ops = fletcher_4_impl_get(); + cdp->acd_private = (void *) ops; + + if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE) + ops->init_native(cdp->acd_ctx); + else + ops->init_byteswap(cdp->acd_ctx); +} + +static void +abd_fletcher_4_fini(zio_abd_checksum_data_t *cdp) +{ + fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private; + + ASSERT(ops); + + if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE) + ops->fini_native(cdp->acd_ctx, cdp->acd_zcp); + else + ops->fini_byteswap(cdp->acd_ctx, cdp->acd_zcp); +} + +static void +abd_fletcher_4_simd2scalar(boolean_t native, void *data, size_t size, + zio_abd_checksum_data_t *cdp) +{ + zio_cksum_t *zcp = cdp->acd_zcp; + + ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE); + + abd_fletcher_4_fini(cdp); + cdp->acd_private = (void *)&fletcher_4_scalar_ops; + + if (native) + fletcher_4_incremental_native(data, size, zcp); + else + fletcher_4_incremental_byteswap(data, size, zcp); +} + +static int +abd_fletcher_4_iter(void *data, size_t size, void *private) +{ + zio_abd_checksum_data_t *cdp = (zio_abd_checksum_data_t *)private; + fletcher_4_ctx_t *ctx = cdp->acd_ctx; + fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private; + boolean_t native = cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE; + uint64_t asize = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE); + + ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); + + if (asize > 0) { + if (native) + ops->compute_native(ctx, data, asize); + else + ops->compute_byteswap(ctx, data, asize); + + size -= asize; + data = (char *)data + asize; + } + + if (size > 0) { + ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE); + /* At this point we have to switch to scalar impl */ + abd_fletcher_4_simd2scalar(native, data, size, cdp); + } + + return (0); +} + +zio_abd_checksum_func_t fletcher_4_abd_ops = { + .acf_init = abd_fletcher_4_init, + .acf_fini = abd_fletcher_4_fini, + .acf_iter = abd_fletcher_4_iter +}; + + #if defined(_KERNEL) && defined(HAVE_SPL) #include <linux/mod_compat.h> @@ -829,4 +911,5 @@ EXPORT_SYMBOL(fletcher_4_native_varsize); EXPORT_SYMBOL(fletcher_4_byteswap); EXPORT_SYMBOL(fletcher_4_incremental_native); EXPORT_SYMBOL(fletcher_4_incremental_byteswap); +EXPORT_SYMBOL(fletcher_4_abd_ops); #endif diff --git a/module/zfs/zio_checksum.c b/module/zfs/zio_checksum.c index e65b3f061..703d9a5e5 100644 --- a/module/zfs/zio_checksum.c +++ b/module/zfs/zio_checksum.c @@ -119,14 +119,29 @@ abd_fletcher_2_byteswap(abd_t *abd, uint64_t size, fletcher_2_incremental_byteswap, zcp); } +static inline void +abd_fletcher_4_impl(abd_t *abd, uint64_t size, zio_abd_checksum_data_t *acdp) +{ + fletcher_4_abd_ops.acf_init(acdp); + abd_iterate_func(abd, 0, size, fletcher_4_abd_ops.acf_iter, acdp); + fletcher_4_abd_ops.acf_fini(acdp); +} + /*ARGSUSED*/ void abd_fletcher_4_native(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { - fletcher_init(zcp); - (void) abd_iterate_func(abd, 0, size, - fletcher_4_incremental_native, zcp); + fletcher_4_ctx_t ctx; + + zio_abd_checksum_data_t acd = { + .acd_byteorder = ZIO_CHECKSUM_NATIVE, + .acd_zcp = zcp, + .acd_ctx = &ctx + }; + + abd_fletcher_4_impl(abd, size, &acd); + } /*ARGSUSED*/ @@ -134,9 +149,15 @@ void abd_fletcher_4_byteswap(abd_t *abd, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { - fletcher_init(zcp); - (void) abd_iterate_func(abd, 0, size, - fletcher_4_incremental_byteswap, zcp); + fletcher_4_ctx_t ctx; + + zio_abd_checksum_data_t acd = { + .acd_byteorder = ZIO_CHECKSUM_BYTESWAP, + .acd_zcp = zcp, + .acd_ctx = &ctx + }; + + abd_fletcher_4_impl(abd, size, &acd); } zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS] = { |