diff options
| -rw-r--r-- | cmd/raidz_test/raidz_test.h | 2 | ||||
| -rw-r--r-- | include/sys/vdev_raidz_impl.h | 6 | ||||
| -rw-r--r-- | lib/libzpool/Makefile.am | 2 | ||||
| -rw-r--r-- | man/man5/zfs-module-parameters.5 | 2 | ||||
| -rw-r--r-- | module/zfs/Makefile.in | 2 | ||||
| -rw-r--r-- | module/zfs/vdev_raidz_math.c | 6 | ||||
| -rw-r--r-- | module/zfs/vdev_raidz_math_avx512bw.c | 428 | ||||
| -rw-r--r-- | module/zfs/vdev_raidz_math_avx512f.c | 523 |
8 files changed, 971 insertions, 0 deletions
diff --git a/cmd/raidz_test/raidz_test.h b/cmd/raidz_test/raidz_test.h index 89bbf413b..b279d82f2 100644 --- a/cmd/raidz_test/raidz_test.h +++ b/cmd/raidz_test/raidz_test.h @@ -34,6 +34,8 @@ static const char *raidz_impl_names[] = { "sse2", "ssse3", "avx2", + "avx512f", + "avx512bw", "aarch64_neon", "aarch64_neonx2", NULL diff --git a/include/sys/vdev_raidz_impl.h b/include/sys/vdev_raidz_impl.h index ca902f15d..735b67764 100644 --- a/include/sys/vdev_raidz_impl.h +++ b/include/sys/vdev_raidz_impl.h @@ -141,6 +141,12 @@ extern const raidz_impl_ops_t vdev_raidz_ssse3_impl; #if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */ extern const raidz_impl_ops_t vdev_raidz_avx2_impl; #endif +#if defined(__x86_64) && defined(HAVE_AVX512F) /* only x86_64 for now */ +extern const raidz_impl_ops_t vdev_raidz_avx512f_impl; +#endif +#if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */ +extern const raidz_impl_ops_t vdev_raidz_avx512bw_impl; +#endif #if defined(__aarch64__) extern const raidz_impl_ops_t vdev_raidz_aarch64_neon_impl; extern const raidz_impl_ops_t vdev_raidz_aarch64_neonx2_impl; diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am index 13353a58a..ac1a78208 100644 --- a/lib/libzpool/Makefile.am +++ b/lib/libzpool/Makefile.am @@ -102,6 +102,8 @@ KERNEL_C = \ vdev_raidz_math_sse2.c \ vdev_raidz_math_ssse3.c \ vdev_raidz_math_avx2.c \ + vdev_raidz_math_avx512f.c \ + vdev_raidz_math_avx512bw.c \ vdev_raidz_math_aarch64_neon.c \ vdev_raidz_math_aarch64_neonx2.c \ vdev_root.c \ diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5 index 8e37a97e2..dc6601581 100644 --- a/man/man5/zfs-module-parameters.5 +++ b/man/man5/zfs-module-parameters.5 @@ -1775,6 +1775,8 @@ Possible options are: sse2 - implementation using SSE2 instruction set (64bit x86 only) ssse3 - implementation using SSSE3 instruction set (64bit x86 only) avx2 - implementation using AVX2 instruction set (64bit x86 only) + avx512f - implementation using AVX512F instruction set (64bit x86 only) + avx512bw - implementation using AVX512F & AVX512BW instruction sets (64bit x86 only) aarch64_neon - implementation using NEON (Aarch64/64 bit ARMv8 only) aarch64_neonx2 - implementation using NEON with more unrolling (Aarch64/64 bit ARMv8 only) .sp diff --git a/module/zfs/Makefile.in b/module/zfs/Makefile.in index ce368880c..5ad319f32 100644 --- a/module/zfs/Makefile.in +++ b/module/zfs/Makefile.in @@ -117,6 +117,8 @@ $(MODULE)-objs += dsl_userhold.o $(MODULE)-$(CONFIG_X86) += vdev_raidz_math_sse2.o $(MODULE)-$(CONFIG_X86) += vdev_raidz_math_ssse3.o $(MODULE)-$(CONFIG_X86) += vdev_raidz_math_avx2.o +$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_avx512f.o +$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_avx512bw.o $(MODULE)-$(CONFIG_ARM64) += vdev_raidz_math_aarch64_neon.o $(MODULE)-$(CONFIG_ARM64) += vdev_raidz_math_aarch64_neonx2.o diff --git a/module/zfs/vdev_raidz_math.c b/module/zfs/vdev_raidz_math.c index c22d54418..33c05dadd 100644 --- a/module/zfs/vdev_raidz_math.c +++ b/module/zfs/vdev_raidz_math.c @@ -57,6 +57,12 @@ const raidz_impl_ops_t *raidz_all_maths[] = { #if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */ &vdev_raidz_avx2_impl, #endif +#if defined(__x86_64) && defined(HAVE_AVX512F) /* only x86_64 for now */ + &vdev_raidz_avx512f_impl, +#endif +#if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */ + &vdev_raidz_avx512bw_impl, +#endif #if defined(__aarch64__) &vdev_raidz_aarch64_neon_impl, &vdev_raidz_aarch64_neonx2_impl, diff --git a/module/zfs/vdev_raidz_math_avx512bw.c b/module/zfs/vdev_raidz_math_avx512bw.c new file mode 100644 index 000000000..bcbe657d0 --- /dev/null +++ b/module/zfs/vdev_raidz_math_avx512bw.c @@ -0,0 +1,428 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2016 Romain Dolbeau. All rights reserved. + */ + +#include <sys/isa_defs.h> + +#if defined(__x86_64) && defined(HAVE_AVX512BW) + +#include <sys/types.h> +#include <linux/simd_x86.h> + +#define __asm __asm__ __volatile__ + +#define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N +#define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) + +#define VR0_(REG, ...) "zmm"#REG +#define VR1_(_1, REG, ...) "zmm"#REG +#define VR2_(_1, _2, REG, ...) "zmm"#REG +#define VR3_(_1, _2, _3, REG, ...) "zmm"#REG +#define VR4_(_1, _2, _3, _4, REG, ...) "zmm"#REG +#define VR5_(_1, _2, _3, _4, _5, REG, ...) "zmm"#REG +#define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "zmm"#REG +#define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "zmm"#REG + +#define VR0(r...) VR0_(r) +#define VR1(r...) VR1_(r) +#define VR2(r...) VR2_(r, 1) +#define VR3(r...) VR3_(r, 1, 2) +#define VR4(r...) VR4_(r, 1, 2) +#define VR5(r...) VR5_(r, 1, 2, 3) +#define VR6(r...) VR6_(r, 1, 2, 3, 4) +#define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) + +#define R_01(REG1, REG2, ...) REG1, REG2 +#define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 +#define R_23(REG...) _R_23(REG, 1, 2, 3) + +#define ASM_BUG() ASSERT(0) + +extern const uint8_t gf_clmul_mod_lt[4*256][16]; + +#define ELEM_SIZE 64 + +typedef struct v { + uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); +} v_t; + +#define PREFETCHNTA(ptr, offset) \ +{ \ + __asm( \ + "prefetchnta " #offset "(%[MEM])\n" \ + : : [MEM] "r" (ptr)); \ +} + +#define PREFETCH(ptr, offset) \ +{ \ + __asm( \ + "prefetcht0 " #offset "(%[MEM])\n" \ + : : [MEM] "r" (ptr)); \ +} + +#define XOR_ACC(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ + "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ + "vpxorq 0x80(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ + "vpxorq 0xc0(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ + "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define XOR(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "vpxorq %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ + "vpxorq %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ + "vpxorq %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ + "vpxorq %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "vpxorq %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ + "vpxorq %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define ZERO(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vpxorq %" VR0(r) ", %" VR0(r)", %" VR0(r) "\n" \ + "vpxorq %" VR1(r) ", %" VR1(r)", %" VR1(r) "\n" \ + "vpxorq %" VR2(r) ", %" VR2(r)", %" VR2(r) "\n" \ + "vpxorq %" VR3(r) ", %" VR3(r)", %" VR3(r)); \ + break; \ + case 2: \ + __asm( \ + "vpxorq %" VR0(r) ", %" VR0(r)", %" VR0(r) "\n" \ + "vpxorq %" VR1(r) ", %" VR1(r)", %" VR1(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define COPY(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "vmovdqa64 %" VR0(r) ", %" VR4(r) "\n" \ + "vmovdqa64 %" VR1(r) ", %" VR5(r) "\n" \ + "vmovdqa64 %" VR2(r) ", %" VR6(r) "\n" \ + "vmovdqa64 %" VR3(r) ", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "vmovdqa64 %" VR0(r) ", %" VR2(r) "\n" \ + "vmovdqa64 %" VR1(r) ", %" VR3(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define LOAD(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ + "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ + "vmovdqa64 0x80(%[SRC]), %%" VR2(r) "\n" \ + "vmovdqa64 0xc0(%[SRC]), %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ + "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define STORE(dst, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ + "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ + "vmovdqa64 %%" VR2(r) ", 0x80(%[DST])\n" \ + "vmovdqa64 %%" VR3(r) ", 0xc0(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + case 2: \ + __asm( \ + "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ + "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define FLUSH() \ +{ \ + __asm("vzeroupper"); \ +} + +#define MUL2_SETUP() \ +{ \ + __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \ + __asm("vpbroadcastq %xmm14, %zmm14"); \ + __asm("vmovq %0, %%xmm13" :: "r"(0x8080808080808080)); \ + __asm("vpbroadcastq %xmm13, %zmm13"); \ + __asm("vmovq %0, %%xmm12" :: "r"(0xfefefefefefefefe)); \ + __asm("vpbroadcastq %xmm12, %zmm12"); \ + __asm("vpxorq %zmm15, %zmm15 ,%zmm15"); \ +} + +#define _MUL2(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + __asm( \ + "vpandq %" VR0(r)", %zmm13, %zmm10\n" \ + "vpandq %" VR1(r)", %zmm13, %zmm11\n" \ + "vpsrlq $7, %zmm10, %zmm8\n" \ + "vpsrlq $7, %zmm11, %zmm9\n" \ + "vpsllq $1, %zmm10, %zmm10\n" \ + "vpsllq $1, %zmm11, %zmm11\n" \ + "vpsubq %zmm8, %zmm10, %zmm10\n" \ + "vpsubq %zmm9, %zmm11, %zmm11\n" \ + "vpsllq $1, %" VR0(r)", %" VR0(r) "\n" \ + "vpsllq $1, %" VR1(r)", %" VR1(r) "\n" \ + "vpandq %zmm10, %zmm14, %zmm10\n" \ + "vpandq %zmm11, %zmm14, %zmm11\n" \ + "vpternlogd $0x6c,%zmm12, %zmm10, %" VR0(r) "\n" \ + "vpternlogd $0x6c,%zmm12, %zmm11, %" VR1(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL2(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + _MUL2(R_01(r)); \ + _MUL2(R_23(r)); \ + break; \ + case 2: \ + _MUL2(r); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL4(r...) \ +{ \ + MUL2(r); \ + MUL2(r); \ +} + +#define _0f "zmm15" +#define _as "zmm14" +#define _bs "zmm13" +#define _ltmod "zmm12" +#define _ltmul "zmm11" +#define _ta "zmm10" +#define _tb "zmm15" + +static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F; + +#define _MULx2(c, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + __asm( \ + "vpbroadcastb (%[mask]), %%" _0f "\n" \ + /* upper bits */ \ + "vbroadcasti32x4 0x00(%[lt]), %%" _ltmod "\n" \ + "vbroadcasti32x4 0x10(%[lt]), %%" _ltmul "\n" \ + \ + "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \ + "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \ + "vpandq %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \ + "vpandq %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \ + "vpandq %%" _0f ", %%" _as ", %%" _as "\n" \ + "vpandq %%" _0f ", %%" _bs ", %%" _bs "\n" \ + \ + "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \ + "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \ + "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \ + "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \ + /* lower bits */ \ + "vbroadcasti32x4 0x20(%[lt]), %%" _ltmod "\n" \ + "vbroadcasti32x4 0x30(%[lt]), %%" _ltmul "\n" \ + \ + "vpxorq %%" _ta ", %%" _as ", %%" _as "\n" \ + "vpxorq %%" _tb ", %%" _bs ", %%" _bs "\n" \ + \ + "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \ + "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \ + "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\ + "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\ + \ + "vpxorq %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \ + "vpxorq %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \ + "vpxorq %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \ + "vpxorq %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \ + : : [mask] "r" (&_mul_mask), \ + [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL(c, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + _MULx2(c, R_01(r)); \ + _MULx2(c, R_23(r)); \ + break; \ + case 2: \ + _MULx2(c, R_01(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define raidz_math_begin() kfpu_begin() +#define raidz_math_end() \ +{ \ + FLUSH(); \ + kfpu_end(); \ +} + +#define GEN_P_DEFINE() {} +#define GEN_P_STRIDE 4 +#define GEN_P_P 0, 1, 2, 3 + +#define GEN_PQ_DEFINE() {} +#define GEN_PQ_STRIDE 4 +#define GEN_PQ_D 0, 1, 2, 3 +#define GEN_PQ_P 4, 5, 6, 7 +#define GEN_PQ_Q 20, 21, 22, 23 + +#define GEN_PQR_DEFINE() {} +#define GEN_PQR_STRIDE 2 +#define GEN_PQR_D 0, 1 +#define GEN_PQR_P 2, 3 +#define GEN_PQR_Q 4, 5 +#define GEN_PQR_R 6, 7 + +#define REC_P_DEFINE() {} +#define REC_P_STRIDE 4 +#define REC_P_X 0, 1, 2, 3 + +#define REC_Q_DEFINE() {} +#define REC_Q_STRIDE 4 +#define REC_Q_X 0, 1, 2, 3 + +#define REC_R_DEFINE() {} +#define REC_R_STRIDE 4 +#define REC_R_X 0, 1, 2, 3 + +#define REC_PQ_DEFINE() {} +#define REC_PQ_STRIDE 4 +#define REC_PQ_X 0, 1, 2, 3 +#define REC_PQ_Y 4, 5, 6, 7 +#define REC_PQ_D 20, 21, 22, 23 + +#define REC_PR_DEFINE() {} +#define REC_PR_STRIDE 4 +#define REC_PR_X 0, 1, 2, 3 +#define REC_PR_Y 4, 5, 6, 7 +#define REC_PR_D 20, 21, 22, 23 + +#define REC_QR_DEFINE() {} +#define REC_QR_STRIDE 4 +#define REC_QR_X 0, 1, 2, 3 +#define REC_QR_Y 4, 5, 6, 7 +#define REC_QR_D 20, 21, 22, 23 + +#define REC_PQR_DEFINE() {} +#define REC_PQR_STRIDE 2 +#define REC_PQR_X 0, 1 +#define REC_PQR_Y 2, 3 +#define REC_PQR_Z 4, 5 +#define REC_PQR_D 6, 7 +#define REC_PQR_XS 6, 7 +#define REC_PQR_YS 8, 9 + + +#include <sys/vdev_raidz_impl.h> +#include "vdev_raidz_math_impl.h" + +DEFINE_GEN_METHODS(avx512bw); +DEFINE_REC_METHODS(avx512bw); + +static boolean_t +raidz_will_avx512bw_work(void) +{ + return (zfs_avx_available() && + zfs_avx512f_available() && + zfs_avx512bw_available()); +} + +const raidz_impl_ops_t vdev_raidz_avx512bw_impl = { + .init = NULL, + .fini = NULL, + .gen = RAIDZ_GEN_METHODS(avx512bw), + .rec = RAIDZ_REC_METHODS(avx512bw), + .is_supported = &raidz_will_avx512bw_work, + .name = "avx512bw" +}; + +#endif /* defined(__x86_64) && defined(HAVE_AVX512BW) */ diff --git a/module/zfs/vdev_raidz_math_avx512f.c b/module/zfs/vdev_raidz_math_avx512f.c new file mode 100644 index 000000000..cc3868bce --- /dev/null +++ b/module/zfs/vdev_raidz_math_avx512f.c @@ -0,0 +1,523 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2016 Romain Dolbeau. All rights reserved. + */ + +#include <sys/isa_defs.h> + +#if defined(__x86_64) && defined(HAVE_AVX512F) + +#include <sys/types.h> +#include <linux/simd_x86.h> + +#define __asm __asm__ __volatile__ + +#define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N +#define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) + +#define VR0_(REG, ...) "zmm"#REG +#define VR1_(_1, REG, ...) "zmm"#REG +#define VR2_(_1, _2, REG, ...) "zmm"#REG +#define VR3_(_1, _2, _3, REG, ...) "zmm"#REG +#define VR4_(_1, _2, _3, _4, REG, ...) "zmm"#REG +#define VR5_(_1, _2, _3, _4, _5, REG, ...) "zmm"#REG +#define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "zmm"#REG +#define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "zmm"#REG + +#define VR0(r...) VR0_(r) +#define VR1(r...) VR1_(r) +#define VR2(r...) VR2_(r, 1) +#define VR3(r...) VR3_(r, 1, 2) +#define VR4(r...) VR4_(r, 1, 2) +#define VR5(r...) VR5_(r, 1, 2, 3) +#define VR6(r...) VR6_(r, 1, 2, 3, 4) +#define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) + +#define VRy0_(REG, ...) "ymm"#REG +#define VRy1_(_1, REG, ...) "ymm"#REG +#define VRy2_(_1, _2, REG, ...) "ymm"#REG +#define VRy3_(_1, _2, _3, REG, ...) "ymm"#REG +#define VRy4_(_1, _2, _3, _4, REG, ...) "ymm"#REG +#define VRy5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG +#define VRy6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG +#define VRy7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG + +#define VRy0(r...) VRy0_(r) +#define VRy1(r...) VRy1_(r) +#define VRy2(r...) VRy2_(r, 1) +#define VRy3(r...) VRy3_(r, 1, 2) +#define VRy4(r...) VRy4_(r, 1, 2) +#define VRy5(r...) VRy5_(r, 1, 2, 3) +#define VRy6(r...) VRy6_(r, 1, 2, 3, 4) +#define VRy7(r...) VRy7_(r, 1, 2, 3, 4, 5) + +#define R_01(REG1, REG2, ...) REG1, REG2 +#define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 +#define R_23(REG...) _R_23(REG, 1, 2, 3) + +#define ASM_BUG() ASSERT(0) + +extern const uint8_t gf_clmul_mod_lt[4*256][16]; + +#define ELEM_SIZE 64 + +typedef struct v { + uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); +} v_t; + +#define PREFETCHNTA(ptr, offset) \ +{ \ + __asm( \ + "prefetchnta " #offset "(%[MEM])\n" \ + : : [MEM] "r" (ptr)); \ +} + +#define PREFETCH(ptr, offset) \ +{ \ + __asm( \ + "prefetcht0 " #offset "(%[MEM])\n" \ + : : [MEM] "r" (ptr)); \ +} + +#define XOR_ACC(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ + "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ + "vpxorq 0x80(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ + "vpxorq 0xc0(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ + "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define XOR(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "vpxorq %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ + "vpxorq %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ + "vpxorq %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ + "vpxorq %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "vpxorq %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ + "vpxorq %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define ZERO(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vpxorq %" VR0(r) ", %" VR0(r)", %" VR0(r) "\n" \ + "vpxorq %" VR1(r) ", %" VR1(r)", %" VR1(r) "\n" \ + "vpxorq %" VR2(r) ", %" VR2(r)", %" VR2(r) "\n" \ + "vpxorq %" VR3(r) ", %" VR3(r)", %" VR3(r)); \ + break; \ + case 2: \ + __asm( \ + "vpxorq %" VR0(r) ", %" VR0(r)", %" VR0(r) "\n" \ + "vpxorq %" VR1(r) ", %" VR1(r)", %" VR1(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define COPY(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "vmovdqa64 %" VR0(r) ", %" VR4(r) "\n" \ + "vmovdqa64 %" VR1(r) ", %" VR5(r) "\n" \ + "vmovdqa64 %" VR2(r) ", %" VR6(r) "\n" \ + "vmovdqa64 %" VR3(r) ", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "vmovdqa64 %" VR0(r) ", %" VR2(r) "\n" \ + "vmovdqa64 %" VR1(r) ", %" VR3(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define LOAD(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ + "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ + "vmovdqa64 0x80(%[SRC]), %%" VR2(r) "\n" \ + "vmovdqa64 0xc0(%[SRC]), %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ + "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define STORE(dst, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ + "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ + "vmovdqa64 %%" VR2(r) ", 0x80(%[DST])\n" \ + "vmovdqa64 %%" VR3(r) ", 0xc0(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + case 2: \ + __asm( \ + "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ + "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define FLUSH() \ +{ \ + __asm("vzeroupper"); \ +} + +#define MUL2_SETUP() \ +{ \ + __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \ + __asm("vpbroadcastq %xmm14, %zmm14"); \ + __asm("vmovq %0, %%xmm13" :: "r"(0x8080808080808080)); \ + __asm("vpbroadcastq %xmm13, %zmm13"); \ + __asm("vmovq %0, %%xmm12" :: "r"(0xfefefefefefefefe)); \ + __asm("vpbroadcastq %xmm12, %zmm12"); \ + __asm("vpxorq %zmm0, %zmm0 ,%zmm0"); \ +} + +#define _MUL2(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + __asm( \ + "vpandq %" VR0(r)", %zmm13, %zmm10\n" \ + "vpandq %" VR1(r)", %zmm13, %zmm11\n" \ + "vpsrlq $7, %zmm10, %zmm30\n" \ + "vpsrlq $7, %zmm11, %zmm31\n" \ + "vpsllq $1, %zmm10, %zmm10\n" \ + "vpsllq $1, %zmm11, %zmm11\n" \ + "vpsubq %zmm30, %zmm10, %zmm10\n" \ + "vpsubq %zmm31, %zmm11, %zmm11\n" \ + "vpsllq $1, %" VR0(r)", %" VR0(r) "\n" \ + "vpsllq $1, %" VR1(r)", %" VR1(r) "\n" \ + "vpandq %zmm10, %zmm14, %zmm10\n" \ + "vpandq %zmm11, %zmm14, %zmm11\n" \ + "vpternlogd $0x6c,%zmm12, %zmm10, %" VR0(r) "\n" \ + "vpternlogd $0x6c,%zmm12, %zmm11, %" VR1(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL2(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + _MUL2(R_01(r)); \ + _MUL2(R_23(r)); \ + break; \ + case 2: \ + _MUL2(r); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL4(r...) \ +{ \ + MUL2(r); \ + MUL2(r); \ +} + +/* + * Must match the init above + */ +#define _0f "zmm0" +#define _as "zmm14" +#define _bs "zmm13" +#define _ltmod "zmm12" +#define _ltmul "zmm11" +#define _ta "zmm10" +#define _tb "zmm15" + +/* + * Must be in the first 16, otherwise an EVEX pshufb is generated + * Must match above + */ +#define _asYlo "ymm14" +#define _bsYlo "ymm13" +#define _ltmodYlo "ymm12" +#define _ltmulYlo "ymm11" +#define _taYlo "ymm10" +#define _tbYlo "ymm15" + +/* + * Must be in the first 16, otherwise an EVEX pshufb is generated + * ... + */ +#define _asYhi "ymm9" +#define _bsYhi "ymm8" +#define _ltmodYhi "ymm7" +#define _ltmulYhi "ymm6" +#define _taYhi "ymm5" +#define _tbYhi "ymm4" + +/* + * This uses a pair of AVX2 pshufb to emulate the missing AVX512 pshufb. + * AVX512BW has the full pshufb + * To get VEX pshufb (AVX2, supported in KNL) instead of EVEX pshufb + * (AVX512BW, not supported on KNL, probably also requiring AVX51VL + * since we use a 256 bits version), all registers in parameters to + * pshufb must be among ymm0-ymm15, since only EVEX can encore + * ymm16-ymm31 + * This is a bit hackish, but short of encoding the instruction in + * binary, how do we force the use of AVX2 pshufb ? + * Note that the other way round (forcing AVX512) is easy, just encode + * k0 as the mask register (k0 is all-1). + */ +#define _MULx2(c, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + __asm( \ + "vmovq %[c0f], %%xmm0\n" \ + "vpbroadcastq %%xmm0, %%" _0f "\n" \ + /* upper bits */ \ + "vbroadcasti32x4 0x00(%[lt]), %%" _ltmod "\n" \ + "vbroadcasti32x4 0x10(%[lt]), %%" _ltmul "\n" \ + \ + "vpsrad $0x4, %%" VR0(r) ", %%"_as "\n" \ + "vpsrad $0x4, %%" VR1(r) ", %%"_bs "\n" \ + "vpandq %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \ + "vpandq %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \ + "vpandq %%" _0f ", %%" _as ", %%" _as "\n" \ + "vpandq %%" _0f ", %%" _bs ", %%" _bs "\n" \ + \ + "vextracti64x4 $1,%%" _ltmod ",%%" _ltmodYhi"\n" \ + \ + "vextracti64x4 $1,%%" _as ",%%" _asYhi"\n" \ + "vpshufb %%" _asYlo ", %%" _ltmodYlo ", %%" _taYlo "\n" \ + "vpshufb %%" _asYhi ", %%" _ltmodYhi ", %%" _taYhi "\n" \ + "vinserti64x4 $1,%%" _taYhi ",%%" _ta ",%%" _ta "\n" \ + \ + "vextracti64x4 $1,%%" _bs ",%%" _bsYhi"\n" \ + "vpshufb %%" _bsYlo ", %%" _ltmodYlo ", %%" _tbYlo "\n" \ + "vpshufb %%" _bsYhi ", %%" _ltmodYhi ", %%" _tbYhi "\n" \ + "vinserti64x4 $1,%%" _tbYhi ",%%" _tb ",%%" _tb "\n" \ + \ + "vextracti64x4 $1,%%" _ltmul ",%%" _ltmulYhi"\n" \ + \ + "vpshufb %%" _asYlo ", %%" _ltmulYlo ", %%" _asYlo "\n" \ + "vpshufb %%" _asYhi ", %%" _ltmulYhi ", %%" _asYhi "\n" \ + "vinserti64x4 $1,%%" _asYhi ",%%" _as ",%%" _as "\n" \ + \ + "vpshufb %%" _bsYlo ", %%" _ltmulYlo ", %%" _bsYlo "\n" \ + "vpshufb %%" _bsYhi ", %%" _ltmulYhi ", %%" _bsYhi "\n" \ + "vinserti64x4 $1,%%" _bsYhi ",%%" _bs ",%%" _bs "\n" \ + \ + /* lower bits */ \ + "vbroadcasti32x4 0x20(%[lt]), %%" _ltmod "\n" \ + "vbroadcasti32x4 0x30(%[lt]), %%" _ltmul "\n" \ + \ + "vpxorq %%" _ta ", %%" _as ", %%" _as "\n" \ + "vpxorq %%" _tb ", %%" _bs ", %%" _bs "\n" \ + \ + "vextracti64x4 $1,%%" _ltmod ",%%" _ltmodYhi"\n" \ + \ + "vextracti64x4 $0,%%" VR0(r) ",%%" "ymm1" "\n" \ + "vextracti64x4 $1,%%" VR0(r) ",%%" _asYhi"\n" \ + "vpshufb %%" "ymm1" ", %%" _ltmodYlo ", %%" _taYlo "\n" \ + "vpshufb %%" _asYhi ", %%" _ltmodYhi ", %%" _taYhi "\n" \ + "vinserti64x4 $1,%%" _taYhi ",%%" _ta ",%%" _ta "\n" \ + \ + "vextracti64x4 $0,%%" VR1(r) ",%%" "ymm2" "\n" \ + "vextracti64x4 $1,%%" VR1(r) ",%%" _bsYhi"\n" \ + "vpshufb %%" "ymm2" ", %%" _ltmodYlo ", %%" _tbYlo "\n" \ + "vpshufb %%" _bsYhi ", %%" _ltmodYhi ", %%" _tbYhi "\n" \ + "vinserti64x4 $1,%%" _tbYhi ",%%" _tb ",%%" _tb "\n" \ + \ + "vextracti64x4 $1,%%" _ltmul ",%%" _ltmulYhi"\n" \ + \ + "vpshufb %%" "ymm1" ", %%" _ltmulYlo ", %%" "ymm1" "\n" \ + "vpshufb %%" _asYhi ", %%" _ltmulYhi ", %%" _asYhi "\n" \ + "vinserti64x4 $1,%%" _asYhi ",%%" "zmm1" ",%%" VR0(r) "\n" \ + \ + "vpshufb %%" "ymm2" ", %%" _ltmulYlo ", %%" "ymm2" "\n" \ + "vpshufb %%" _bsYhi ", %%" _ltmulYhi ", %%" _bsYhi "\n" \ + "vinserti64x4 $1,%%" _bsYhi ",%%" "zmm2" ",%%" VR1(r) "\n" \ + \ + "vpxorq %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \ + "vpxorq %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \ + "vpxorq %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \ + "vpxorq %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \ + : : [c0f] "r" (0x0f0f0f0f0f0f0f0f), \ + [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL(c, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + _MULx2(c, R_01(r)); \ + _MULx2(c, R_23(r)); \ + break; \ + case 2: \ + _MULx2(c, R_01(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define raidz_math_begin() kfpu_begin() +#define raidz_math_end() \ +{ \ + FLUSH(); \ + kfpu_end(); \ +} + +/* + * This use zmm16-zmm31 registers to free up zmm0-zmm15 + * to use with the AVX2 pshufb, see above + */ +#define GEN_P_DEFINE() {} +#define GEN_P_STRIDE 4 +#define GEN_P_P 20, 21, 22, 23 + +#define GEN_PQ_DEFINE() {} +#define GEN_PQ_STRIDE 4 +#define GEN_PQ_D 20, 21, 22, 23 +#define GEN_PQ_P 24, 25, 26, 27 +#define GEN_PQ_Q 28, 29, 3, 4 + +#define GEN_PQR_DEFINE() {} +#define GEN_PQR_STRIDE 2 +#define GEN_PQR_D 20, 21 +#define GEN_PQR_P 22, 23 +#define GEN_PQR_Q 24, 25 +#define GEN_PQR_R 26, 27 + +#define REC_P_DEFINE() {} +#define REC_P_STRIDE 4 +#define REC_P_X 20, 21, 22, 23 + +#define REC_Q_DEFINE() {} +#define REC_Q_STRIDE 4 +#define REC_Q_X 20, 21, 22, 23 + +#define REC_R_DEFINE() {} +#define REC_R_STRIDE 4 +#define REC_R_X 20, 21, 22, 23 + +#define REC_PQ_DEFINE() {} +#define REC_PQ_STRIDE 2 +#define REC_PQ_X 20, 21 +#define REC_PQ_Y 22, 23 +#define REC_PQ_D 24, 25 + +#define REC_PR_DEFINE() {} +#define REC_PR_STRIDE 2 +#define REC_PR_X 20, 21 +#define REC_PR_Y 22, 23 +#define REC_PR_D 24, 25 + +#define REC_QR_DEFINE() {} +#define REC_QR_STRIDE 2 +#define REC_QR_X 20, 21 +#define REC_QR_Y 22, 23 +#define REC_QR_D 24, 25 + +#define REC_PQR_DEFINE() {} +#define REC_PQR_STRIDE 2 +#define REC_PQR_X 20, 21 +#define REC_PQR_Y 22, 23 +#define REC_PQR_Z 24, 25 +#define REC_PQR_D 26, 27 +#define REC_PQR_XS 26, 27 +#define REC_PQR_YS 28, 29 + + +#include <sys/vdev_raidz_impl.h> +#include "vdev_raidz_math_impl.h" + +DEFINE_GEN_METHODS(avx512f); +DEFINE_REC_METHODS(avx512f); + +static boolean_t +raidz_will_avx512f_work(void) +{ + return (zfs_avx_available() && + zfs_avx512f_available()); +} + +const raidz_impl_ops_t vdev_raidz_avx512f_impl = { + .init = NULL, + .fini = NULL, + .gen = RAIDZ_GEN_METHODS(avx512f), + .rec = RAIDZ_REC_METHODS(avx512f), + .is_supported = &raidz_will_avx512f_work, + .name = "avx512f" +}; + +#endif /* defined(__x86_64) && defined(HAVE_AVX512F) */ |