Linux 5.0 compat: SIMD compatibility

Restore the SIMD optimization for 4.19.38 LTS, 4.14.120 LTS,
and 5.0 and newer kernels.  This is accomplished by leveraging
the fact that by definition dedicated kernel threads never need
to concern themselves with saving and restoring the user FPU state.
Therefore, they may use the FPU as long as we can guarantee user
tasks always restore their FPU state before context switching back
to user space.

For the 5.0 and 5.1 kernels disabling preemption and local
interrupts is sufficient to allow the FPU to be used.  All non-kernel
threads will restore the preserved user FPU state.

For 5.2 and latter kernels the user FPU state restoration will be
skipped if the kernel determines the registers have not changed.
Therefore, for these kernels we need to perform the additional
step of saving and restoring the FPU registers.  Invalidating the
per-cpu global tracking the FPU state would force a restore but
that functionality is private to the core x86 FPU implementation
and unavailable.

In practice, restricting SIMD to kernel threads is not a major
restriction for ZFS.  The vast majority of SIMD operations are
already performed by the IO pipeline.  The remaining cases are
relatively infrequent and can be handled by the generic code
without significant impact.  The two most noteworthy cases are:

  1) Decrypting the wrapping key for an encrypted dataset,
     i.e. `zfs load-key`.  All other encryption and decryption
     operations will use the SIMD optimized implementations.

  2) Generating the payload checksums for a `zfs send` stream.

In order to avoid making any changes to the higher layers of ZFS
all of the `*_get_ops()` functions were updated to take in to
consideration the calling context.  This allows for the fastest
implementation to be used as appropriate (see kfpu_allowed()).

The only other notable instance of SIMD operations being used
outside a kernel thread was at module load time.  This code
was moved in to a taskq in order to accommodate the new kernel
thread restriction.

Finally, a few other modifications were made in order to further
harden this code and facilitate testing.  They include updating
each implementations operations structure to be declared as a
constant.  And allowing "cycle" to be set when selecting the
preferred ops in the kernel as well as user space.

Reviewed-by: Tony Hutter <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #8754 
Closes #8793 
Closes #8965 

6 files changed, 183 insertions, 73 deletions

diff --git a/include/linux/Makefile.am b/include/linux/Makefile.am
index efb49520e..2455759e8 100644
--- a/include/linux/Makefile.am
+++ b/include/linux/Makefile.am
@@ -7,6 +7,7 @@ KERNEL_H = \
 	$(top_srcdir)/include/linux/blkdev_compat.h \
 	$(top_srcdir)/include/linux/utsname_compat.h \
 	$(top_srcdir)/include/linux/kmap_compat.h \
+	$(top_srcdir)/include/linux/simd.h \
 	$(top_srcdir)/include/linux/simd_x86.h \
 	$(top_srcdir)/include/linux/simd_aarch64.h \
 	$(top_srcdir)/include/linux/mod_compat.h \
diff --git a/include/linux/simd.h b/include/linux/simd.h
new file mode 100644
index 000000000..d2b60996a
--- /dev/null
+++ b/include/linux/simd.h
@@ -0,0 +1,41 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (C) 2019 Lawrence Livermore National Security, LLC.
+ */
+
+#ifndef _SIMD_H
+#define	_SIMD_H
+
+#if defined(__x86)
+#include <linux/simd_x86.h>
+
+#elif defined(__aarch64__)
+#include <linux/simd_aarch64.h>
+#else
+
+#define	kfpu_allowed()		1
+#define	kfpu_initialize(tsk)	do {} while (0)
+#define	kfpu_begin()		do {} while (0)
+#define	kfpu_end()		do {} while (0)
+
+#endif
+#endif /* _SIMD_H */
diff --git a/include/linux/simd_aarch64.h b/include/linux/simd_aarch64.h
index 155ef6205..1cfcd01e4 100644
--- a/include/linux/simd_aarch64.h
+++ b/include/linux/simd_aarch64.h
@@ -41,20 +41,18 @@
 
 #if defined(_KERNEL)
 #include <asm/neon.h>
-#define	kfpu_begin()		\
-{					\
-	kernel_neon_begin();		\
-}
-#define	kfpu_end()			\
-{					\
-	kernel_neon_end();		\
-}
+#define	kfpu_allowed()		1
+#define	kfpu_initialize(tsk)	do {} while (0)
+#define	kfpu_begin()		kernel_neon_begin()
+#define	kfpu_end()		kernel_neon_end()
 #else
 /*
  * fpu dummy methods for userspace
  */
-#define	kfpu_begin() 	do {} while (0)
-#define	kfpu_end() 		do {} while (0)
+#define	kfpu_allowed()		1
+#define	kfpu_initialize(tsk)	do {} while (0)
+#define	kfpu_begin()		do {} while (0)
+#define	kfpu_end()		do {} while (0)
 #endif /* defined(_KERNEL) */
 
 #endif /* __aarch64__ */
diff --git a/include/linux/simd_x86.h b/include/linux/simd_x86.h
index 12cd74677..2d7a1c3a5 100644
--- a/include/linux/simd_x86.h
+++ b/include/linux/simd_x86.h
@@ -90,33 +90,135 @@
 #include <asm/xcr.h>
 #endif
 
+/*
+ * The following cases are for kernels which export either the
+ * kernel_fpu_* or __kernel_fpu_* functions.
+ */
+#if defined(KERNEL_EXPORTS_X86_FPU)
+
+#define	kfpu_allowed()		1
+#define	kfpu_initialize(tsk)	do {} while (0)
+
 #if defined(HAVE_UNDERSCORE_KERNEL_FPU)
 #define	kfpu_begin()		\
-{							\
-	preempt_disable();		\
+{				\
+	preempt_disable();	\
 	__kernel_fpu_begin();	\
 }
-#define	kfpu_end()			\
-{							\
-	__kernel_fpu_end();		\
-	preempt_enable();		\
+#define	kfpu_end()		\
+{				\
+	__kernel_fpu_end();	\
+	preempt_enable();	\
 }
+
 #elif defined(HAVE_KERNEL_FPU)
-#define	kfpu_begin()	kernel_fpu_begin()
+#define	kfpu_begin()		kernel_fpu_begin()
 #define	kfpu_end()		kernel_fpu_end()
+
 #else
-/* Kernel doesn't export any kernel_fpu_* functions */
-#include <asm/fpu/internal.h>	/* For kernel xgetbv() */
-#define	kfpu_begin() 	panic("This code should never run")
-#define	kfpu_end() 	panic("This code should never run")
-#endif /* defined(HAVE_KERNEL_FPU) */
+/*
+ * This case is unreachable.  When KERNEL_EXPORTS_X86_FPU is defined then
+ * either HAVE_UNDERSCORE_KERNEL_FPU or HAVE_KERNEL_FPU must be defined.
+ */
+#error "Unreachable kernel configuration"
+#endif
+
+#else /* defined(KERNEL_EXPORTS_X86_FPU) */
+/*
+ * When the kernel_fpu_* symbols are unavailable then provide our own
+ * versions which allow the FPU to be safely used in kernel threads.
+ * In practice, this is not a significant restriction for ZFS since the
+ * vast majority of SIMD operations are performed by the IO pipeline.
+ */
 
+/*
+ * Returns non-zero if FPU operations are allowed in the current context.
+ */
+#if defined(HAVE_KERNEL_TIF_NEED_FPU_LOAD)
+#define	kfpu_allowed()		((current->flags & PF_KTHREAD) && \
+				test_thread_flag(TIF_NEED_FPU_LOAD))
+#elif defined(HAVE_KERNEL_FPU_INITIALIZED)
+#define	kfpu_allowed()		((current->flags & PF_KTHREAD) && \
+				current->thread.fpu.initialized)
 #else
+#define	kfpu_allowed()		0
+#endif
+
+static inline void
+kfpu_initialize(void)
+{
+	WARN_ON_ONCE(!(current->flags & PF_KTHREAD));
+
+#if defined(HAVE_KERNEL_TIF_NEED_FPU_LOAD)
+	__fpu_invalidate_fpregs_state(&current->thread.fpu);
+	set_thread_flag(TIF_NEED_FPU_LOAD);
+#elif defined(HAVE_KERNEL_FPU_INITIALIZED)
+	__fpu_invalidate_fpregs_state(&current->thread.fpu);
+	current->thread.fpu.initialized = 1;
+#endif
+}
+
+static inline void
+kfpu_begin(void)
+{
+	WARN_ON_ONCE(!kfpu_allowed());
+
+	/*
+	 * Preemption and interrupts must be disabled for the critical
+	 * region where the FPU state is being modified.
+	 */
+	preempt_disable();
+	local_irq_disable();
+
+#if defined(HAVE_KERNEL_TIF_NEED_FPU_LOAD)
+	/*
+	 * The current FPU registers need to be preserved by kfpu_begin()
+	 * and restored by kfpu_end().  This is required because we can
+	 * not call __cpu_invalidate_fpregs_state() to invalidate the
+	 * per-cpu FPU state and force them to be restored during a
+	 * context switch.
+	 */
+	copy_fpregs_to_fpstate(&current->thread.fpu);
+#elif defined(HAVE_KERNEL_FPU_INITIALIZED)
+	/*
+	 * There is no need to preserve and restore the FPU registers.
+	 * They will always be restored from the task's stored FPU state
+	 * when switching contexts.
+	 */
+	WARN_ON_ONCE(current->thread.fpu.initialized == 0);
+#endif
+}
+
+static inline void
+kfpu_end(void)
+{
+#if defined(HAVE_KERNEL_TIF_NEED_FPU_LOAD)
+	union fpregs_state *state = &current->thread.fpu.state;
+	int error;
+
+	if (use_xsave()) {
+		error = copy_kernel_to_xregs_err(&state->xsave, -1);
+	} else if (use_fxsr()) {
+		error = copy_kernel_to_fxregs_err(&state->fxsave);
+	} else {
+		error = copy_kernel_to_fregs_err(&state->fsave);
+	}
+	WARN_ON_ONCE(error);
+#endif
+
+	local_irq_enable();
+	preempt_enable();
+}
+#endif /* defined(HAVE_KERNEL_FPU) */
+
+#else /* defined(_KERNEL) */
 /*
- * fpu dummy methods for userspace
+ * FPU dummy methods for user space.
  */
-#define	kfpu_begin() 	do {} while (0)
-#define	kfpu_end() 		do {} while (0)
+#define	kfpu_allowed()		1
+#define	kfpu_initialize(tsk)	do {} while (0)
+#define	kfpu_begin()		do {} while (0)
+#define	kfpu_end()		do {} while (0)
 #endif /* defined(_KERNEL) */
 
 /*
@@ -298,7 +400,7 @@ __simd_state_enabled(const uint64_t state)
 	uint64_t xcr0;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_OSXSAVE) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_OSXSAVE)
 	has_osxsave = !!boot_cpu_has(X86_FEATURE_OSXSAVE);
 #else
 	has_osxsave = B_FALSE;
@@ -328,11 +430,7 @@ static inline boolean_t
 zfs_sse_available(void)
 {
 #if defined(_KERNEL)
-#if defined(KERNEL_EXPORTS_X86_FPU)
 	return (!!boot_cpu_has(X86_FEATURE_XMM));
-#else
-	return (B_FALSE);
-#endif
 #elif !defined(_KERNEL)
 	return (__cpuid_has_sse());
 #endif
@@ -345,11 +443,7 @@ static inline boolean_t
 zfs_sse2_available(void)
 {
 #if defined(_KERNEL)
-#if defined(KERNEL_EXPORTS_X86_FPU)
 	return (!!boot_cpu_has(X86_FEATURE_XMM2));
-#else
-	return (B_FALSE);
-#endif
 #elif !defined(_KERNEL)
 	return (__cpuid_has_sse2());
 #endif
@@ -362,11 +456,7 @@ static inline boolean_t
 zfs_sse3_available(void)
 {
 #if defined(_KERNEL)
-#if defined(KERNEL_EXPORTS_X86_FPU)
 	return (!!boot_cpu_has(X86_FEATURE_XMM3));
-#else
-	return (B_FALSE);
-#endif
 #elif !defined(_KERNEL)
 	return (__cpuid_has_sse3());
 #endif
@@ -379,11 +469,7 @@ static inline boolean_t
 zfs_ssse3_available(void)
 {
 #if defined(_KERNEL)
-#if defined(KERNEL_EXPORTS_X86_FPU)
 	return (!!boot_cpu_has(X86_FEATURE_SSSE3));
-#else
-	return (B_FALSE);
-#endif
 #elif !defined(_KERNEL)
 	return (__cpuid_has_ssse3());
 #endif
@@ -396,11 +482,7 @@ static inline boolean_t
 zfs_sse4_1_available(void)
 {
 #if defined(_KERNEL)
-#if defined(KERNEL_EXPORTS_X86_FPU)
 	return (!!boot_cpu_has(X86_FEATURE_XMM4_1));
-#else
-	return (B_FALSE);
-#endif
 #elif !defined(_KERNEL)
 	return (__cpuid_has_sse4_1());
 #endif
@@ -413,11 +495,7 @@ static inline boolean_t
 zfs_sse4_2_available(void)
 {
 #if defined(_KERNEL)
-#if defined(KERNEL_EXPORTS_X86_FPU)
 	return (!!boot_cpu_has(X86_FEATURE_XMM4_2));
-#else
-	return (B_FALSE);
-#endif
 #elif !defined(_KERNEL)
 	return (__cpuid_has_sse4_2());
 #endif
@@ -431,11 +509,7 @@ zfs_avx_available(void)
 {
 	boolean_t has_avx;
 #if defined(_KERNEL)
-#if defined(KERNEL_EXPORTS_X86_FPU)
 	has_avx = !!boot_cpu_has(X86_FEATURE_AVX);
-#else
-	has_avx = B_FALSE;
-#endif
 #elif !defined(_KERNEL)
 	has_avx = __cpuid_has_avx();
 #endif
@@ -451,11 +525,7 @@ zfs_avx2_available(void)
 {
 	boolean_t has_avx2;
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX2) && defined(KERNEL_EXPORTS_X86_FPU)
 	has_avx2 = !!boot_cpu_has(X86_FEATURE_AVX2);
-#else
-	has_avx2 = B_FALSE;
-#endif
 #elif !defined(_KERNEL)
 	has_avx2 = __cpuid_has_avx2();
 #endif
@@ -470,7 +540,7 @@ static inline boolean_t
 zfs_bmi1_available(void)
 {
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_BMI1) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_BMI1)
 	return (!!boot_cpu_has(X86_FEATURE_BMI1));
 #else
 	return (B_FALSE);
@@ -487,7 +557,7 @@ static inline boolean_t
 zfs_bmi2_available(void)
 {
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_BMI2) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_BMI2)
 	return (!!boot_cpu_has(X86_FEATURE_BMI2));
 #else
 	return (B_FALSE);
@@ -504,7 +574,7 @@ static inline boolean_t
 zfs_aes_available(void)
 {
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AES) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AES)
 	return (!!boot_cpu_has(X86_FEATURE_AES));
 #else
 	return (B_FALSE);
@@ -521,7 +591,7 @@ static inline boolean_t
 zfs_pclmulqdq_available(void)
 {
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_PCLMULQDQ) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_PCLMULQDQ)
 	return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ));
 #else
 	return (B_FALSE);
@@ -555,7 +625,7 @@ zfs_avx512f_available(void)
 	boolean_t has_avx512 = B_FALSE;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX512F) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AVX512F)
 	has_avx512 = !!boot_cpu_has(X86_FEATURE_AVX512F);
 #else
 	has_avx512 = B_FALSE;
@@ -574,7 +644,7 @@ zfs_avx512cd_available(void)
 	boolean_t has_avx512 = B_FALSE;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX512CD) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AVX512CD)
 	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
 	    boot_cpu_has(X86_FEATURE_AVX512CD);
 #else
@@ -594,7 +664,7 @@ zfs_avx512er_available(void)
 	boolean_t has_avx512 = B_FALSE;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX512ER) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AVX512ER)
 	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
 	    boot_cpu_has(X86_FEATURE_AVX512ER);
 #else
@@ -614,7 +684,7 @@ zfs_avx512pf_available(void)
 	boolean_t has_avx512 = B_FALSE;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX512PF) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AVX512PF)
 	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
 	    boot_cpu_has(X86_FEATURE_AVX512PF);
 #else
@@ -634,7 +704,7 @@ zfs_avx512bw_available(void)
 	boolean_t has_avx512 = B_FALSE;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX512BW) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AVX512BW)
 	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
 	    boot_cpu_has(X86_FEATURE_AVX512BW);
 #else
@@ -654,7 +724,7 @@ zfs_avx512dq_available(void)
 	boolean_t has_avx512 = B_FALSE;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX512DQ) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AVX512DQ)
 	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
 	    boot_cpu_has(X86_FEATURE_AVX512DQ);
 #else
@@ -674,7 +744,7 @@ zfs_avx512vl_available(void)
 	boolean_t has_avx512 = B_FALSE;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX512VL) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AVX512VL)
 	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
 	    boot_cpu_has(X86_FEATURE_AVX512VL);
 #else
@@ -694,7 +764,7 @@ zfs_avx512ifma_available(void)
 	boolean_t has_avx512 = B_FALSE;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX512IFMA) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AVX512IFMA)
 	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
 	    boot_cpu_has(X86_FEATURE_AVX512IFMA);
 #else
@@ -714,7 +784,7 @@ zfs_avx512vbmi_available(void)
 	boolean_t has_avx512 = B_FALSE;
 
 #if defined(_KERNEL)
-#if defined(X86_FEATURE_AVX512VBMI) && defined(KERNEL_EXPORTS_X86_FPU)
+#if defined(X86_FEATURE_AVX512VBMI)
 	has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) &&
 	    boot_cpu_has(X86_FEATURE_AVX512VBMI);
 #else
diff --git a/include/sys/vdev_raidz.h b/include/sys/vdev_raidz.h
index 2ce32469d..0ce2b5ea1 100644
--- a/include/sys/vdev_raidz.h
+++ b/include/sys/vdev_raidz.h
@@ -51,7 +51,7 @@ int vdev_raidz_reconstruct(struct raidz_map *, const int *, int);
  */
 void vdev_raidz_math_init(void);
 void vdev_raidz_math_fini(void);
-struct raidz_impl_ops *vdev_raidz_math_get_ops(void);
+const struct raidz_impl_ops *vdev_raidz_math_get_ops(void);
 int vdev_raidz_math_generate(struct raidz_map *);
 int vdev_raidz_math_reconstruct(struct raidz_map *, const int *, const int *,
     const int);
diff --git a/include/sys/vdev_raidz_impl.h b/include/sys/vdev_raidz_impl.h
index 0799ed19d..4969d110b 100644
--- a/include/sys/vdev_raidz_impl.h
+++ b/include/sys/vdev_raidz_impl.h
@@ -126,7 +126,7 @@ typedef struct raidz_map {
 	uintptr_t rm_reports;		/* # of referencing checksum reports */
 	uint8_t	rm_freed;		/* map no longer has referencing ZIO */
 	uint8_t	rm_ecksuminjected;	/* checksum error was injected */
-	raidz_impl_ops_t *rm_ops;	/* RAIDZ math operations */
+	const raidz_impl_ops_t *rm_ops;	/* RAIDZ math operations */
 	raidz_col_t rm_col[1];		/* Flexible array of I/O columns */
 } raidz_map_t;