Linux 4.14, 4.19, 5.0+ compat: SIMD save/restore

Contrary to initial testing we cannot rely on these kernels to
invalidate the per-cpu FPU state and restore the FPU registers.
Nor can we guarantee that the kernel won't modify the FPU state
which we saved in the task struck.

Therefore, the kfpu_begin() and kfpu_end() functions have been
updated to save and restore the FPU state using our own dedicated
per-cpu FPU state variables.

This has the additional advantage of allowing us to use the FPU
again in user threads.  So we remove the code which was added to
use task queues to ensure some functions ran in kernel threads.

Reviewed-by: Fabian Grünbichler <[email protected]>
Reviewed-by: Tony Hutter <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Issue #9346
Closes #9403 

1 files changed, 41 insertions, 39 deletions

diff --git a/config/kernel-fpu.m4 b/config/kernel-fpu.m4
index a2c47d65a..3c7933413 100644
--- a/config/kernel-fpu.m4
+++ b/config/kernel-fpu.m4
@@ -2,15 +2,9 @@ dnl #
 dnl # Handle differences in kernel FPU code.
 dnl #
 dnl # Kernel
-dnl # 5.2:	The fpu->initialized flag was replaced by TIF_NEED_FPU_LOAD.
-dnl #		HAVE_KERNEL_TIF_NEED_FPU_LOAD
-dnl #
-dnl # 5.0:	As an optimization SIMD operations performed by kernel
-dnl #		threads can skip saving and restoring their FPU context.
-dnl #		Wrappers have been introduced to determine the running
-dnl #		context and use either the SIMD or generic implementation.
+dnl # 5.0:	Wrappers have been introduced to save/restore the FPU state.
 dnl #		This change was made to the 4.19.38 and 4.14.120 LTS kernels.
-dnl #		HAVE_KERNEL_FPU_INITIALIZED
+dnl #		HAVE_KERNEL_FPU_INTERNAL
 dnl #
 dnl # 4.2:	Use __kernel_fpu_{begin,end}()
 dnl #		HAVE_UNDERSCORE_KERNEL_FPU & KERNEL_EXPORTS_X86_FPU
@@ -38,6 +32,7 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU_HEADER], [
 
 AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [
 	ZFS_LINUX_TEST_SRC([kernel_fpu], [
+		#include <linux/types.h>
 		#ifdef HAVE_KERNEL_FPU_API_HEADER
 		#include <asm/fpu/api.h>
 		#else
@@ -50,6 +45,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [
 	], [], [$ZFS_META_LICENSE])
 
 	ZFS_LINUX_TEST_SRC([__kernel_fpu], [
+		#include <linux/types.h>
 		#ifdef HAVE_KERNEL_FPU_API_HEADER
 		#include <asm/fpu/api.h>
 		#else
@@ -61,22 +57,41 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [
 		__kernel_fpu_end();
 	], [], [$ZFS_META_LICENSE])
 
-	ZFS_LINUX_TEST_SRC([fpu_initialized], [
-		#include <linux/module.h>
-		#include <linux/sched.h>
-	],[
-		struct fpu *fpu = &current->thread.fpu;
-		if (fpu->initialized) { return (0); };
-	])
+	ZFS_LINUX_TEST_SRC([fpu_internal], [
+		#if defined(__x86_64) || defined(__x86_64__) || \
+		    defined(__i386) || defined(__i386__)
+		#if !defined(__x86)
+		#define __x86
+		#endif
+		#endif
 
-	ZFS_LINUX_TEST_SRC([tif_need_fpu_load], [
-		#include <linux/module.h>
-		#include <asm/thread_info.h>
+		#if !defined(__x86)
+		#error Unsupported architecture
+		#endif
 
-		#if !defined(TIF_NEED_FPU_LOAD)
-		#error "TIF_NEED_FPU_LOAD undefined"
+		#include <linux/types.h>
+		#ifdef HAVE_KERNEL_FPU_API_HEADER
+		#include <asm/fpu/api.h>
+		#include <asm/fpu/internal.h>
+		#else
+		#include <asm/i387.h>
+		#include <asm/xcr.h>
+		#endif
+
+		#if !defined(XSTATE_XSAVE)
+		#error XSTATE_XSAVE not defined
 		#endif
-	],[])
+
+		#if !defined(XSTATE_XRESTORE)
+		#error XSTATE_XRESTORE not defined
+		#endif
+	],[
+		struct fpu *fpu = &current->thread.fpu;
+		union fpregs_state *st = &fpu->state;
+		struct fregs_state *fr __attribute__ ((unused)) = &st->fsave;
+		struct fxregs_state *fxr __attribute__ ((unused)) = &st->fxsave;
+		struct xregs_state *xr __attribute__ ((unused)) = &st->xsave;
+	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_FPU], [
@@ -104,25 +119,12 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU], [
 			AC_DEFINE(KERNEL_EXPORTS_X86_FPU, 1,
 			    [kernel exports FPU functions])
 		],[
-			dnl #
-			dnl # Linux 5.0 kernel
-			dnl #
-			ZFS_LINUX_TEST_RESULT([fpu_initialized], [
-				AC_MSG_RESULT(fpu.initialized)
-				AC_DEFINE(HAVE_KERNEL_FPU_INITIALIZED, 1,
-				    [kernel fpu.initialized exists])
+			ZFS_LINUX_TEST_RESULT([fpu_internal], [
+				AC_MSG_RESULT(internal)
+				AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1,
+				    [kernel fpu internal])
 			],[
-				dnl #
-				dnl # Linux 5.2 kernel
-				dnl #
-				ZFS_LINUX_TEST_RESULT([tif_need_fpu_load], [
-					AC_MSG_RESULT(TIF_NEED_FPU_LOAD)
-					AC_DEFINE(
-					    HAVE_KERNEL_TIF_NEED_FPU_LOAD, 1,
-					    [kernel TIF_NEED_FPU_LOAD exists])
-				],[
-					AC_MSG_RESULT(unavailable)
-				])
+				AC_MSG_RESULT(unavailable)
 			])
 		])
 	])