Fletcher4: save/reload implementation context

Init, compute, and fini methods are changed to work on internal context object.
This is necessary because ABI does not guarantee that SIMD registers will be preserved
on function calls. This is technically the case in Linux kernel in between
`kfpu_begin()/kfpu_end()`, but it breaks user-space tests and some kernels that
don't require disabling preemption for using SIMD (osx).

Use scalar compute methods in-place for small buffers, and when the buffer size
does not meet SIMD size alignment.

Signed-off-by: Gvozden Neskovic <[email protected]>

1 files changed, 35 insertions, 3 deletions

diff --git a/include/zfs_fletcher.h b/include/zfs_fletcher.h
index 83f92a096..85c2b5a7e 100644
--- a/include/zfs_fletcher.h
+++ b/include/zfs_fletcher.h
@@ -62,12 +62,43 @@ void fletcher_4_init(void);
 void fletcher_4_fini(void);
 
 
+
+/* Internal fletcher ctx */
+
+typedef struct zfs_fletcher_sse {
+	uint64_t v[2] __attribute__((aligned(16)));
+} zfs_fletcher_sse_t;
+
+typedef struct zfs_fletcher_avx {
+	uint64_t v[4] __attribute__((aligned(32)));
+} zfs_fletcher_avx_t;
+
+typedef struct zfs_fletcher_avx512 {
+	uint64_t v[8] __attribute__((aligned(64)));
+} zfs_fletcher_avx512_t;
+
+
+typedef union fletcher_4_ctx {
+	zio_cksum_t scalar;
+
+#if defined(HAVE_SSE2) || (defined(HAVE_SSE2) && defined(HAVE_SSSE3))
+	zfs_fletcher_sse_t sse[4];
+#endif
+#if defined(HAVE_AVX) && defined(HAVE_AVX2)
+	zfs_fletcher_avx_t avx[4];
+#endif
+#if defined(__x86_64) && defined(HAVE_AVX512F)
+	zfs_fletcher_avx512_t avx512[4];
+#endif
+} fletcher_4_ctx_t;
+
 /*
  * fletcher checksum struct
  */
-typedef void (*fletcher_4_init_f)(zio_cksum_t *);
-typedef void (*fletcher_4_fini_f)(zio_cksum_t *);
-typedef void (*fletcher_4_compute_f)(const void *, uint64_t, zio_cksum_t *);
+typedef void (*fletcher_4_init_f)(fletcher_4_ctx_t *);
+typedef void (*fletcher_4_fini_f)(fletcher_4_ctx_t *, zio_cksum_t *);
+typedef void (*fletcher_4_compute_f)(fletcher_4_ctx_t *,
+    const void *, uint64_t);
 
 typedef struct fletcher_4_func {
 	fletcher_4_init_f init_native;
@@ -80,6 +111,7 @@ typedef struct fletcher_4_func {
 	const char *name;
 } fletcher_4_ops_t;
 
+
 #if defined(HAVE_SSE2)
 extern const fletcher_4_ops_t fletcher_4_sse2_ops;
 #endif