Add support for selecting encryption backend

- Add two new module parameters to icp (icp_aes_impl, icp_gcm_impl)
  that control the crypto implementation.  At the moment there is a
  choice between generic and aesni (on platforms that support it).
- This enables support for AES-NI and PCLMULQDQ-NI on AMD Family
  15h (bulldozer) and newer CPUs (zen).
- Modify aes_key_t to track what implementation it was generated
  with as key schedules generated with various implementations
  are not necessarily interchangable.

Reviewed by: Gvozden Neskovic <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Tom Caputi <[email protected]>
Reviewed-by: Richard Laager <[email protected]>
Signed-off-by: Nathaniel R. Lewis <[email protected]>
Closes #7102 
Closes #7103 

2 files changed, 42 insertions, 2 deletions

diff --git a/include/linux/simd_x86.h b/include/linux/simd_x86.h
index c9e3970c0..10ce02e37 100644
--- a/include/linux/simd_x86.h
+++ b/include/linux/simd_x86.h
@@ -148,7 +148,9 @@ typedef enum cpuid_inst_sets {
 	AVX512VBMI,
 	AVX512PF,
 	AVX512ER,
-	AVX512VL
+	AVX512VL,
+	AES,
+	PCLMULQDQ
 } cpuid_inst_sets_t;
 
 /*
@@ -170,6 +172,8 @@ typedef struct cpuid_feature_desc {
 #define	_AVX512PF_BIT		(_AVX512F_BIT | (1U << 26))
 #define	_AVX512ER_BIT		(_AVX512F_BIT | (1U << 27))
 #define	_AVX512VL_BIT		(1U << 31) /* if used also check other levels */
+#define	_AES_BIT		(1U << 25)
+#define	_PCLMULQDQ_BIT		(1U << 1)
 
 /*
  * Descriptions of supported instruction sets
@@ -194,7 +198,9 @@ static const cpuid_feature_desc_t cpuid_features[] = {
 	[AVX512VBMI]	= {7U, 0U, _AVX512VBMI_BIT,	ECX	},
 	[AVX512PF]	= {7U, 0U, _AVX512PF_BIT,	EBX	},
 	[AVX512ER]	= {7U, 0U, _AVX512ER_BIT,	EBX	},
-	[AVX512VL]	= {7U, 0U, _AVX512ER_BIT,	EBX	}
+	[AVX512VL]	= {7U, 0U, _AVX512ER_BIT,	EBX	},
+	[AES]		= {1U, 0U, _AES_BIT,		ECX	},
+	[PCLMULQDQ]	= {1U, 0U, _PCLMULQDQ_BIT,	ECX	},
 };
 
 /*
@@ -265,6 +271,8 @@ CPUID_FEATURE_CHECK(avx512vbmi, AVX512VBMI);
 CPUID_FEATURE_CHECK(avx512pf, AVX512PF);
 CPUID_FEATURE_CHECK(avx512er, AVX512ER);
 CPUID_FEATURE_CHECK(avx512vl, AVX512VL);
+CPUID_FEATURE_CHECK(aes, AES);
+CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ);
 
 #endif /* !defined(_KERNEL) */
 
@@ -442,6 +450,35 @@ zfs_bmi2_available(void)
 #endif
 }
 
+/*
+ * Check if AES instruction set is available
+ */
+static inline boolean_t
+zfs_aes_available(void)
+{
+#if defined(_KERNEL) && defined(X86_FEATURE_AES)
+	return (!!boot_cpu_has(X86_FEATURE_AES));
+#elif defined(_KERNEL) && !defined(X86_FEATURE_AES)
+	return (B_FALSE);
+#else
+	return (__cpuid_has_aes());
+#endif
+}
+
+/*
+ * Check if PCLMULQDQ instruction set is available
+ */
+static inline boolean_t
+zfs_pclmulqdq_available(void)
+{
+#if defined(_KERNEL) && defined(X86_FEATURE_PCLMULQDQ)
+	return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ));
+#elif defined(_KERNEL) && !defined(X86_FEATURE_PCLMULQDQ)
+	return (B_FALSE);
+#else
+	return (__cpuid_has_pclmulqdq());
+#endif
+}
 
 /*
  * AVX-512 family of instruction sets:
diff --git a/include/sys/crypto/icp.h b/include/sys/crypto/icp.h
index d8948e022..4609e3a1d 100644
--- a/include/sys/crypto/icp.h
+++ b/include/sys/crypto/icp.h
@@ -44,4 +44,7 @@ int skein_mod_fini(void);
 int icp_init(void);
 void icp_fini(void);
 
+int aes_impl_set(const char *);
+int gcm_impl_set(const char *);
+
 #endif /* _SYS_CRYPTO_ALGS_H */