aboutsummaryrefslogtreecommitdiffstats
path: root/src/lib/modes/aead
diff options
context:
space:
mode:
authorJack Lloyd <[email protected]>2017-10-20 18:04:17 -0400
committerJack Lloyd <[email protected]>2017-10-20 18:04:17 -0400
commit2a5f2c78fcf44987e38c79ea92b95bacec6ee004 (patch)
tree6945fbdfec95e569097dd738c57d0075eae09096 /src/lib/modes/aead
parentb1566227fc22d4b94b1828b0916864a621ad22d5 (diff)
Add GHASH using SSSE3
About 30% faster than scalar on Skylake
Diffstat (limited to 'src/lib/modes/aead')
-rw-r--r--src/lib/modes/aead/gcm/clmul_ssse3/clmul_ssse3.cpp60
-rw-r--r--src/lib/modes/aead/gcm/clmul_ssse3/clmul_ssse3.h20
-rw-r--r--src/lib/modes/aead/gcm/clmul_ssse3/info.txt9
-rw-r--r--src/lib/modes/aead/gcm/ghash.cpp18
4 files changed, 105 insertions, 2 deletions
diff --git a/src/lib/modes/aead/gcm/clmul_ssse3/clmul_ssse3.cpp b/src/lib/modes/aead/gcm/clmul_ssse3/clmul_ssse3.cpp
new file mode 100644
index 000000000..3aace1094
--- /dev/null
+++ b/src/lib/modes/aead/gcm/clmul_ssse3/clmul_ssse3.cpp
@@ -0,0 +1,60 @@
+/*
+* (C) 2017 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/internal/clmul_ssse3.h>
+#include <immintrin.h>
+
+namespace Botan {
+
+BOTAN_FUNC_ISA("ssse3")
+void gcm_multiply_ssse3(uint8_t x[16],
+ const uint64_t HM[256],
+ const uint8_t input_bytes[], size_t blocks)
+ {
+ const __m128i BSWAP_MASK = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
+
+ const __m128i* HM_mm = reinterpret_cast<const __m128i*>(HM);
+
+ __m128i X = _mm_loadu_si128(reinterpret_cast<__m128i*>(x));
+ X = _mm_shuffle_epi8(X, BSWAP_MASK);
+
+ const __m128i ones = _mm_set1_epi8(-1);
+
+ for(size_t b = 0; b != blocks; ++b)
+ {
+ __m128i M = _mm_loadu_si128(reinterpret_cast<const __m128i*>(input_bytes) + b);
+ M = _mm_shuffle_epi8(M, BSWAP_MASK);
+
+ X = _mm_xor_si128(X, M);
+
+ __m128i Z = _mm_setzero_si128();
+
+ for(size_t i = 0; i != 64; i += 2)
+ {
+ const __m128i HM0 = _mm_load_si128(HM_mm + 2*i);
+ const __m128i HM1 = _mm_load_si128(HM_mm + 2*i + 1);
+ const __m128i HM2 = _mm_load_si128(HM_mm + 2*i + 2);
+ const __m128i HM3 = _mm_load_si128(HM_mm + 2*i + 3);
+
+ const __m128i XMASK1 = _mm_add_epi64(_mm_srli_epi64(X, 63), ones);
+ X = _mm_slli_epi64(X, 1);
+ const __m128i XMASK2 = _mm_add_epi64(_mm_srli_epi64(X, 63), ones);
+ X = _mm_slli_epi64(X, 1);
+
+ Z = _mm_xor_si128(Z, _mm_andnot_si128(_mm_unpackhi_epi64(XMASK1, XMASK1), HM0));
+ Z = _mm_xor_si128(Z, _mm_andnot_si128(_mm_unpacklo_epi64(XMASK1, XMASK1), HM1));
+ Z = _mm_xor_si128(Z, _mm_andnot_si128(_mm_unpackhi_epi64(XMASK2, XMASK2), HM2));
+ Z = _mm_xor_si128(Z, _mm_andnot_si128(_mm_unpacklo_epi64(XMASK2, XMASK2), HM3));
+ }
+
+ X = _mm_shuffle_epi32(Z, _MM_SHUFFLE(1, 0, 3, 2));
+ }
+
+ X = _mm_shuffle_epi8(X, BSWAP_MASK);
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(x), X);
+ }
+
+}
diff --git a/src/lib/modes/aead/gcm/clmul_ssse3/clmul_ssse3.h b/src/lib/modes/aead/gcm/clmul_ssse3/clmul_ssse3.h
new file mode 100644
index 000000000..7b1d19d9b
--- /dev/null
+++ b/src/lib/modes/aead/gcm/clmul_ssse3/clmul_ssse3.h
@@ -0,0 +1,20 @@
+/*
+* (C) 2017 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_GCM_CLMUL_SSSE3_H_
+#define BOTAN_GCM_CLMUL_SSSE3_H_
+
+#include <botan/types.h>
+
+namespace Botan {
+
+void gcm_multiply_ssse3(uint8_t x[16],
+ const uint64_t HM[256],
+ const uint8_t input[], size_t blocks);
+
+}
+
+#endif
diff --git a/src/lib/modes/aead/gcm/clmul_ssse3/info.txt b/src/lib/modes/aead/gcm/clmul_ssse3/info.txt
new file mode 100644
index 000000000..a802b5f20
--- /dev/null
+++ b/src/lib/modes/aead/gcm/clmul_ssse3/info.txt
@@ -0,0 +1,9 @@
+<defines>
+GCM_CLMUL_SSSE3 -> 20171020
+</defines>
+
+need_isa ssse3
+
+<header:internal>
+clmul_ssse3.h
+</header:internal>
diff --git a/src/lib/modes/aead/gcm/ghash.cpp b/src/lib/modes/aead/gcm/ghash.cpp
index c3c2453e8..51477f43a 100644
--- a/src/lib/modes/aead/gcm/ghash.cpp
+++ b/src/lib/modes/aead/gcm/ghash.cpp
@@ -15,6 +15,10 @@
#include <botan/internal/clmul.h>
#endif
+#if defined(BOTAN_HAS_GCM_CLMUL_SSSE3)
+ #include <botan/internal/clmul_ssse3.h>
+#endif
+
#if defined(BOTAN_HAS_GCM_PMULL)
#include <botan/internal/pmull.h>
#endif
@@ -28,6 +32,11 @@ std::string GHASH::provider() const
return "clmul";
#endif
+#if defined(BOTAN_HAS_GCM_CLMUL_SSSE3)
+ if(CPUID::has_ssse3())
+ return "ssse3";
+#endif
+
#if defined(BOTAN_HAS_GCM_PMULL)
if(CPUID::has_arm_pmull())
return "pmull";
@@ -47,6 +56,13 @@ void GHASH::gcm_multiply(secure_vector<uint8_t>& x,
}
#endif
+#if defined(BOTAN_HAS_GCM_CLMUL_SSSE3)
+ if(CPUID::has_ssse3())
+ {
+ return gcm_multiply_ssse3(x.data(), m_HM.data(), input, blocks);
+ }
+#endif
+
#if defined(BOTAN_HAS_GCM_PMULL)
if(CPUID::has_arm_pmull())
{
@@ -56,8 +72,6 @@ void GHASH::gcm_multiply(secure_vector<uint8_t>& x,
CT::poison(x.data(), x.size());
- // SSE2 might be useful here
-
const uint64_t ALL_BITS = 0xFFFFFFFFFFFFFFFF;
uint64_t X[2] = {