aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/lib/mac/gmac/gmac.cpp9
-rw-r--r--src/lib/mac/gmac/gmac.h12
-rw-r--r--src/lib/mac/mac.cpp2
-rw-r--r--src/lib/modes/aead/gcm/clmul/clmul.cpp203
-rw-r--r--src/lib/modes/aead/gcm/clmul/clmul.h5
-rw-r--r--src/lib/modes/aead/gcm/gcm.cpp216
-rw-r--r--src/lib/modes/aead/gcm/gcm.h58
-rw-r--r--src/lib/modes/aead/gcm/ghash.cpp250
-rw-r--r--src/lib/modes/aead/gcm/ghash.h78
-rw-r--r--src/lib/modes/aead/gcm/pmull/pmull.cpp238
-rw-r--r--src/lib/modes/aead/gcm/pmull/pmull.h5
-rw-r--r--src/lib/stream/ctr/ctr.cpp141
-rw-r--r--src/lib/stream/ctr/ctr.h8
13 files changed, 789 insertions, 436 deletions
diff --git a/src/lib/mac/gmac/gmac.cpp b/src/lib/mac/gmac/gmac.cpp
index be27aba4a..a4e84f57b 100644
--- a/src/lib/mac/gmac/gmac.cpp
+++ b/src/lib/mac/gmac/gmac.cpp
@@ -7,6 +7,8 @@
*/
#include <botan/gmac.h>
+#include <botan/ghash.h>
+#include <botan/block_cipher.h>
namespace Botan {
@@ -28,6 +30,13 @@ void GMAC::clear()
m_initialized = false;
}
+GMAC::~GMAC() { /* for unique_ptr */ }
+
+Key_Length_Specification GMAC::key_spec() const
+ {
+ return m_cipher->key_spec();
+ }
+
std::string GMAC::name() const
{
return "GMAC(" + m_cipher->name() + ")";
diff --git a/src/lib/mac/gmac/gmac.h b/src/lib/mac/gmac/gmac.h
index ef54a42bf..83094a5bc 100644
--- a/src/lib/mac/gmac/gmac.h
+++ b/src/lib/mac/gmac/gmac.h
@@ -10,11 +10,12 @@
#define BOTAN_GMAC_H_
#include <botan/mac.h>
-#include <botan/gcm.h>
-#include <botan/block_cipher.h>
namespace Botan {
+class BlockCipher;
+class GHASH;
+
/**
* GMAC
*
@@ -29,10 +30,7 @@ class BOTAN_PUBLIC_API(2,0) GMAC final : public MessageAuthenticationCode
size_t output_length() const override;
MessageAuthenticationCode* clone() const override;
- Key_Length_Specification key_spec() const override
- {
- return m_cipher->key_spec();
- }
+ Key_Length_Specification key_spec() const override;
/**
* Creates a new GMAC instance.
@@ -44,6 +42,8 @@ class BOTAN_PUBLIC_API(2,0) GMAC final : public MessageAuthenticationCode
GMAC(const GMAC&) = delete;
GMAC& operator=(const GMAC&) = delete;
+ virtual ~GMAC();
+
private:
void add_data(const uint8_t[], size_t) override;
void final_result(uint8_t[]) override;
diff --git a/src/lib/mac/mac.cpp b/src/lib/mac/mac.cpp
index 053e36b6f..65107470b 100644
--- a/src/lib/mac/mac.cpp
+++ b/src/lib/mac/mac.cpp
@@ -19,10 +19,12 @@
#if defined(BOTAN_HAS_GMAC)
#include <botan/gmac.h>
+ #include <botan/block_cipher.h>
#endif
#if defined(BOTAN_HAS_HMAC)
#include <botan/hmac.h>
+ #include <botan/hash.h>
#endif
#if defined(BOTAN_HAS_POLY1305)
diff --git a/src/lib/modes/aead/gcm/clmul/clmul.cpp b/src/lib/modes/aead/gcm/clmul/clmul.cpp
index 33378d833..632de6d33 100644
--- a/src/lib/modes/aead/gcm/clmul/clmul.cpp
+++ b/src/lib/modes/aead/gcm/clmul/clmul.cpp
@@ -1,6 +1,6 @@
/*
* CLMUL hook
-* (C) 2013 Jack Lloyd
+* (C) 2013,2017 Jack Lloyd
*
* Botan is released under the Simplified BSD License (see license.txt)
*/
@@ -11,73 +11,168 @@
namespace Botan {
-BOTAN_FUNC_ISA("pclmul,ssse3")
-void gcm_multiply_clmul(uint8_t x[16], const uint8_t H[16],
- const uint8_t input[], size_t blocks)
+namespace {
+
+BOTAN_FUNC_ISA("sse2")
+inline __m128i gcm_reduce(const __m128i& B0, const __m128i& B1)
+ {
+ __m128i T0, T1, T2, T3;
+
+ T0 = _mm_srli_epi32(B1, 31);
+ T1 = _mm_slli_epi32(B1, 1);
+ T2 = _mm_srli_epi32(B0, 31);
+ T3 = _mm_slli_epi32(B0, 1);
+
+ T3 = _mm_or_si128(T3, _mm_srli_si128(T0, 12));
+ T3 = _mm_or_si128(T3, _mm_slli_si128(T2, 4));
+ T1 = _mm_or_si128(T1, _mm_slli_si128(T0, 4));
+
+ T0 = _mm_xor_si128(_mm_slli_epi32(T1, 31), _mm_slli_epi32(T1, 30));
+ T0 = _mm_xor_si128(T0, _mm_slli_epi32(T1, 25));
+
+ T1 = _mm_xor_si128(T1, _mm_slli_si128(T0, 12));
+
+ T0 = _mm_xor_si128(T3, _mm_srli_si128(T0, 4));
+ T0 = _mm_xor_si128(T0, T1);
+ T0 = _mm_xor_si128(T0, _mm_srli_epi32(T1, 7));
+ T0 = _mm_xor_si128(T0, _mm_srli_epi32(T1, 1));
+ T0 = _mm_xor_si128(T0, _mm_srli_epi32(T1, 2));
+ return T0;
+ }
+
+BOTAN_FUNC_ISA("pclmul,sse2")
+inline __m128i gcm_multiply(const __m128i& H, const __m128i& x)
+ {
+ __m128i T0, T1, T2, T3, T4;
+
+ T0 = _mm_clmulepi64_si128(x, H, 0x11);
+ T1 = _mm_clmulepi64_si128(x, H, 0x10);
+ T2 = _mm_clmulepi64_si128(x, H, 0x01);
+ T3 = _mm_clmulepi64_si128(x, H, 0x00);
+
+ T1 = _mm_xor_si128(T1, T2);
+ T0 = _mm_xor_si128(T0, _mm_srli_si128(T1, 8));
+ T3 = _mm_xor_si128(T3, _mm_slli_si128(T1, 8));
+
+ return gcm_reduce(T0, T3);
+ }
+
+BOTAN_FUNC_ISA("pclmul,sse2")
+inline __m128i gcm_multiply_x4(const __m128i& H1, const __m128i& H2, const __m128i& H3, const __m128i& H4,
+ const __m128i& X1, const __m128i& X2, const __m128i& X3, const __m128i& X4)
+ {
+ /*
+ * Mutiply with delayed reduction, algorithm by Krzysztof Jankowski
+ * and Pierre Laurent of Intel
+ */
+
+ const __m128i H1_X1_lo = _mm_clmulepi64_si128(H1, X1, 0x00);
+ const __m128i H2_X2_lo = _mm_clmulepi64_si128(H2, X2, 0x00);
+ const __m128i H3_X3_lo = _mm_clmulepi64_si128(H3, X3, 0x00);
+ const __m128i H4_X4_lo = _mm_clmulepi64_si128(H4, X4, 0x00);
+
+ const __m128i lo = _mm_xor_si128(
+ _mm_xor_si128(H1_X1_lo, H2_X2_lo),
+ _mm_xor_si128(H3_X3_lo, H4_X4_lo));
+
+ const __m128i H1_X1_hi = _mm_clmulepi64_si128(H1, X1, 0x11);
+ const __m128i H2_X2_hi = _mm_clmulepi64_si128(H2, X2, 0x11);
+ const __m128i H3_X3_hi = _mm_clmulepi64_si128(H3, X3, 0x11);
+ const __m128i H4_X4_hi = _mm_clmulepi64_si128(H4, X4, 0x11);
+
+ const __m128i hi = _mm_xor_si128(
+ _mm_xor_si128(H1_X1_hi, H2_X2_hi),
+ _mm_xor_si128(H3_X3_hi, H4_X4_hi));
+
+ __m128i T0 = _mm_xor_si128(lo, hi);
+ __m128i T1, T2, T3, T4;
+
+ T1 = _mm_xor_si128(_mm_srli_si128(H1, 8), H1);
+ T2 = _mm_xor_si128(_mm_srli_si128(X1, 8), X1);
+ T3 = _mm_xor_si128(_mm_srli_si128(H2, 8), H2);
+ T4 = _mm_xor_si128(_mm_srli_si128(X2, 8), X2);
+ T0 = _mm_xor_si128(T0, _mm_clmulepi64_si128(T1, T2, 0x00));
+ T0 = _mm_xor_si128(T0, _mm_clmulepi64_si128(T3, T4, 0x00));
+
+ T1 = _mm_xor_si128(_mm_srli_si128(H3, 8), H3);
+ T2 = _mm_xor_si128(_mm_srli_si128(X3, 8), X3);
+ T3 = _mm_xor_si128(_mm_srli_si128(H4, 8), H4);
+ T4 = _mm_xor_si128(_mm_srli_si128(X4, 8), X4);
+ T0 = _mm_xor_si128(T0, _mm_clmulepi64_si128(T1, T2, 0x00));
+ T0 = _mm_xor_si128(T0, _mm_clmulepi64_si128(T3, T4, 0x00));
+
+ T1 = _mm_xor_si128(_mm_srli_si128(T0, 8), hi);
+ T2 = _mm_xor_si128(_mm_slli_si128(T0, 8), lo);
+
+ return gcm_reduce(T1, T2);
+ }
+
+}
+
+BOTAN_FUNC_ISA("ssse3")
+void gcm_clmul_precompute(const uint8_t H_bytes[16], uint64_t H_pow[4*2])
+ {
+ const __m128i BSWAP_MASK = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
+
+ const __m128i H = _mm_shuffle_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i*>(H_bytes)), BSWAP_MASK);
+ const __m128i H2 = gcm_multiply(H, H);
+ const __m128i H3 = gcm_multiply(H, H2);
+ const __m128i H4 = gcm_multiply(H, H3);
+
+ __m128i* H_pow_mm = reinterpret_cast<__m128i*>(H_pow);
+
+ _mm_storeu_si128(H_pow_mm+0, H);
+ _mm_storeu_si128(H_pow_mm+1, H2);
+ _mm_storeu_si128(H_pow_mm+2, H3);
+ _mm_storeu_si128(H_pow_mm+3, H4);
+ }
+
+BOTAN_FUNC_ISA("ssse3")
+void gcm_multiply_clmul(uint8_t x[16],
+ const uint64_t H_pow[8],
+ const uint8_t input_bytes[], size_t blocks)
{
/*
* Algorithms 1 and 5 from Intel's CLMUL guide
*/
const __m128i BSWAP_MASK = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
- const __m128i b = _mm_shuffle_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i*>(H)), BSWAP_MASK);
+ const __m128i* input = reinterpret_cast<const __m128i*>(input_bytes);
+
+ const __m128i* H_pow_mm = reinterpret_cast<const __m128i*>(H_pow);
+
+ const __m128i H = _mm_loadu_si128(H_pow_mm);
__m128i a = _mm_loadu_si128(reinterpret_cast<const __m128i*>(x));
a = _mm_shuffle_epi8(a, BSWAP_MASK);
+ if(blocks >= 4)
+ {
+ const __m128i H2 = _mm_loadu_si128(H_pow_mm + 1);
+ const __m128i H3 = _mm_loadu_si128(H_pow_mm + 2);
+ const __m128i H4 = _mm_loadu_si128(H_pow_mm + 3);
+
+ while(blocks >= 4)
+ {
+ const __m128i m0 = _mm_shuffle_epi8(_mm_loadu_si128(input + 0), BSWAP_MASK);
+ const __m128i m1 = _mm_shuffle_epi8(_mm_loadu_si128(input + 1), BSWAP_MASK);
+ const __m128i m2 = _mm_shuffle_epi8(_mm_loadu_si128(input + 2), BSWAP_MASK);
+ const __m128i m3 = _mm_shuffle_epi8(_mm_loadu_si128(input + 3), BSWAP_MASK);
+
+ a = _mm_xor_si128(a, m0);
+ a = gcm_multiply_x4(H, H2, H3, H4, m3, m2, m1, a);
+
+ input += 4;
+ blocks -= 4;
+ }
+ }
+
for(size_t i = 0; i != blocks; ++i)
{
- __m128i m = _mm_loadu_si128(reinterpret_cast<const __m128i*>(input) + i);
- m = _mm_shuffle_epi8(m, BSWAP_MASK);
+ const __m128i m = _mm_shuffle_epi8(_mm_loadu_si128(input + i), BSWAP_MASK);
a = _mm_xor_si128(a, m);
-
- __m128i T0, T1, T2, T3, T4, T5;
-
- T0 = _mm_clmulepi64_si128(a, b, 0x00);
- T1 = _mm_clmulepi64_si128(a, b, 0x01);
- T2 = _mm_clmulepi64_si128(a, b, 0x10);
- T3 = _mm_clmulepi64_si128(a, b, 0x11);
-
- T1 = _mm_xor_si128(T1, T2);
- T2 = _mm_slli_si128(T1, 8);
- T1 = _mm_srli_si128(T1, 8);
- T0 = _mm_xor_si128(T0, T2);
- T3 = _mm_xor_si128(T3, T1);
-
- T4 = _mm_srli_epi32(T0, 31);
- T0 = _mm_slli_epi32(T0, 1);
-
- T5 = _mm_srli_epi32(T3, 31);
- T3 = _mm_slli_epi32(T3, 1);
-
- T2 = _mm_srli_si128(T4, 12);
- T5 = _mm_slli_si128(T5, 4);
- T4 = _mm_slli_si128(T4, 4);
- T0 = _mm_or_si128(T0, T4);
- T3 = _mm_or_si128(T3, T5);
- T3 = _mm_or_si128(T3, T2);
-
- T4 = _mm_slli_epi32(T0, 31);
- T5 = _mm_slli_epi32(T0, 30);
- T2 = _mm_slli_epi32(T0, 25);
-
- T4 = _mm_xor_si128(T4, T5);
- T4 = _mm_xor_si128(T4, T2);
- T5 = _mm_srli_si128(T4, 4);
- T3 = _mm_xor_si128(T3, T5);
- T4 = _mm_slli_si128(T4, 12);
- T0 = _mm_xor_si128(T0, T4);
- T3 = _mm_xor_si128(T3, T0);
-
- T4 = _mm_srli_epi32(T0, 1);
- T1 = _mm_srli_epi32(T0, 2);
- T2 = _mm_srli_epi32(T0, 7);
- T3 = _mm_xor_si128(T3, T1);
- T3 = _mm_xor_si128(T3, T2);
- T3 = _mm_xor_si128(T3, T4);
-
- a = T3;
+ a = gcm_multiply(H, a);
}
a = _mm_shuffle_epi8(a, BSWAP_MASK);
diff --git a/src/lib/modes/aead/gcm/clmul/clmul.h b/src/lib/modes/aead/gcm/clmul/clmul.h
index d68e021d2..25cdfbd96 100644
--- a/src/lib/modes/aead/gcm/clmul/clmul.h
+++ b/src/lib/modes/aead/gcm/clmul/clmul.h
@@ -12,7 +12,10 @@
namespace Botan {
-void gcm_multiply_clmul(uint8_t x[16], const uint8_t H[16],
+void gcm_clmul_precompute(const uint8_t H[16], uint64_t H_pow[4*2]);
+
+void gcm_multiply_clmul(uint8_t x[16],
+ const uint64_t H_pow[4*2],
const uint8_t input[], size_t blocks);
}
diff --git a/src/lib/modes/aead/gcm/gcm.cpp b/src/lib/modes/aead/gcm/gcm.cpp
index 57272f2ac..dfaffedb7 100644
--- a/src/lib/modes/aead/gcm/gcm.cpp
+++ b/src/lib/modes/aead/gcm/gcm.cpp
@@ -7,213 +7,12 @@
*/
#include <botan/gcm.h>
+#include <botan/ghash.h>
#include <botan/block_cipher.h>
-#include <botan/internal/ct_utils.h>
-#include <botan/loadstor.h>
#include <botan/ctr.h>
-#include <botan/cpuid.h>
-
-#if defined(BOTAN_HAS_GCM_CLMUL)
- #include <botan/internal/clmul.h>
-#endif
-
-#if defined(BOTAN_HAS_GCM_PMULL)
- #include <botan/internal/pmull.h>
-#endif
namespace Botan {
-static const size_t GCM_BS = 16;
-
-void GHASH::gcm_multiply(secure_vector<uint8_t>& x,
- const uint8_t input[],
- size_t blocks)
- {
-#if defined(BOTAN_HAS_GCM_CLMUL)
- if(CPUID::has_clmul())
- {
- return gcm_multiply_clmul(x.data(), m_H.data(), input, blocks);
- }
-#endif
-
-#if defined(BOTAN_HAS_GCM_PMULL)
- if(CPUID::has_arm_pmull())
- {
- return gcm_multiply_pmull(x.data(), m_H.data(), input, blocks);
- }
-#endif
-
- CT::poison(x.data(), x.size());
-
- // SSE2 might be useful here
-
- const uint64_t ALL_BITS = 0xFFFFFFFFFFFFFFFF;
-
- uint64_t X[2] = {
- load_be<uint64_t>(x.data(), 0),
- load_be<uint64_t>(x.data(), 1)
- };
-
- for(size_t b = 0; b != blocks; ++b)
- {
- X[0] ^= load_be<uint64_t>(input, 2*b);
- X[1] ^= load_be<uint64_t>(input, 2*b+1);
-
- uint64_t Z[2] = { 0, 0 };
-
- for(size_t i = 0; i != 64; ++i)
- {
- const uint64_t X0MASK = (ALL_BITS + (X[0] >> 63)) ^ ALL_BITS;
- const uint64_t X1MASK = (ALL_BITS + (X[1] >> 63)) ^ ALL_BITS;
-
- X[0] <<= 1;
- X[1] <<= 1;
-
- Z[0] ^= m_HM[4*i ] & X0MASK;
- Z[1] ^= m_HM[4*i+1] & X0MASK;
- Z[0] ^= m_HM[4*i+2] & X1MASK;
- Z[1] ^= m_HM[4*i+3] & X1MASK;
- }
-
- X[0] = Z[0];
- X[1] = Z[1];
- }
-
- store_be<uint64_t>(x.data(), X[0], X[1]);
- CT::unpoison(x.data(), x.size());
- }
-
-void GHASH::ghash_update(secure_vector<uint8_t>& ghash,
- const uint8_t input[], size_t length)
- {
- /*
- This assumes if less than block size input then we're just on the
- final block and should pad with zeros
- */
-
- const size_t full_blocks = length / GCM_BS;
- const size_t final_bytes = length - (full_blocks * GCM_BS);
-
- if(full_blocks > 0)
- {
- gcm_multiply(ghash, input, full_blocks);
- }
-
- if(final_bytes)
- {
- secure_vector<uint8_t> last_block(GCM_BS);
- copy_mem(last_block.data(), input + full_blocks * GCM_BS, final_bytes);
- gcm_multiply(ghash, last_block.data(), 1);
- }
- }
-
-void GHASH::key_schedule(const uint8_t key[], size_t length)
- {
- m_H.assign(key, key+length);
- m_H_ad.resize(GCM_BS);
- m_ad_len = 0;
- m_text_len = 0;
-
- uint64_t H0 = load_be<uint64_t>(m_H.data(), 0);
- uint64_t H1 = load_be<uint64_t>(m_H.data(), 1);
-
- const uint64_t R = 0xE100000000000000;
-
- m_HM.resize(256);
-
- // precompute the multiples of H
- for(size_t i = 0; i != 2; ++i)
- {
- for(size_t j = 0; j != 64; ++j)
- {
- /*
- we interleave H^1, H^65, H^2, H^66, ...
- to make indexing nicer in the multiplication code
- */
- m_HM[4*j+2*i] = H0;
- m_HM[4*j+2*i+1] = H1;
-
- // GCM's bit ops are reversed so we carry out of the bottom
- const uint64_t carry = R * (H1 & 1);
- H1 = (H1 >> 1) | (H0 << 63);
- H0 = (H0 >> 1) ^ carry;
- }
- }
- }
-
-void GHASH::start(const uint8_t nonce[], size_t len)
- {
- m_nonce.assign(nonce, nonce + len);
- m_ghash = m_H_ad;
- }
-
-void GHASH::set_associated_data(const uint8_t input[], size_t length)
- {
- zeroise(m_H_ad);
-
- ghash_update(m_H_ad, input, length);
- m_ad_len = length;
- }
-
-void GHASH::update_associated_data(const uint8_t ad[], size_t length)
- {
- BOTAN_ASSERT(m_ghash.size() == GCM_BS, "Key was set");
- m_ad_len += length;
- ghash_update(m_ghash, ad, length);
- }
-
-void GHASH::update(const uint8_t input[], size_t length)
- {
- BOTAN_ASSERT(m_ghash.size() == GCM_BS, "Key was set");
- m_text_len += length;
- ghash_update(m_ghash, input, length);
- }
-
-void GHASH::add_final_block(secure_vector<uint8_t>& hash,
- size_t ad_len, size_t text_len)
- {
- secure_vector<uint8_t> final_block(GCM_BS);
- store_be<uint64_t>(final_block.data(), 8*ad_len, 8*text_len);
- ghash_update(hash, final_block.data(), final_block.size());
- }
-
-secure_vector<uint8_t> GHASH::final()
- {
- add_final_block(m_ghash, m_ad_len, m_text_len);
-
- secure_vector<uint8_t> mac;
- mac.swap(m_ghash);
-
- mac ^= m_nonce;
- m_text_len = 0;
- return mac;
- }
-
-secure_vector<uint8_t> GHASH::nonce_hash(const uint8_t nonce[], size_t nonce_len)
- {
- BOTAN_ASSERT(m_ghash.size() == 0, "nonce_hash called during wrong time");
- secure_vector<uint8_t> y0(GCM_BS);
-
- ghash_update(y0, nonce, nonce_len);
- add_final_block(y0, 0, nonce_len);
-
- return y0;
- }
-
-void GHASH::clear()
- {
- zeroise(m_H);
- reset();
- }
-
-void GHASH::reset()
- {
- zeroise(m_H_ad);
- m_ghash.clear();
- m_nonce.clear();
- m_text_len = m_ad_len = 0;
- }
-
/*
* GCM_Mode Constructor
*/
@@ -255,12 +54,7 @@ std::string GCM_Mode::name() const
std::string GCM_Mode::provider() const
{
-#if defined(BOTAN_HAS_GCM_CLMUL)
- if(CPUID::has_clmul())
- return "clmul";
-#endif
-
- return "base";
+ return m_ghash->provider();
}
size_t GCM_Mode::update_granularity() const
@@ -309,10 +103,10 @@ void GCM_Mode::start_msg(const uint8_t nonce[], size_t nonce_len)
m_ctr->set_iv(y0.data(), y0.size());
- secure_vector<uint8_t> m_enc_y0(GCM_BS);
- m_ctr->encipher(m_enc_y0);
+ zeroise(y0);
+ m_ctr->encipher(y0);
- m_ghash->start(m_enc_y0.data(), m_enc_y0.size());
+ m_ghash->start(y0.data(), y0.size());
}
size_t GCM_Encryption::process(uint8_t buf[], size_t sz)
diff --git a/src/lib/modes/aead/gcm/gcm.h b/src/lib/modes/aead/gcm/gcm.h
index eac2add93..de7c1ea9a 100644
--- a/src/lib/modes/aead/gcm/gcm.h
+++ b/src/lib/modes/aead/gcm/gcm.h
@@ -47,7 +47,7 @@ class BOTAN_PUBLIC_API(2,0) GCM_Mode : public AEAD_Mode
~GCM_Mode();
- const size_t m_BS = 16;
+ static const size_t GCM_BS = 16;
const size_t m_tag_size;
const std::string m_cipher_name;
@@ -109,62 +109,6 @@ class BOTAN_PUBLIC_API(2,0) GCM_Decryption final : public GCM_Mode
void finish(secure_vector<uint8_t>& final_block, size_t offset = 0) override;
};
-/**
-* GCM's GHASH
-* This is not intended for general use, but is exposed to allow
-* shared code between GCM and GMAC
-*/
-class BOTAN_PUBLIC_API(2,0) GHASH final : public SymmetricAlgorithm
- {
- public:
- void set_associated_data(const uint8_t ad[], size_t ad_len);
-
- secure_vector<uint8_t> nonce_hash(const uint8_t nonce[], size_t len);
-
- void start(const uint8_t nonce[], size_t len);
-
- /*
- * Assumes input len is multiple of 16
- */
- void update(const uint8_t in[], size_t len);
-
- /*
- * Incremental update of associated data
- */
- void update_associated_data(const uint8_t ad[], size_t len);
-
- secure_vector<uint8_t> final();
-
- Key_Length_Specification key_spec() const override
- { return Key_Length_Specification(16); }
-
- void clear() override;
-
- void reset();
-
- std::string name() const override { return "GHASH"; }
-
- void ghash_update(secure_vector<uint8_t>& x,
- const uint8_t input[], size_t input_len);
-
- void add_final_block(secure_vector<uint8_t>& x,
- size_t ad_len, size_t pt_len);
- private:
- void key_schedule(const uint8_t key[], size_t key_len) override;
-
- void gcm_multiply(secure_vector<uint8_t>& x,
- const uint8_t input[],
- size_t blocks);
-
- secure_vector<uint8_t> m_H;
- secure_vector<uint8_t> m_H_ad;
- secure_vector<uint8_t> m_ghash;
- secure_vector<uint8_t> m_nonce;
- secure_vector<uint64_t> m_HM;
- size_t m_ad_len = 0;
- size_t m_text_len = 0;
- };
-
}
#endif
diff --git a/src/lib/modes/aead/gcm/ghash.cpp b/src/lib/modes/aead/gcm/ghash.cpp
new file mode 100644
index 000000000..c3c2453e8
--- /dev/null
+++ b/src/lib/modes/aead/gcm/ghash.cpp
@@ -0,0 +1,250 @@
+/*
+* GCM GHASH
+* (C) 2013,2015,2017 Jack Lloyd
+* (C) 2016 Daniel Neus, Rohde & Schwarz Cybersecurity
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/ghash.h>
+#include <botan/internal/ct_utils.h>
+#include <botan/loadstor.h>
+#include <botan/cpuid.h>
+
+#if defined(BOTAN_HAS_GCM_CLMUL)
+ #include <botan/internal/clmul.h>
+#endif
+
+#if defined(BOTAN_HAS_GCM_PMULL)
+ #include <botan/internal/pmull.h>
+#endif
+
+namespace Botan {
+
+std::string GHASH::provider() const
+ {
+#if defined(BOTAN_HAS_GCM_CLMUL)
+ if(CPUID::has_clmul())
+ return "clmul";
+#endif
+
+#if defined(BOTAN_HAS_GCM_PMULL)
+ if(CPUID::has_arm_pmull())
+ return "pmull";
+#endif
+
+ return "base";
+ }
+
+void GHASH::gcm_multiply(secure_vector<uint8_t>& x,
+ const uint8_t input[],
+ size_t blocks)
+ {
+#if defined(BOTAN_HAS_GCM_CLMUL)
+ if(CPUID::has_clmul())
+ {
+ return gcm_multiply_clmul(x.data(), m_H_pow.data(), input, blocks);
+ }
+#endif
+
+#if defined(BOTAN_HAS_GCM_PMULL)
+ if(CPUID::has_arm_pmull())
+ {
+ return gcm_multiply_pmull(x.data(), m_H_pow.data(), input, blocks);
+ }
+#endif
+
+ CT::poison(x.data(), x.size());
+
+ // SSE2 might be useful here
+
+ const uint64_t ALL_BITS = 0xFFFFFFFFFFFFFFFF;
+
+ uint64_t X[2] = {
+ load_be<uint64_t>(x.data(), 0),
+ load_be<uint64_t>(x.data(), 1)
+ };
+
+ for(size_t b = 0; b != blocks; ++b)
+ {
+ X[0] ^= load_be<uint64_t>(input, 2*b);
+ X[1] ^= load_be<uint64_t>(input, 2*b+1);
+
+ uint64_t Z[2] = { 0, 0 };
+
+ for(size_t i = 0; i != 64; ++i)
+ {
+ const uint64_t X0MASK = (ALL_BITS + (X[0] >> 63)) ^ ALL_BITS;
+ const uint64_t X1MASK = (ALL_BITS + (X[1] >> 63)) ^ ALL_BITS;
+
+ X[0] <<= 1;
+ X[1] <<= 1;
+
+ Z[0] ^= m_HM[4*i ] & X0MASK;
+ Z[1] ^= m_HM[4*i+1] & X0MASK;
+ Z[0] ^= m_HM[4*i+2] & X1MASK;
+ Z[1] ^= m_HM[4*i+3] & X1MASK;
+ }
+
+ X[0] = Z[0];
+ X[1] = Z[1];
+ }
+
+ store_be<uint64_t>(x.data(), X[0], X[1]);
+ CT::unpoison(x.data(), x.size());
+ }
+
+void GHASH::ghash_update(secure_vector<uint8_t>& ghash,
+ const uint8_t input[], size_t length)
+ {
+ /*
+ This assumes if less than block size input then we're just on the
+ final block and should pad with zeros
+ */
+
+ const size_t full_blocks = length / GCM_BS;
+ const size_t final_bytes = length - (full_blocks * GCM_BS);
+
+ if(full_blocks > 0)
+ {
+ gcm_multiply(ghash, input, full_blocks);
+ }
+
+ if(final_bytes)
+ {
+ secure_vector<uint8_t> last_block(GCM_BS);
+ copy_mem(last_block.data(), input + full_blocks * GCM_BS, final_bytes);
+ gcm_multiply(ghash, last_block.data(), 1);
+ }
+ }
+
+void GHASH::key_schedule(const uint8_t key[], size_t length)
+ {
+ m_H.assign(key, key+length);
+ m_H_ad.resize(GCM_BS);
+ m_ad_len = 0;
+ m_text_len = 0;
+
+ uint64_t H0 = load_be<uint64_t>(m_H.data(), 0);
+ uint64_t H1 = load_be<uint64_t>(m_H.data(), 1);
+
+ const uint64_t R = 0xE100000000000000;
+
+ m_HM.resize(256);
+
+ // precompute the multiples of H
+ for(size_t i = 0; i != 2; ++i)
+ {
+ for(size_t j = 0; j != 64; ++j)
+ {
+ /*
+ we interleave H^1, H^65, H^2, H^66, H3, H67, H4, H68
+ to make indexing nicer in the multiplication code
+ */
+ m_HM[4*j+2*i] = H0;
+ m_HM[4*j+2*i+1] = H1;
+
+ // GCM's bit ops are reversed so we carry out of the bottom
+ const uint64_t carry = R * (H1 & 1);
+ H1 = (H1 >> 1) | (H0 << 63);
+ H0 = (H0 >> 1) ^ carry;
+ }
+ }
+
+#if defined(BOTAN_HAS_GCM_CLMUL)
+ if(CPUID::has_clmul())
+ {
+ m_H_pow.resize(8);
+ gcm_clmul_precompute(m_H.data(), m_H_pow.data());
+ }
+#endif
+
+#if defined(BOTAN_HAS_GCM_PMULL)
+ if(CPUID::has_arm_pmull())
+ {
+ m_H_pow.resize(8);
+ gcm_pmull_precompute(m_H.data(), m_H_pow.data());
+ }
+#endif
+
+ }
+
+void GHASH::start(const uint8_t nonce[], size_t len)
+ {
+ m_nonce.assign(nonce, nonce + len);
+ m_ghash = m_H_ad;
+ }
+
+void GHASH::set_associated_data(const uint8_t input[], size_t length)
+ {
+ zeroise(m_H_ad);
+
+ ghash_update(m_H_ad, input, length);
+ m_ad_len = length;
+ }
+
+void GHASH::update_associated_data(const uint8_t ad[], size_t length)
+ {
+ BOTAN_ASSERT(m_ghash.size() == GCM_BS, "Key was set");
+ m_ad_len += length;
+ ghash_update(m_ghash, ad, length);
+ }
+
+void GHASH::update(const uint8_t input[], size_t length)
+ {
+ BOTAN_ASSERT(m_ghash.size() == GCM_BS, "Key was set");
+ m_text_len += length;
+ ghash_update(m_ghash, input, length);
+ }
+
+void GHASH::add_final_block(secure_vector<uint8_t>& hash,
+ size_t ad_len, size_t text_len)
+ {
+ /*
+ * stack buffer is fine here since the text len is public
+ * and the length of the AD is probably not sensitive either.
+ */
+ uint8_t final_block[GCM_BS];
+ store_be<uint64_t>(final_block, 8*ad_len, 8*text_len);
+ ghash_update(hash, final_block, GCM_BS);
+ }
+
+secure_vector<uint8_t> GHASH::final()
+ {
+ add_final_block(m_ghash, m_ad_len, m_text_len);
+
+ secure_vector<uint8_t> mac;
+ mac.swap(m_ghash);
+
+ mac ^= m_nonce;
+ m_text_len = 0;
+ return mac;
+ }
+
+secure_vector<uint8_t> GHASH::nonce_hash(const uint8_t nonce[], size_t nonce_len)
+ {
+ BOTAN_ASSERT(m_ghash.size() == 0, "nonce_hash called during wrong time");
+ secure_vector<uint8_t> y0(GCM_BS);
+
+ ghash_update(y0, nonce, nonce_len);
+ add_final_block(y0, 0, nonce_len);
+
+ return y0;
+ }
+
+void GHASH::clear()
+ {
+ zeroise(m_H);
+ zeroise(m_HM);
+ reset();
+ }
+
+void GHASH::reset()
+ {
+ zeroise(m_H_ad);
+ m_ghash.clear();
+ m_nonce.clear();
+ m_text_len = m_ad_len = 0;
+ }
+
+}
diff --git a/src/lib/modes/aead/gcm/ghash.h b/src/lib/modes/aead/gcm/ghash.h
new file mode 100644
index 000000000..7fcf7cfaa
--- /dev/null
+++ b/src/lib/modes/aead/gcm/ghash.h
@@ -0,0 +1,78 @@
+/*
+* (C) 2013 Jack Lloyd
+* (C) 2016 Daniel Neus, Rohde & Schwarz Cybersecurity
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_GCM_GHASH_H_
+#define BOTAN_GCM_GHASH_H_
+
+#include <botan/sym_algo.h>
+
+namespace Botan {
+
+/**
+* GCM's GHASH
+* This is not intended for general use, but is exposed to allow
+* shared code between GCM and GMAC
+*/
+class BOTAN_PUBLIC_API(2,0) GHASH final : public SymmetricAlgorithm
+ {
+ public:
+ void set_associated_data(const uint8_t ad[], size_t ad_len);
+
+ secure_vector<uint8_t> nonce_hash(const uint8_t nonce[], size_t len);
+
+ void start(const uint8_t nonce[], size_t len);
+
+ /*
+ * Assumes input len is multiple of 16
+ */
+ void update(const uint8_t in[], size_t len);
+
+ /*
+ * Incremental update of associated data
+ */
+ void update_associated_data(const uint8_t ad[], size_t len);
+
+ secure_vector<uint8_t> final();
+
+ Key_Length_Specification key_spec() const override
+ { return Key_Length_Specification(16); }
+
+ void clear() override;
+
+ void reset();
+
+ std::string name() const override { return "GHASH"; }
+
+ std::string provider() const;
+
+ void ghash_update(secure_vector<uint8_t>& x,
+ const uint8_t input[], size_t input_len);
+
+ void add_final_block(secure_vector<uint8_t>& x,
+ size_t ad_len, size_t pt_len);
+ private:
+ void key_schedule(const uint8_t key[], size_t key_len) override;
+
+ void gcm_multiply(secure_vector<uint8_t>& x,
+ const uint8_t input[],
+ size_t blocks);
+
+ static const size_t GCM_BS = 16;
+
+ secure_vector<uint8_t> m_H;
+ secure_vector<uint8_t> m_H_ad;
+ secure_vector<uint8_t> m_ghash;
+ secure_vector<uint8_t> m_nonce;
+ secure_vector<uint64_t> m_HM;
+ secure_vector<uint64_t> m_H_pow;
+ size_t m_ad_len = 0;
+ size_t m_text_len = 0;
+ };
+
+}
+
+#endif
diff --git a/src/lib/modes/aead/gcm/pmull/pmull.cpp b/src/lib/modes/aead/gcm/pmull/pmull.cpp
index 12d6ff7d1..77eb1909f 100644
--- a/src/lib/modes/aead/gcm/pmull/pmull.cpp
+++ b/src/lib/modes/aead/gcm/pmull/pmull.cpp
@@ -1,6 +1,9 @@
/*
* Contributed by Jeffrey Walton
*
+* Further changes
+* (C) 2017 Jack Lloyd
+*
* Botan is released under the Simplified BSD License (see license.txt)
*/
@@ -9,74 +12,191 @@
namespace Botan {
+/*
+This follows the same pattern as the clmul implementation.
+
+See also http://conradoplg.cryptoland.net/files/2010/12/gcm14.pdf
+*/
+
+namespace {
+
+BOTAN_FUNC_ISA("+simd")
+inline uint64x2_t gcm_reduce(uint32x4_t B0, uint32x4_t B1)
+ {
+ const uint32x4_t zero = vdupq_n_u32(0);
+
+ uint32x4_t T0, T1, T2, T3, T4, T5;
+
+ T4 = vshrq_n_u32(B0, 31);
+ T0 = vshlq_n_u32(B0, 1);
+ T5 = vshrq_n_u32(B1, 31);
+ T3 = vshlq_n_u32(B1, 1);
+
+ T2 = vextq_u32(T4, zero, 3);
+ T5 = vextq_u32(zero, T5, 3);
+ T4 = vextq_u32(zero, T4, 3);
+ T0 = vorrq_u32(T0, T4);
+ T3 = vorrq_u32(T3, T5);
+ T3 = vorrq_u32(T3, T2);
+
+ T4 = vshlq_n_u32(T0, 31);
+ T5 = vshlq_n_u32(T0, 30);
+ T2 = vshlq_n_u32(T0, 25);
+
+ T4 = veorq_u32(T4, T5);
+ T4 = veorq_u32(T4, T2);
+ T5 = vextq_u32(T4, zero, 1);
+ T3 = veorq_u32(T3, T5);
+ T4 = vextq_u32(zero, T4, 1);
+ T0 = veorq_u32(T0, T4);
+ T3 = veorq_u32(T3, T0);
+
+ T4 = vshrq_n_u32(T0, 1);
+ T1 = vshrq_n_u32(T0, 2);
+ T2 = vshrq_n_u32(T0, 7);
+ T3 = veorq_u32(T3, T1);
+ T3 = veorq_u32(T3, T2);
+ T3 = veorq_u32(T3, T4);
+
+ return vreinterpretq_u64_u32(T3);
+ }
+
+BOTAN_FUNC_ISA("+crypto")
+inline uint64x2_t gcm_multiply(uint64x2_t H, uint64x2_t x)
+ {
+ const uint32x4_t zero = vdupq_n_u32(0);
+
+ const uint64_t x_hi = vgetq_lane_u64(x, 0);
+ const uint64_t x_lo = vgetq_lane_u64(x, 1);
+ const uint64_t H_hi = vgetq_lane_u64(H, 0);
+ const uint64_t H_lo = vgetq_lane_u64(H, 1);
+
+ uint32x4_t T0 = (uint32x4_t)vmull_p64(x_hi, H_hi);
+ uint32x4_t T1 = (uint32x4_t)vmull_p64(x_lo, H_hi);
+ uint32x4_t T2 = (uint32x4_t)vmull_p64(x_hi, H_lo);
+ uint32x4_t T3 = (uint32x4_t)vmull_p64(x_lo, H_lo);
+
+ T1 = veorq_u32(T1, T2);
+ T0 = veorq_u32(T0, vextq_u32(zero, T1, 2));
+ T3 = veorq_u32(T3, vextq_u32(T1, zero, 2));
+
+ return gcm_reduce(T0, T3);
+ }
+
BOTAN_FUNC_ISA("+crypto")
-void gcm_multiply_pmull(uint8_t x[16], const uint8_t H[16],
+inline uint64x2_t gcm_multiply_x4(uint64x2_t H1, uint64x2_t H2, uint64x2_t H3, uint64x2_t H4,
+ uint64x2_t X1, uint64x2_t X2, uint64x2_t X3, uint64x2_t X4)
+ {
+ const uint64_t H1_hi = vgetq_lane_u64(H1, 0);
+ const uint64_t H1_lo = vgetq_lane_u64(H1, 1);
+ const uint64_t H2_hi = vgetq_lane_u64(H2, 0);
+ const uint64_t H2_lo = vgetq_lane_u64(H2, 1);
+ const uint64_t H3_hi = vgetq_lane_u64(H3, 0);
+ const uint64_t H3_lo = vgetq_lane_u64(H3, 1);
+ const uint64_t H4_hi = vgetq_lane_u64(H4, 0);
+ const uint64_t H4_lo = vgetq_lane_u64(H4, 1);
+
+ const uint64_t X1_hi = vgetq_lane_u64(X1, 0);
+ const uint64_t X1_lo = vgetq_lane_u64(X1, 1);
+ const uint64_t X2_hi = vgetq_lane_u64(X2, 0);
+ const uint64_t X2_lo = vgetq_lane_u64(X2, 1);
+ const uint64_t X3_hi = vgetq_lane_u64(X3, 0);
+ const uint64_t X3_lo = vgetq_lane_u64(X3, 1);
+ const uint64_t X4_hi = vgetq_lane_u64(X4, 0);
+ const uint64_t X4_lo = vgetq_lane_u64(X4, 1);
+
+ const uint32x4_t H1_X1_lo = (uint32x4_t)vmull_p64(X1_lo, H1_lo);
+ const uint32x4_t H2_X2_lo = (uint32x4_t)vmull_p64(X2_lo, H2_lo);
+ const uint32x4_t H3_X3_lo = (uint32x4_t)vmull_p64(X3_lo, H3_lo);
+ const uint32x4_t H4_X4_lo = (uint32x4_t)vmull_p64(X4_lo, H4_lo);
+
+ const uint32x4_t lo = veorq_u32(
+ veorq_u32(H1_X1_lo, H2_X2_lo),
+ veorq_u32(H3_X3_lo, H4_X4_lo));
+
+ const uint32x4_t H1_X1_hi = (uint32x4_t)vmull_p64(X1_hi, H1_hi);
+ const uint32x4_t H2_X2_hi = (uint32x4_t)vmull_p64(X2_hi, H2_hi);
+ const uint32x4_t H3_X3_hi = (uint32x4_t)vmull_p64(X3_hi, H3_hi);
+ const uint32x4_t H4_X4_hi = (uint32x4_t)vmull_p64(X4_hi, H4_hi);
+
+ const uint32x4_t hi = veorq_u32(
+ veorq_u32(H1_X1_hi, H2_X2_hi),
+ veorq_u32(H3_X3_hi, H4_X4_hi));
+
+ uint32x4_t T0 = veorq_u32(lo, hi);
+
+ T0 = veorq_u32(T0, (uint32x4_t)vmull_p64(X1_hi ^ X1_lo, H1_hi ^ H1_lo));
+ T0 = veorq_u32(T0, (uint32x4_t)vmull_p64(X2_hi ^ X2_lo, H2_hi ^ H2_lo));
+ T0 = veorq_u32(T0, (uint32x4_t)vmull_p64(X3_hi ^ X3_lo, H3_hi ^ H3_lo));
+ T0 = veorq_u32(T0, (uint32x4_t)vmull_p64(X4_hi ^ X4_lo, H4_hi ^ H4_lo));
+
+ const uint32x4_t zero = vdupq_n_u32(0);
+ uint32x4_t B0 = veorq_u32(vextq_u32(zero, T0, 2), hi);
+ uint32x4_t B1 = veorq_u32(vextq_u32(T0, zero, 2), lo);
+ return gcm_reduce(B0, B1);
+ }
+
+BOTAN_FUNC_ISA("+simd")
+inline uint8x16_t bswap_vec(uint8x16_t v)
+ {
+ const uint8_t maskb[16] = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
+ const uint8x16_t mask = vld1q_u8(maskb);
+ return vqtbl1q_u8(v, mask);
+ }
+
+}
+
+BOTAN_FUNC_ISA("+simd")
+void gcm_pmull_precompute(const uint8_t H_bytes[16], uint64_t H_pow[4*2])
+ {
+ const uint64x2_t H = vreinterpretq_u64_u8(bswap_vec(vld1q_u8(H_bytes)));
+ const uint64x2_t H2 = gcm_multiply(H, H);
+ const uint64x2_t H3 = gcm_multiply(H, H2);
+ const uint64x2_t H4 = gcm_multiply(H, H3);
+
+ vst1q_u64(H_pow , H);
+ vst1q_u64(H_pow+2, H2);
+ vst1q_u64(H_pow+4, H3);
+ vst1q_u64(H_pow+6, H4);
+ }
+
+BOTAN_FUNC_ISA("+simd")
+void gcm_multiply_pmull(uint8_t x[16],
+ const uint64_t H64[8],
const uint8_t input[], size_t blocks)
{
- /*
- * Implementing GCM on ARMv8, http://conradoplg.cryptoland.net/files/2010/12/gcm14.pdf
- */
+ const uint64x2_t H = vld1q_u64(H64);
+ uint64x2_t a = vreinterpretq_u64_u8(bswap_vec(vld1q_u8(x)));
- uint64x2_t a64 = vreinterpretq_u64_u8(vcombine_u8(vrev64_u8(vld1_u8(x+8)), vrev64_u8(vld1_u8(x))));
- const uint64x2_t b64 = vreinterpretq_u64_u8(vcombine_u8(vrev64_u8(vld1_u8(H+8)), vrev64_u8(vld1_u8(H))));
+ if(blocks >= 4)
+ {
+ const uint64x2_t H2 = vld1q_u64(H64 + 2);
+ const uint64x2_t H3 = vld1q_u64(H64 + 4);
+ const uint64x2_t H4 = vld1q_u64(H64 + 6);
+
+ while(blocks >= 4)
+ {
+ const uint64x2_t m0 = vreinterpretq_u64_u8(bswap_vec(vld1q_u8(input)));
+ const uint64x2_t m1 = vreinterpretq_u64_u8(bswap_vec(vld1q_u8(input + 16)));
+ const uint64x2_t m2 = vreinterpretq_u64_u8(bswap_vec(vld1q_u8(input + 32)));
+ const uint64x2_t m3 = vreinterpretq_u64_u8(bswap_vec(vld1q_u8(input + 48)));
+
+ a = veorq_u64(a, m0);
+ a = gcm_multiply_x4(H, H2, H3, H4, m3, m2, m1, a);
+
+ input += 64;
+ blocks -= 4;
+ }
+ }
for(size_t i = 0; i != blocks; ++i)
{
- const uint64x2_t m64 = vreinterpretq_u64_u8(vcombine_u8(vrev64_u8(vld1_u8(input+8)), vrev64_u8(vld1_u8(input))));
- input += 16;
-
- a64 = veorq_u64(a64, m64);
-
- uint64x2_t T0, T1, T2, T3, T4, T5;
-
- T0 = (uint64x2_t)vmull_p64(vgetq_lane_u64(a64, 0), vgetq_lane_u64(b64, 0));
- T1 = (uint64x2_t)vmull_p64(vgetq_lane_u64(a64, 1), vgetq_lane_u64(b64, 0));
- T2 = (uint64x2_t)vmull_p64(vgetq_lane_u64(a64, 0), vgetq_lane_u64(b64, 1));
- T3 = (uint64x2_t)vmull_p64(vgetq_lane_u64(a64, 1), vgetq_lane_u64(b64, 1));
-
- T1 = veorq_u64(T1, T2);
- T2 = vreinterpretq_u64_u8(vextq_u8(vdupq_n_u8(0), vreinterpretq_u8_u64(T1), 8));
- T1 = vreinterpretq_u64_u8(vextq_u8(vreinterpretq_u8_u64(T1), vdupq_n_u8(0), 8));
- T0 = veorq_u64(T0, T2);
- T3 = veorq_u64(T3, T1);
-
- T4 = vshrq_n_u64(T0, 31);
- T0 = vshlq_n_u64(T0, 1);
-
- T5 = vshrq_n_u64(T3, 31);
- T3 = vshlq_n_u64(T3, 1);
-
- T2 = vreinterpretq_u64_u8(vextq_u8(vreinterpretq_u8_u64(T4), vdupq_n_u8(0), 12));
- T5 = vreinterpretq_u64_u8(vextq_u8(vdupq_n_u8(0), vreinterpretq_u8_u64(T5), 12));
- T4 = vreinterpretq_u64_u8(vextq_u8(vdupq_n_u8(0), vreinterpretq_u8_u64(T4), 12));
- T0 = vorrq_u64(T0, T4);
- T3 = vorrq_u64(T3, T5);
- T3 = vorrq_u64(T3, T2);
-
- T4 = vreinterpretq_u64_u32(vshlq_n_u32(vreinterpretq_u32_u64(T0), 31));
- T5 = vreinterpretq_u64_u32(vshlq_n_u32(vreinterpretq_u32_u64(T0), 30));
- T2 = vreinterpretq_u64_u32(vshlq_n_u32(vreinterpretq_u32_u64(T0), 25));
-
- T4 = veorq_u64(T4, T5);
- T4 = veorq_u64(T4, T2);
- T5 = vreinterpretq_u64_u8(vextq_u8(vreinterpretq_u8_u64(T4), vdupq_n_u8(0), 4));
- T3 = veorq_u64(T3, T5);
- T4 = vreinterpretq_u64_u8(vextq_u8(vdupq_n_u8(0), vreinterpretq_u8_u64(T4), 4));
- T0 = veorq_u64(T0, T4);
- T3 = veorq_u64(T3, T0);
-
- T4 = vreinterpretq_u64_u32(vshrq_n_u32(vreinterpretq_u32_u64(T0), 1));
- T1 = vreinterpretq_u64_u32(vshrq_n_u32(vreinterpretq_u32_u64(T0), 2));
- T2 = vreinterpretq_u64_u32(vshrq_n_u32(vreinterpretq_u32_u64(T0), 7));
- T3 = veorq_u64(T3, T1);
- T3 = veorq_u64(T3, T2);
- T3 = veorq_u64(T3, T4);
-
- a64 = T3;
+ const uint64x2_t m = vreinterpretq_u64_u8(bswap_vec(vld1q_u8(input + 16*i)));
+ a = veorq_u64(a, m);
+ a = gcm_multiply(H, a);
}
- vst1_u8(x+0, vrev64_u8(vreinterpret_u8_u64(vget_high_u64(a64))));
- vst1_u8(x+8, vrev64_u8(vreinterpret_u8_u64(vget_low_u64(a64))));
+ vst1q_u8(x, bswap_vec(vreinterpretq_u8_u64(a)));
}
}
diff --git a/src/lib/modes/aead/gcm/pmull/pmull.h b/src/lib/modes/aead/gcm/pmull/pmull.h
index 638b845cd..17e61097f 100644
--- a/src/lib/modes/aead/gcm/pmull/pmull.h
+++ b/src/lib/modes/aead/gcm/pmull/pmull.h
@@ -12,7 +12,10 @@
namespace Botan {
-void gcm_multiply_pmull(uint8_t x[16], const uint8_t H[16],
+void gcm_pmull_precompute(const uint8_t H[16], uint64_t H_pow[4*2]);
+
+void gcm_multiply_pmull(uint8_t x[16],
+ const uint64_t H[8],
const uint8_t input[], size_t blocks);
}
diff --git a/src/lib/stream/ctr/ctr.cpp b/src/lib/stream/ctr/ctr.cpp
index e81373a82..b4c931980 100644
--- a/src/lib/stream/ctr/ctr.cpp
+++ b/src/lib/stream/ctr/ctr.cpp
@@ -6,27 +6,30 @@
*/
#include <botan/ctr.h>
+#include <botan/loadstor.h>
namespace Botan {
CTR_BE::CTR_BE(BlockCipher* ciph) :
m_cipher(ciph),
+ m_block_size(m_cipher->block_size()),
+ m_ctr_size(m_block_size),
+ m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),
m_counter(m_cipher->parallel_bytes()),
m_pad(m_counter.size()),
m_iv(m_cipher->block_size()),
- m_block_size(m_cipher->block_size()),
- m_ctr_size(m_block_size),
m_pad_pos(0)
{
}
CTR_BE::CTR_BE(BlockCipher* cipher, size_t ctr_size) :
m_cipher(cipher),
+ m_block_size(m_cipher->block_size()),
+ m_ctr_size(ctr_size),
+ m_ctr_blocks(m_cipher->parallel_bytes() / m_block_size),
m_counter(m_cipher->parallel_bytes()),
m_pad(m_counter.size()),
m_iv(m_cipher->block_size()),
- m_block_size(m_cipher->block_size()),
- m_ctr_size(ctr_size),
m_pad_pos(0)
{
if(m_ctr_size == 0 || m_ctr_size > m_block_size)
@@ -57,15 +60,39 @@ std::string CTR_BE::name() const
void CTR_BE::cipher(const uint8_t in[], uint8_t out[], size_t length)
{
- while(length >= m_pad.size() - m_pad_pos)
+ const uint8_t* pad_bits = &m_pad[0];
+ const size_t pad_size = m_pad.size();
+
+ if(m_pad_pos > 0)
{
- xor_buf(out, in, &m_pad[m_pad_pos], m_pad.size() - m_pad_pos);
- length -= (m_pad.size() - m_pad_pos);
- in += (m_pad.size() - m_pad_pos);
- out += (m_pad.size() - m_pad_pos);
- increment_counter();
+ const size_t avail = pad_size - m_pad_pos;
+ const size_t take = std::min(length, avail);
+ xor_buf(out, in, pad_bits + m_pad_pos, take);
+ length -= take;
+ in += take;
+ out += take;
+ m_pad_pos += take;
+
+ if(take == avail)
+ {
+ add_counter(m_ctr_blocks);
+ m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
+ m_pad_pos = 0;
+ }
+ }
+
+ while(length >= pad_size)
+ {
+ xor_buf(out, in, pad_bits, pad_size);
+ length -= pad_size;
+ in += pad_size;
+ out += pad_size;
+
+ add_counter(m_ctr_blocks);
+ m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
}
- xor_buf(out, in, &m_pad[m_pad_pos], length);
+
+ xor_buf(out, in, pad_bits, length);
m_pad_pos += length;
}
@@ -80,63 +107,89 @@ void CTR_BE::set_iv(const uint8_t iv[], size_t iv_len)
seek(0);
}
-/*
-* Increment the counter and update the buffer
-*/
-void CTR_BE::increment_counter()
- {
- const size_t n_wide = m_counter.size() / m_block_size;
-
- add_counter(n_wide);
-
- m_cipher->encrypt_n(m_counter.data(), m_pad.data(), n_wide);
- m_pad_pos = 0;
- }
-
void CTR_BE::add_counter(const uint64_t counter)
{
- const size_t n_wide = m_counter.size() / m_block_size;
+ const size_t ctr_size = m_ctr_size;
+ const size_t ctr_blocks = m_ctr_blocks;
+ const size_t BS = m_block_size;
- for(size_t i = 0; i != n_wide; ++i)
+ if(ctr_size == 4)
+ {
+ size_t off = (BS - 4);
+ for(size_t i = 0; i != ctr_blocks; ++i)
+ {
+ uint32_t low32 = load_be<uint32_t>(&m_counter[off], 0);
+ low32 += counter;
+ store_be(low32, &m_counter[off]);
+ off += BS;
+ }
+ }
+ else if(ctr_size == 8)
{
- uint64_t local_counter = counter;
- uint16_t carry = static_cast<uint8_t>(local_counter);
- for(size_t j = 0; (carry || local_counter) && j != m_ctr_size; ++j)
+ size_t off = (BS - 8);
+ for(size_t i = 0; i != ctr_blocks; ++i)
{
- const size_t off = i*m_block_size + (m_block_size-1-j);
- const uint16_t cnt = static_cast<uint16_t>(m_counter[off]) + carry;
- m_counter[off] = static_cast<uint8_t>(cnt);
- local_counter = (local_counter >> 8);
- carry = (cnt >> 8) + static_cast<uint8_t>(local_counter);
+ uint64_t low64 = load_be<uint64_t>(&m_counter[off], 0);
+ low64 += counter;
+ store_be(low64, &m_counter[off]);
+ off += BS;
+ }
+ }
+ else if(ctr_size == 16)
+ {
+ size_t off = (BS - 16);
+ for(size_t i = 0; i != ctr_blocks; ++i)
+ {
+ uint64_t b0 = load_be<uint64_t>(&m_counter[off], 0);
+ uint64_t b1 = load_be<uint64_t>(&m_counter[off], 1);
+ b1 += counter;
+ b1 += (b1 < counter); // carry
+ store_be(b0, &m_counter[off]);
+ store_be(b1, &m_counter[off+8]);
+ off += BS;
+ }
+ }
+ else
+ {
+ for(size_t i = 0; i != ctr_blocks; ++i)
+ {
+ uint64_t local_counter = counter;
+ uint16_t carry = static_cast<uint8_t>(local_counter);
+ for(size_t j = 0; (carry || local_counter) && j != ctr_size; ++j)
+ {
+ const size_t off = i*BS + (BS-1-j);
+ const uint16_t cnt = static_cast<uint16_t>(m_counter[off]) + carry;
+ m_counter[off] = static_cast<uint8_t>(cnt);
+ local_counter = (local_counter >> 8);
+ carry = (cnt >> 8) + static_cast<uint8_t>(local_counter);
+ }
}
}
}
void CTR_BE::seek(uint64_t offset)
{
- const size_t n_wide = m_counter.size() / m_block_size;
- const uint64_t base_counter = n_wide * (offset / m_counter.size());
+ const uint64_t base_counter = m_ctr_blocks * (offset / m_counter.size());
zeroise(m_counter);
buffer_insert(m_counter, 0, m_iv);
+ const size_t BS = m_block_size;
+
// Set m_counter blocks to IV, IV + 1, ... IV + n
- for(size_t i = 1; i != n_wide; ++i)
+ for(size_t i = 1; i != m_ctr_blocks; ++i)
{
- buffer_insert(m_counter,
- i*m_block_size,
- &m_counter[(i-1)*m_block_size],
- m_block_size);
+ buffer_insert(m_counter, i*BS, &m_counter[(i-1)*BS], BS);
for(size_t j = 0; j != m_ctr_size; ++j)
- if(++m_counter[i*m_block_size + (m_block_size - 1 - j)])
+ if(++m_counter[i*BS + (BS - 1 - j)])
break;
}
- if (base_counter > 0)
+ if(base_counter > 0)
add_counter(base_counter);
- m_cipher->encrypt_n(m_counter.data(), m_pad.data(), n_wide);
+ m_cipher->encrypt_n(m_counter.data(), m_pad.data(), m_ctr_blocks);
m_pad_pos = offset % m_counter.size();
}
}
diff --git a/src/lib/stream/ctr/ctr.h b/src/lib/stream/ctr/ctr.h
index e174848b8..3ff63b8e5 100644
--- a/src/lib/stream/ctr/ctr.h
+++ b/src/lib/stream/ctr/ctr.h
@@ -48,14 +48,16 @@ class BOTAN_PUBLIC_API(2,0) CTR_BE final : public StreamCipher
void seek(uint64_t offset) override;
private:
void key_schedule(const uint8_t key[], size_t key_len) override;
- void increment_counter();
void add_counter(const uint64_t counter);
std::unique_ptr<BlockCipher> m_cipher;
+
+ const size_t m_block_size;
+ const size_t m_ctr_size;
+ const size_t m_ctr_blocks;
+
secure_vector<uint8_t> m_counter, m_pad;
std::vector<uint8_t> m_iv;
- const size_t m_block_size;
- size_t m_ctr_size;
size_t m_pad_pos;
};
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-29 06:18:05 +0000