Avoid allocations during GCM message processing

On Skylake with 1024 byte buffer brings perf from 2.69 cpb to 2.2 cpb.
And over 50% improvement for small messages.

4 files changed, 41 insertions, 23 deletions

diff --git a/src/lib/modes/aead/gcm/gcm.cpp b/src/lib/modes/aead/gcm/gcm.cpp
index d299af35f..5dd6408d8 100644
--- a/src/lib/modes/aead/gcm/gcm.cpp
+++ b/src/lib/modes/aead/gcm/gcm.cpp
@@ -95,24 +95,25 @@ void GCM_Mode::start_msg(const uint8_t nonce[], size_t nonce_len)
    if(!valid_nonce_length(nonce_len))
       throw Invalid_IV_Length(name(), nonce_len);
 
-   secure_vector<uint8_t> y0(GCM_BS);
+   m_y0.resize(GCM_BS);
 
    if(nonce_len == 12)
       {
-      copy_mem(y0.data(), nonce, nonce_len);
-      y0[15] = 1;
+      copy_mem(m_y0.data(), nonce, nonce_len);
+      m_y0[15] = 1;
       }
    else
       {
-      y0 = m_ghash->nonce_hash(nonce, nonce_len);
+      m_ghash->nonce_hash(m_y0, nonce, nonce_len);
       }
 
-   m_ctr->set_iv(y0.data(), y0.size());
+   m_ctr->set_iv(m_y0.data(), m_y0.size());
 
-   zeroise(y0);
-   m_ctr->encipher(y0);
+   zeroise(m_y0);
+   m_ctr->encipher(m_y0);
 
-   m_ghash->start(y0.data(), y0.size());
+   m_ghash->start(m_y0.data(), m_y0.size());
+   m_y0.clear();
    }
 
 size_t GCM_Encryption::process(uint8_t buf[], size_t sz)
@@ -131,8 +132,10 @@ void GCM_Encryption::finish(secure_vector<uint8_t>& buffer, size_t offset)
 
    m_ctr->cipher(buf, buf, sz);
    m_ghash->update(buf, sz);
-   auto mac = m_ghash->final();
-   buffer += std::make_pair(mac.data(), tag_size());
+
+   uint8_t mac[16] = { 0 };
+   m_ghash->final(mac, tag_size());
+   buffer += std::make_pair(mac, tag_size());
    }
 
 size_t GCM_Decryption::process(uint8_t buf[], size_t sz)
@@ -161,11 +164,12 @@ void GCM_Decryption::finish(secure_vector<uint8_t>& buffer, size_t offset)
       m_ctr->cipher(buf, buf, remaining);
       }
 
-   auto mac = m_ghash->final();
+   uint8_t mac[16] = { 0 };
+   m_ghash->final(mac, tag_size());
 
    const uint8_t* included_tag = &buffer[remaining+offset];
 
-   if(!constant_time_compare(mac.data(), included_tag, tag_size()))
+   if(!constant_time_compare(mac, included_tag, tag_size()))
       throw Invalid_Authentication_Tag("GCM tag check failed");
 
    buffer.resize(offset + remaining);
diff --git a/src/lib/modes/aead/gcm/gcm.h b/src/lib/modes/aead/gcm/gcm.h
index b46f47fa0..8f27c3679 100644
--- a/src/lib/modes/aead/gcm/gcm.h
+++ b/src/lib/modes/aead/gcm/gcm.h
@@ -57,6 +57,8 @@ class BOTAN_PUBLIC_API(2,0) GCM_Mode : public AEAD_Mode
       void start_msg(const uint8_t nonce[], size_t nonce_len) override;
 
       void key_schedule(const uint8_t key[], size_t length) override;
+
+      secure_vector<uint8_t> m_y0;
    };
 
 /**
diff --git a/src/lib/modes/aead/gcm/ghash.cpp b/src/lib/modes/aead/gcm/ghash.cpp
index 29bf996f2..64a9b7221 100644
--- a/src/lib/modes/aead/gcm/ghash.cpp
+++ b/src/lib/modes/aead/gcm/ghash.cpp
@@ -230,27 +230,24 @@ void GHASH::add_final_block(secure_vector<uint8_t>& hash,
    ghash_update(hash, final_block, GCM_BS);
    }
 
-secure_vector<uint8_t> GHASH::final()
+void GHASH::final(uint8_t mac[], size_t mac_len)
    {
+   BOTAN_ARG_CHECK(mac_len > 0 && mac_len <= 16, "GHASH output length");
    add_final_block(m_ghash, m_ad_len, m_text_len);
 
-   secure_vector<uint8_t> mac;
-   mac.swap(m_ghash);
+   for(size_t i = 0; i != mac_len; ++i)
+      mac[i] = m_ghash[i] ^ m_nonce[i];
 
-   mac ^= m_nonce;
+   m_ghash.clear();
    m_text_len = 0;
-   return mac;
    }
 
-secure_vector<uint8_t> GHASH::nonce_hash(const uint8_t nonce[], size_t nonce_len)
+void GHASH::nonce_hash(secure_vector<uint8_t>& y0, const uint8_t nonce[], size_t nonce_len)
    {
    BOTAN_ASSERT(m_ghash.size() == 0, "nonce_hash called during wrong time");
-   secure_vector<uint8_t> y0(GCM_BS);
 
    ghash_update(y0, nonce, nonce_len);
    add_final_block(y0, 0, nonce_len);
-
-   return y0;
    }
 
 void GHASH::clear()
diff --git a/src/lib/modes/aead/gcm/ghash.h b/src/lib/modes/aead/gcm/ghash.h
index 7fcf7cfaa..fb7565134 100644
--- a/src/lib/modes/aead/gcm/ghash.h
+++ b/src/lib/modes/aead/gcm/ghash.h
@@ -22,7 +22,15 @@ class BOTAN_PUBLIC_API(2,0) GHASH final : public SymmetricAlgorithm
    public:
       void set_associated_data(const uint8_t ad[], size_t ad_len);
 
-      secure_vector<uint8_t> nonce_hash(const uint8_t nonce[], size_t len);
+      secure_vector<uint8_t> BOTAN_DEPRECATED("Use other impl")
+         nonce_hash(const uint8_t nonce[], size_t nonce_len)
+         {
+         secure_vector<uint8_t> y0(GCM_BS);
+         nonce_hash(y0, nonce, nonce_len);
+         return y0;
+         }
+
+      void nonce_hash(secure_vector<uint8_t>& y0, const uint8_t nonce[], size_t len);
 
       void start(const uint8_t nonce[], size_t len);
 
@@ -36,7 +44,14 @@ class BOTAN_PUBLIC_API(2,0) GHASH final : public SymmetricAlgorithm
       */
       void update_associated_data(const uint8_t ad[], size_t len);
 
-      secure_vector<uint8_t> final();
+      secure_vector<uint8_t> BOTAN_DEPRECATED("Use version taking output params") final()
+         {
+         secure_vector<uint8_t> mac(GCM_BS);
+         final(mac.data(), mac.size());
+         return mac;
+         }
+
+      void final(uint8_t out[], size_t out_len);
 
       Key_Length_Specification key_spec() const override
          { return Key_Length_Specification(16); }