Merge GH #2042 Optimize DTLS MTU splitting

2 files changed, 30 insertions, 25 deletions

diff --git a/src/bogo_shim/config.json b/src/bogo_shim/config.json
index afbdd9822..905a7e8cd 100644
--- a/src/bogo_shim/config.json
+++ b/src/bogo_shim/config.json
@@ -94,7 +94,6 @@
         "ClientAuth-Verify-ECDSA-SHA1-TLS12": "BoringSSL will sign SHA-1 and SHA-512 with ECDSA but not accept them.",
 
         "AppDataAfterChangeCipherSpec-DTLS*": "BoringSSL DTLS drops out of order AppData, we reject",
-        "MTUExceeded": "BoringSSL splits DTLS handshakes differently",
 
         "Resume-Client-NoResume-TLS1-TLS11": "BoGo expects resumption attempt sends latest version",
         "Resume-Client-NoResume-TLS1-TLS12": "BoGo expects resumption attempt sends latest version",
@@ -106,19 +105,11 @@
         "Resume-Client-Mismatch-TLS11-TLS12": "BoGo expects resumption attempt sends latest version",
         "Resume-Client-Mismatch-TLS1-TLS12-DTLS": "BoGo expects resumption attempt sends latest version",
 
-        "CurveTest-Client-Compressed*": "Point compression is supported, which BoGo doesn't expect",
-        "PointFormat-Client-MissingUncompressed": "Point compression is supported, which BoGo doesn't expect",
-        "CurveTest-Server-Compressed*": "Point compression is supported, which BoGo doesn't expect",
-        "PointFormat-Server-MissingUncompressed": "Point compression is supported, which BoGo doesn't expect",
-
-        "RSAPSSSupport-ConfigNoPSS-NoCerts-TLS12-Client": "Not possible to disable PSS",
-        "RSAPSSSupport-ConfigNoPSS-TLS12-Client": "Not possible to disable PSS",
-        "RSAPSSSupport-ConfigPSS-NoCerts-TLS12-Client": "Not possible to disable PSS",
-        "RSAPSSSupport-Default-NoCerts-TLS12-Client": "Not possible to disable PSS",
-        "RSAPSSSupport-ConfigNoPSS-NoCerts-TLS12-Server": "Not possible to disable PSS",
-        "RSAPSSSupport-ConfigNoPSS-TLS12-Server": "Not possible to disable PSS",
-        "RSAPSSSupport-ConfigPSS-NoCerts-TLS12-Server": "Not possible to disable PSS",
-        "RSAPSSSupport-Default-NoCerts-TLS12-Server": "Not possible to disable PSS",
+        "CurveTest-*-Compressed*": "Point compression is supported, which BoGo doesn't expect",
+        "PointFormat-*-MissingUncompressed": "Point compression is supported, which BoGo doesn't expect",
+
+        "RSAPSSSupport-ConfigPSS-NoCerts-TLS12-*": "Needs investigation",
+        "RSAPSSSupport-Default-NoCerts-TLS12-*": "Needs investigation",
 
         "DTLS-Retransmit*": "Shim needs timeout support",
 
diff --git a/src/lib/tls/tls_handshake_io.cpp b/src/lib/tls/tls_handshake_io.cpp
index 3f3e672de..6e2bf0284 100644
--- a/src/lib/tls/tls_handshake_io.cpp
+++ b/src/lib/tls/tls_handshake_io.cpp
@@ -426,6 +426,8 @@ std::vector<uint8_t> Datagram_Handshake_IO::send_message(uint16_t msg_seq,
                                                       Handshake_Type msg_type,
                                                       const std::vector<uint8_t>& msg_bits)
    {
+   const size_t DTLS_HANDSHAKE_HEADER_LEN = 12;
+
    const std::vector<uint8_t> no_fragment =
       format_w_seq(msg_bits, msg_type, msg_seq);
 
@@ -437,22 +439,34 @@ std::vector<uint8_t> Datagram_Handshake_IO::send_message(uint16_t msg_seq,
       {
       size_t frag_offset = 0;
 
-      const size_t DTLS_HANDSHAKE_HEADERS = 32;
-      const size_t ciphersuite_overhead = (epoch > 0) ? 32 : 0;
-      const size_t max_rec_size = m_mtu - DTLS_HANDSHAKE_HEADERS - ciphersuite_overhead;
+      /**
+      * Largest possible overhead is for SHA-384 CBC ciphers, with 16 byte IV,
+      * 16+ for padding and 48 bytes for MAC. 128 is probably a strict
+      * over-estimate here. When CBC ciphers are removed this can be reduced
+      * since AEAD modes have no padding, at most 16 byte mac, and smaller
+      * per-record nonce.
+      */
+      const size_t ciphersuite_overhead = (epoch > 0) ? 128 : 0;
+      const size_t header_overhead = DTLS_HEADER_SIZE + DTLS_HANDSHAKE_HEADER_LEN;
+
+      if(m_mtu <= (header_overhead + ciphersuite_overhead))
+         throw Invalid_Argument("DTLS MTU is too small to send headers");
+
+      const size_t max_rec_size = m_mtu - (header_overhead + ciphersuite_overhead);
 
       while(frag_offset != msg_bits.size())
          {
          const size_t frag_len = std::min<size_t>(msg_bits.size() - frag_offset, max_rec_size);
 
-         m_send_hs(epoch,
-                   HANDSHAKE,
-                   format_fragment(&msg_bits[frag_offset],
-                                   frag_len,
-                                   static_cast<uint16_t>(frag_offset),
-                                   static_cast<uint16_t>(msg_bits.size()),
-                                   msg_type,
-                                   msg_seq));
+         const std::vector<uint8_t> frag =
+            format_fragment(&msg_bits[frag_offset],
+                            frag_len,
+                            static_cast<uint16_t>(frag_offset),
+                            static_cast<uint16_t>(msg_bits.size()),
+                            msg_type,
+                            msg_seq);
+
+         m_send_hs(epoch, HANDSHAKE, frag);
 
          frag_offset += frag_len;
          }