From b403ea921b9ebb35bea8bc19eeae4fa75b71bd22 Mon Sep 17 00:00:00 2001 From: Jack Lloyd Date: Sat, 29 Aug 2015 11:55:36 -0400 Subject: Fix OpenSSL RSA to better handle no padding mode. In raw mode pad out plaintext inputs with zeros as needed as otherwise OpenSSL rejects the input as too small. And when decrypting, strip leading zeros to match the behavior of the base implementation. --- src/lib/vendor/openssl/openssl.h | 3 ++ src/lib/vendor/openssl/openssl_ecdsa.cpp | 5 ++-- src/lib/vendor/openssl/openssl_rsa.cpp | 47 ++++++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 10 deletions(-) (limited to 'src/lib') diff --git a/src/lib/vendor/openssl/openssl.h b/src/lib/vendor/openssl/openssl.h index 4fa37cc54..86cc8fd35 100644 --- a/src/lib/vendor/openssl/openssl.h +++ b/src/lib/vendor/openssl/openssl.h @@ -27,6 +27,9 @@ class OpenSSL_Error : public Exception #define BOTAN_OPENSSL_HASH_PRIO 150 #define BOTAN_OPENSSL_RC4_PRIO 150 +#define BOTAN_OPENSSL_RSA_PRIO 90 +#define BOTAN_OPENSSL_ECDSA_PRIO 90 + } diff --git a/src/lib/vendor/openssl/openssl_ecdsa.cpp b/src/lib/vendor/openssl/openssl_ecdsa.cpp index 9031944d4..0651cc280 100644 --- a/src/lib/vendor/openssl/openssl_ecdsa.cpp +++ b/src/lib/vendor/openssl/openssl_ecdsa.cpp @@ -194,10 +194,11 @@ class OpenSSL_ECDSA_Signing_Operation : public PK_Ops::Signature_with_EMSA BOTAN_REGISTER_TYPE(PK_Ops::Verification, OpenSSL_ECDSA_Verification_Operation, "ECDSA", OpenSSL_ECDSA_Verification_Operation::make, - "openssl", 255); + "openssl", BOTAN_OPENSSL_ECDSA_PRIO); BOTAN_REGISTER_TYPE(PK_Ops::Signature, OpenSSL_ECDSA_Signing_Operation, "ECDSA", - OpenSSL_ECDSA_Signing_Operation::make, "openssl", 255); + OpenSSL_ECDSA_Signing_Operation::make, + "openssl", BOTAN_OPENSSL_ECDSA_PRIO); } diff --git a/src/lib/vendor/openssl/openssl_rsa.cpp b/src/lib/vendor/openssl/openssl_rsa.cpp index ad2e47683..f8ab2bcd1 100644 --- a/src/lib/vendor/openssl/openssl_rsa.cpp +++ b/src/lib/vendor/openssl/openssl_rsa.cpp @@ -59,11 +59,11 @@ class OpenSSL_RSA_Encryption_Operation : public PK_Ops::Encryption { const std::vector der = rsa.x509_subject_public_key(); const byte* der_ptr = der.data(); - m_openssl_rsa.reset(d2i_RSAPublicKey(nullptr, &der_ptr, der.size())); + m_openssl_rsa.reset(::d2i_RSAPublicKey(nullptr, &der_ptr, der.size())); if(!m_openssl_rsa) throw OpenSSL_Error("d2i_RSAPublicKey"); - m_bits = 8 * (RSA_size(m_openssl_rsa.get()) - pad_overhead); + m_bits = 8 * n_size() - pad_overhead; } size_t max_input_bits() const override { return m_bits; }; @@ -71,15 +71,35 @@ class OpenSSL_RSA_Encryption_Operation : public PK_Ops::Encryption secure_vector encrypt(const byte msg[], size_t msg_len, RandomNumberGenerator&) override { + const size_t mod_sz = n_size(); - secure_vector buf(::RSA_size(m_openssl_rsa.get())); - int rc = ::RSA_public_encrypt(msg_len, msg, buf.data(), m_openssl_rsa.get(), m_padding); + if(msg_len > mod_sz) + throw Invalid_Argument("Input too large for RSA key"); + + secure_vector outbuf(mod_sz); + + secure_vector inbuf; + + if(m_padding == RSA_NO_PADDING) + { + inbuf.resize(mod_sz); + copy_mem(&inbuf[mod_sz - msg_len], msg, msg_len); + } + else + { + inbuf.assign(msg, msg + msg_len); + } + + int rc = ::RSA_public_encrypt(inbuf.size(), inbuf.data(), outbuf.data(), + m_openssl_rsa.get(), m_padding); if(rc < 0) throw OpenSSL_Error("RSA_public_encrypt"); - return buf; + + return outbuf; } private: + size_t n_size() const { return ::RSA_size(m_openssl_rsa.get()); } std::unique_ptr> m_openssl_rsa; size_t m_bits = 0; int m_padding = 0; @@ -114,7 +134,7 @@ class OpenSSL_RSA_Decryption_Operation : public PK_Ops::Decryption if(!m_openssl_rsa) throw OpenSSL_Error("d2i_RSAPrivateKey"); - m_bits = 8 * RSA_size(m_openssl_rsa.get()); + m_bits = 8 * ::RSA_size(m_openssl_rsa.get()); } size_t max_input_bits() const override { return m_bits; }; @@ -126,6 +146,17 @@ class OpenSSL_RSA_Decryption_Operation : public PK_Ops::Decryption if(rc < 0 || static_cast(rc) > buf.size()) throw OpenSSL_Error("RSA_private_decrypt"); buf.resize(rc); + + if(m_padding == RSA_NO_PADDING) + { + size_t leading_0s = 0; + while(leading_0s < buf.size() && buf[leading_0s] == 0) + leading_0s++; + + if(leading_0s) + return secure_vector(&buf[leading_0s], &buf[buf.size()]); + } + return buf; } @@ -136,9 +167,9 @@ class OpenSSL_RSA_Decryption_Operation : public PK_Ops::Decryption }; BOTAN_REGISTER_TYPE(PK_Ops::Encryption, OpenSSL_RSA_Encryption_Operation, "RSA", - OpenSSL_RSA_Encryption_Operation::make, "openssl", 255); + OpenSSL_RSA_Encryption_Operation::make, "openssl", BOTAN_OPENSSL_RSA_PRIO); BOTAN_REGISTER_TYPE(PK_Ops::Decryption, OpenSSL_RSA_Decryption_Operation, "RSA", - OpenSSL_RSA_Decryption_Operation::make, "openssl", 255); + OpenSSL_RSA_Decryption_Operation::make, "openssl", BOTAN_OPENSSL_RSA_PRIO); } -- cgit v1.2.3