Clean up PK decryption encoding.

Previously RSA and ElGamal stripped off leading zeros which were then
assumed by the padding decoders. Instead have them produce ciphertexts
with leading zeros. Changes EME_Raw to strip leading zeros to match
existing behavior.

1 files changed, 11 insertions, 3 deletions

diff --git a/src/lib/prov/openssl/openssl_rsa.cpp b/src/lib/prov/openssl/openssl_rsa.cpp
index e3c0c0fec..ed8f2b0fd 100644
--- a/src/lib/prov/openssl/openssl_rsa.cpp
+++ b/src/lib/prov/openssl/openssl_rsa.cpp
@@ -140,13 +140,21 @@ class OpenSSL_RSA_Decryption_Operation : public PK_Ops::Decryption
 
       size_t max_input_bits() const override { return ::BN_num_bits(m_openssl_rsa->n) - 1; }
 
-      secure_vector<byte> decrypt(const byte msg[], size_t msg_len) override
+      secure_vector<byte> decrypt(byte& valid_mask,
+                                  const byte msg[], size_t msg_len) override
          {
          secure_vector<byte> buf(::RSA_size(m_openssl_rsa.get()));
          int rc = ::RSA_private_decrypt(msg_len, msg, buf.data(), m_openssl_rsa.get(), m_padding);
          if(rc < 0 || static_cast<size_t>(rc) > buf.size())
-            throw OpenSSL_Error("RSA_private_decrypt");
-         buf.resize(rc);
+            {
+            valid_mask = 0;
+            buf.resize(0);
+            }
+         else
+            {
+            valid_mask = 0xFF;
+            buf.resize(rc);
+            }
 
          if(m_padding == RSA_NO_PADDING)
             {