Resolve a leak in OpenSSL ECDSA verification for old OpenSSL

The code was using the 1.0 API incorrectly and causing a leak.

https://github.com/riboseinc/rnp/issues/757

2 files changed, 24 insertions, 6 deletions

diff --git a/src/lib/prov/openssl/openssl_ec.cpp b/src/lib/prov/openssl/openssl_ec.cpp
index e8df0598c..5018bb027 100644
--- a/src/lib/prov/openssl/openssl_ec.cpp
+++ b/src/lib/prov/openssl/openssl_ec.cpp
@@ -171,15 +171,15 @@ class OpenSSL_ECDSA_Verification_Operation final : public PK_Ops::Verification_w
          std::unique_ptr<ECDSA_SIG, std::function<void (ECDSA_SIG*)>> sig(nullptr, ECDSA_SIG_free);
          sig.reset(::ECDSA_SIG_new());
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+         sig->r = BN_bin2bn(sig_bytes              , sig_len / 2, sig->r);
+         sig->s = BN_bin2bn(sig_bytes + sig_len / 2, sig_len / 2, sig->s);
+#else
          BIGNUM* r = BN_bin2bn(sig_bytes              , sig_len / 2, nullptr);
          BIGNUM* s = BN_bin2bn(sig_bytes + sig_len / 2, sig_len / 2, nullptr);
          if(r == nullptr || s == nullptr)
             throw OpenSSL_Error("BN_bin2bn sig s");
 
-#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
-         sig->r = r;
-         sig->s = s;
-#else
          ECDSA_SIG_set0(sig.get(), r, s);
 #endif
 
@@ -278,7 +278,15 @@ make_openssl_ecdsa_ver_op(const ECDSA_PublicKey& key, const std::string& params)
       {
       throw Lookup_Error("OpenSSL ECDSA does not support this curve");
       }
-   return std::unique_ptr<PK_Ops::Verification>(new OpenSSL_ECDSA_Verification_Operation(key, params, nid));
+
+   try
+      {
+      return std::unique_ptr<PK_Ops::Verification>(new OpenSSL_ECDSA_Verification_Operation(key, params, nid));
+      }
+   catch(OpenSSL_Error&)
+      {
+      throw Lookup_Error("OpenSSL ECDSA does not support this key");
+      }
    }
 
 std::unique_ptr<PK_Ops::Signature>
diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index ef1a16ba7..6777f85f4 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -97,7 +97,17 @@ int main(int argc, char* argv[])
 
       Botan_Tests::Test_Runner tests(std::cout);
 
-      return tests.run(opts);
+      int rc = tests.run(opts);
+
+#if defined(BOTAN_HAS_OPENSSL)
+      if(opts.provider().empty() || opts.provider() == "openssl")
+         {
+         ::ERR_free_strings();
+         ::ERR_remove_thread_state(nullptr);
+         }
+#endif
+
+      return rc;
       }
    catch(std::exception& e)
       {