Update and consolidate the test framework.

The tests previously had used 4 to 6 different schemes internally (the vec file
reader framework, Catch, the old InSiTo Boost.Test tests, the PK/BigInt tests
which escaped the rewrite in 1.11.7, plus a number of one-offs). Converge on a
design that works everywhere, and update all the things.

Fix also a few bugs found by the test changes: SHA-512-256 name incorrect,
OpenSSL RC4 name incorrect, signature of FFI function botan_pubkey_destroy
was wrong.

1 files changed, 161 insertions, 312 deletions

diff --git a/src/tests/test_pubkey.cpp b/src/tests/test_pubkey.cpp
index 09f3843bb..2a49be840 100644
--- a/src/tests/test_pubkey.cpp
+++ b/src/tests/test_pubkey.cpp
@@ -1,405 +1,254 @@
 /*
-* (C) 2009 Jack Lloyd
+* (C) 2009,2015 Jack Lloyd
 *
 * Botan is released under the Simplified BSD License (see license.txt)
 */
 
-#include "tests.h"
+#include "test_pubkey.h"
 
 #if defined(BOTAN_HAS_PUBLIC_KEY_CRYPTO)
 
 #include "test_rng.h"
-#include "test_pubkey.h"
 
-#include <iostream>
-#include <fstream>
-#include <string>
-#include <vector>
-#include <cstdlib>
-#include <memory>
-
-#include <botan/oids.h>
+#include <botan/pubkey.h>
 #include <botan/x509_key.h>
 #include <botan/pkcs8.h>
-#include <botan/pubkey.h>
+#include <botan/oids.h>
 #include <botan/hex.h>
 
-#if defined(BOTAN_HAS_RSA)
-  #include <botan/rsa.h>
-#endif
-
-#if defined(BOTAN_HAS_DSA)
-  #include <botan/dsa.h>
-#endif
-
-#if defined(BOTAN_HAS_DIFFIE_HELLMAN)
-  #include <botan/dh.h>
-#endif
-
-#if defined(BOTAN_HAS_NYBERG_RUEPPEL)
-  #include <botan/nr.h>
-#endif
-
-#if defined(BOTAN_HAS_RW)
-  #include <botan/rw.h>
-#endif
-
-#if defined(BOTAN_HAS_ELGAMAL)
-  #include <botan/elgamal.h>
-#endif
-
-#if defined(BOTAN_HAS_ECDSA)
-  #include <botan/ecdsa.h>
-#endif
-
-#if defined(BOTAN_HAS_ECDH)
-  #include <botan/ecdh.h>
-#endif
-
-#if defined(BOTAN_HAS_GOST_34_10_2001)
-  #include <botan/gost_3410.h>
-#endif
-
-#if defined(BOTAN_HAS_DLIES)
-  #include <botan/dlies.h>
-  #include <botan/kdf.h>
-#endif
-
-#include <botan/numthry.h>
+namespace Botan_Tests {
 
-using namespace Botan;
-
-namespace {
-
-void dump_data(const std::vector<byte>& out,
-               const std::vector<byte>& expected)
+void check_invalid_signatures(Test::Result& result,
+                              Botan::PK_Verifier& verifier,
+                              const std::vector<uint8_t>& message,
+                              const std::vector<uint8_t>& signature)
    {
-   std::cout << "Got: " << hex_encode(out) << std::endl;
-   std::cout << "Exp: " << hex_encode(expected) << std::endl;
-   }
+   const std::vector<uint8_t> zero_sig(signature.size());
+   result.test_eq("all zero signature invalid", verifier.verify_message(message, zero_sig), false);
 
-size_t validate_save_and_load(const Private_Key* priv_key,
-                              RandomNumberGenerator& rng)
-   {
-   std::string name = priv_key->algo_name();
+   std::vector<uint8_t> bad_sig = signature;
 
-   size_t fails = 0;
-   std::string pub_pem = X509::PEM_encode(*priv_key);
-
-   try
+   for(size_t i = 0; i <= Test::soak_level(); ++i)
       {
-      DataSource_Memory input_pub(pub_pem);
-      std::unique_ptr<Public_Key> restored_pub(X509::load_key(input_pub));
+      while(bad_sig == signature)
+         bad_sig = Test::mutate_vec(bad_sig, true);
 
-      if(!restored_pub.get())
-         {
-         std::cout << "Could not recover " << name << " public key" << std::endl;
-         ++fails;
-         }
-      else if(restored_pub->check_key(rng, true) == false)
+      if(!result.test_eq("incorrect signature invalid", verifier.verify_message(message, bad_sig), false))
          {
-         std::cout << "Restored pubkey failed self tests " << name << std::endl;
-         ++fails;
+         result.test_note("Accepted invalid signature " + Botan::hex_encode(bad_sig));
          }
       }
-   catch(std::exception& e)
-      {
-      std::cout << "Exception during load of " << name
-                << " key: " << e.what() << std::endl;
-      std::cout << "PEM for pubkey was:\n" << pub_pem << std::endl;
-      ++fails;
-      }
+   }
 
-   std::string priv_pem = PKCS8::PEM_encode(*priv_key);
+void check_invalid_ciphertexts(Test::Result& result,
+                               Botan::PK_Decryptor& decryptor,
+                               const std::vector<uint8_t>& plaintext,
+                               const std::vector<uint8_t>& ciphertext)
+   {
+   std::vector<uint8_t> bad_ctext = ciphertext;
 
-   try
+   size_t ciphertext_accepted = 0, ciphertext_rejected = 0;
+
+   for(size_t i = 0; i <= Test::soak_level(); ++i)
       {
-      DataSource_Memory input_priv(priv_pem);
-      std::unique_ptr<Private_Key> restored_priv(
-         PKCS8::load_key(input_priv, rng));
+      while(bad_ctext == ciphertext)
+         bad_ctext = Test::mutate_vec(bad_ctext, true);
 
-      if(!restored_priv.get())
+      try
          {
-         std::cout << "Could not recover " << name << " privlic key" << std::endl;
-         ++fails;
+         const Botan::secure_vector<uint8_t> decrypted = decryptor.decrypt(bad_ctext);
+         ++ciphertext_accepted;
+
+         if(!result.test_ne("incorrect ciphertext different", decrypted, plaintext))
+            {
+            result.test_eq("used corrupted ciphertext", bad_ctext, ciphertext);
+            }
          }
-      else if(restored_priv->check_key(rng, true) == false)
+      catch(std::exception& e)
          {
-         std::cout << "Restored privkey failed self tests " << name << std::endl;
-         ++fails;
+         ++ciphertext_rejected;
          }
       }
-   catch(std::exception& e)
-      {
-      std::cout << "Exception during load of " << name
-                << " key: " << e.what() << std::endl;
-      std::cout << "PEM for privkey was:\n" << priv_pem << std::endl;
-      ++fails;
-      }
 
-   return fails;
+   result.test_note("Accepted " + std::to_string(ciphertext_accepted) +
+                    " invalid ciphertexts, rejected " + std::to_string(ciphertext_rejected));
    }
 
-byte nonzero_byte(RandomNumberGenerator& rng)
+Test::Result
+PK_Signature_Generation_Test::run_one_test(const std::string&, const VarMap& vars)
    {
-   byte b = 0;
-   while(b == 0)
-      b = rng.next_byte();
-   return b;
-   }
+   const std::vector<uint8_t> message   = get_req_bin(vars, "Msg");
+   const std::vector<uint8_t> signature = get_req_bin(vars, "Signature");
+   const std::string padding = get_opt_str(vars, "Padding", default_padding(vars));
 
-}
+   std::unique_ptr<Botan::RandomNumberGenerator> rng;
+   if(vars.count("Nonce"))
+      {
+      rng.reset(new Fixed_Output_RNG(get_req_bin(vars, "Nonce")));
+      }
 
-#define PK_TEST(expr, msg)                                \
-   do {                                                \
-      const bool test_result = expr;                           \
-      if(!test_result)                                           \
-         {                                             \
-         std::cout << "Test " << #expr << " failed: " << msg << std::endl; \
-         ++fails;                                      \
-         }                                             \
-   } while(0)
-
-size_t validate_encryption(PK_Encryptor& e, PK_Decryptor& d,
-                           const std::string& algo, const std::string& input,
-                           const std::string& random, const std::string& exp)
-   {
-   std::vector<byte> message = hex_decode(input);
-   std::vector<byte> expected = hex_decode(exp);
-   Fixed_Output_RNG kat_rng(hex_decode(random));
+   Test::Result result(algo_name() + "/" + padding + " signature generation");
 
-   size_t fails = 0;
+   std::unique_ptr<Botan::Private_Key> privkey = load_private_key(vars);
+   std::unique_ptr<Botan::Public_Key> pubkey(Botan::X509::load_key(Botan::X509::BER_encode(*privkey)));
 
-   const std::vector<byte> ctext = e.encrypt(message, kat_rng);
-   if(ctext != expected)
-      {
-      std::cout << "FAILED (encrypt): " << algo << std::endl;
-      dump_data(ctext, expected);
-      ++fails;
-      }
+   Botan::PK_Signer signer(*privkey, padding);
+   Botan::PK_Verifier verifier(*pubkey, padding);
 
-   std::vector<byte> decrypted = unlock(d.decrypt(ctext));
+   const std::vector<uint8_t> generated_signature = signer.sign_message(message, rng ? *rng : Test::rng());
+   result.test_eq("generated signature matches KAT", generated_signature, signature);
 
-   if(decrypted != message)
-      {
-      std::cout << "FAILED (decrypt): " << algo << std::endl;
-      dump_data(decrypted, message);
-      ++fails;
-      }
+   result.test_eq("generated signature valid", verifier.verify_message(message, generated_signature), true);
+   check_invalid_signatures(result, verifier, message, signature);
+   result.test_eq("correct signature valid", verifier.verify_message(message, signature), true);
 
-   if(algo.find("/Raw") == std::string::npos)
-      {
-      auto& rng = test_rng();
+   return result;
+   }
 
-      for(size_t i = 0; i != ctext.size(); ++i)
-         {
-         std::vector<byte> bad_ctext = ctext;
+Test::Result
+PK_Signature_Verification_Test::run_one_test(const std::string&, const VarMap& vars)
+   {
+   const std::vector<uint8_t> message   = get_req_bin(vars, "Msg");
+   const std::vector<uint8_t> signature = get_req_bin(vars, "Signature");
+   const std::string padding = get_opt_str(vars, "Padding", default_padding(vars));
+   std::unique_ptr<Botan::Public_Key> pubkey = load_public_key(vars);
 
-         bad_ctext[i] ^= nonzero_byte(rng);
+   Test::Result result(algo_name() + "/" + padding + " signature verification");
 
-         BOTAN_ASSERT(bad_ctext != ctext, "Made them different");
+   Botan::PK_Verifier verifier(*pubkey, padding);
 
-         try
-            {
-            auto bad_ptext = unlock(d.decrypt(bad_ctext));
-            std::cout << algo << " failed - decrypted bad data" << std::endl;
-            std::cout << hex_encode(bad_ctext) << " -> " << hex_encode(bad_ptext) << std::endl;
-            std::cout << hex_encode(ctext) << " -> " << hex_encode(decrypted) << std::endl;
-
-            // Ignore PKCS #1 failures as they do occur occasionally (million message attack)
-            const bool is_pkcs1 = algo.find("/EME-PKCS1-v1_5") != std::string::npos;
-
-            if(is_pkcs1)
-               std::cout << "Ignoring PKCS #1 failure" << std::endl;
-            else
-               ++fails;
-            }
-         catch(...) {}
-         }
-      }
+   result.test_eq("correct signature valid", verifier.verify_message(message, signature), true);
 
-   return fails;
-   }
+   check_invalid_signatures(result, verifier, message, signature);
 
-size_t validate_signature(PK_Verifier& v, PK_Signer& s, const std::string& algo,
-                          const std::string& input,
-                          RandomNumberGenerator& rng,
-                          const std::string& exp)
-   {
-   return validate_signature(v, s, algo, input, rng, rng, exp);
+   return result;
    }
 
-size_t validate_signature(PK_Verifier& v, PK_Signer& s, const std::string& algo,
-                          const std::string& input,
-                          RandomNumberGenerator& signer_rng,
-                          RandomNumberGenerator& test_rng,
-                          const std::string& exp)
+Test::Result
+PK_Encryption_Decryption_Test::run_one_test(const std::string&, const VarMap& vars)
    {
-   std::vector<byte> message = hex_decode(input);
-   std::vector<byte> expected = hex_decode(exp);
-   std::vector<byte> sig = s.sign_message(message, signer_rng);
+   const std::vector<uint8_t> plaintext  = get_req_bin(vars, "Msg");
+   const std::vector<uint8_t> ciphertext = get_req_bin(vars, "Ciphertext");
 
-   size_t fails = 0;
+   const std::string padding = get_opt_str(vars, "Padding", default_padding(vars));
 
-   if(sig != expected)
+   std::unique_ptr<Botan::RandomNumberGenerator> kat_rng;
+   if(vars.count("Nonce"))
       {
-      std::cout << "FAILED (sign): " << algo << std::endl;
-      dump_data(sig, expected);
-      ++fails;
+      kat_rng.reset(new Fixed_Output_RNG(get_req_bin(vars, "Nonce")));
       }
 
-   PK_TEST(v.verify_message(message, sig), "Correct signature is valid");
+   Test::Result result(algo_name() + "/" + padding + " decryption");
 
-   for(size_t i = 0; i != 5; ++i)
-      {
-      auto bad_sig = sig;
+   std::unique_ptr<Botan::Private_Key> privkey = load_private_key(vars);
 
-      const size_t idx = (test_rng.next_byte() * 256 + test_rng.next_byte()) % sig.size();
-      bad_sig[idx] ^= nonzero_byte(test_rng);
+   // instead slice the private key to work around elgamal test inputs
+   //std::unique_ptr<Botan::Public_Key> pubkey(Botan::X509::load_key(Botan::X509::BER_encode(*privkey)));
 
-      PK_TEST(!v.verify_message(message, bad_sig), "Incorrect signature is invalid");
-      }
+   Botan::PK_Encryptor_EME encryptor(*privkey, padding);
+   result.test_eq("encryption", encryptor.encrypt(plaintext, kat_rng ? *kat_rng : Test::rng()), ciphertext);
 
-   zero_mem(sig.data(), sig.size());
+   Botan::PK_Decryptor_EME decryptor(*privkey, padding);
+   result.test_eq("decryption", decryptor.decrypt(ciphertext), plaintext);
 
-   PK_TEST(!v.verify_message(message, sig), "All-zero signature is invalid");
+   check_invalid_ciphertexts(result, decryptor, plaintext, ciphertext);
 
-   return fails;
+   return result;
    }
 
-size_t validate_signature(PK_Verifier& v, PK_Signer& s, const std::string& algo,
-                          const std::string& input,
-                          RandomNumberGenerator& rng,
-                          const std::string& random,
-                          const std::string& exp)
+Test::Result PK_Key_Agreement_Test::run_one_test(const std::string&, const VarMap& vars)
    {
-   Fixed_Output_RNG fixed_rng(hex_decode(random));
+   const std::vector<uint8_t> shared = get_req_bin(vars, "K");
+   const std::string kdf = get_opt_str(vars, "KDF", default_kdf(vars));
 
-   return validate_signature(v, s, algo, input, fixed_rng, rng, exp);
-   }
+   Test::Result result(algo_name() + "/" + kdf + " key agreement");
 
-size_t validate_kas(PK_Key_Agreement& kas, const std::string& algo,
-                    const std::vector<byte>& pubkey, const std::string& output,
-                    size_t keylen)
-   {
-   std::vector<byte> expected = hex_decode(output);
+   std::unique_ptr<Botan::Private_Key> privkey = load_our_key(vars);
+   const std::vector<uint8_t> pubkey = load_their_key(vars);
 
-   std::vector<byte> got = unlock(kas.derive_key(keylen, pubkey).bits_of());
+   const size_t key_len = get_opt_sz(vars, "OutLen", 0);
 
-   size_t fails = 0;
+   Botan::PK_Key_Agreement kas(*privkey, kdf);
 
-   if(got != expected)
-      {
-      std::cout << "FAILED: " << algo << std::endl;
-      dump_data(got, expected);
-      ++fails;
-      }
+   result.test_eq("agreement", kas.derive_key(key_len, pubkey).bits_of(), shared);
 
-   return fails;
+   return result;
    }
 
-size_t test_pk_keygen()
+std::vector<Test::Result> PK_Key_Generation_Test::run()
    {
-   auto& rng = test_rng();
+   std::vector<Test::Result> results;
 
-   size_t tests = 0;
-   size_t fails = 0;
+   for(auto&& param : keygen_params())
+      {
+      std::unique_ptr<Botan::Private_Key> key = make_key(Test::rng(), param);
 
-#define DL_KEY(TYPE, GROUP)                             \
-   {                                                    \
-   TYPE key(rng, DL_Group(GROUP));                      \
-   key.check_key(rng, true);                            \
-   ++tests;                                             \
-   fails += validate_save_and_load(&key, rng);          \
-   }
+      const std::string report_name = key->algo_name() + (param.empty() ? param : " " + param);
 
-#define EC_KEY(TYPE, GROUP)                             \
-   {                                                    \
-   TYPE key(rng, EC_Group(OIDS::lookup(GROUP)));        \
-   key.check_key(rng, true);                            \
-   ++tests;                                             \
-   fails += validate_save_and_load(&key, rng);          \
+      results.push_back(test_key(report_name, *key));
+      }
+   return results;
    }
 
-#if defined(BOTAN_HAS_RSA)
+Test::Result
+PK_Key_Generation_Test::test_key(const std::string& algo, const Botan::Private_Key& key)
+   {
+   Test::Result result(algo + " keygen");
+
+   const std::string pub_pem = Botan::X509::PEM_encode(key);
+
+   try
       {
-      RSA_PrivateKey rsa1024(rng, 1024);
-      rsa1024.check_key(rng, true);
-      ++tests;
-      fails += validate_save_and_load(&rsa1024, rng);
-
-      RSA_PrivateKey rsa2048(rng, 2048);
-      rsa2048.check_key(rng, true);
-      ++tests;
-      fails += validate_save_and_load(&rsa2048, rng);
-      }
-#endif
+      Botan::DataSource_Memory input_pub(pub_pem);
+      std::unique_ptr<Botan::Public_Key> restored_pub(Botan::X509::load_key(input_pub));
 
-#if defined(BOTAN_HAS_RW)
+      result.test_eq("recovered public key from private", restored_pub.get(), true);
+      result.test_eq("public key has same type", restored_pub->algo_name(), key.algo_name());
+      result.test_eq("public key passes checks", restored_pub->check_key(Test::rng(), false), true);
+      }
+   catch(std::exception& e)
       {
-      RW_PrivateKey rw1024(rng, 1024);
-      rw1024.check_key(rng, true);
-      ++tests;
-      fails += validate_save_and_load(&rw1024, rng);
+      result.test_failure("roundtrip public key", e.what());
       }
-#endif
-
-#if defined(BOTAN_HAS_DSA)
-   DL_KEY(DSA_PrivateKey, "dsa/jce/1024");
-   DL_KEY(DSA_PrivateKey, "dsa/botan/2048");
-   DL_KEY(DSA_PrivateKey, "dsa/botan/3072");
-#endif
-
-#if defined(BOTAN_HAS_DIFFIE_HELLMAN)
-   DL_KEY(DH_PrivateKey, "modp/ietf/1024");
-   DL_KEY(DH_PrivateKey, "modp/ietf/2048");
-   DL_KEY(DH_PrivateKey, "modp/ietf/4096");
-   DL_KEY(DH_PrivateKey, "dsa/jce/1024");
-#endif
 
-#if defined(BOTAN_HAS_NYBERG_RUEPPEL)
-   DL_KEY(NR_PrivateKey, "dsa/jce/1024");
-   DL_KEY(NR_PrivateKey, "dsa/botan/2048");
-   DL_KEY(NR_PrivateKey, "dsa/botan/3072");
-#endif
+   const std::string priv_pem = Botan::PKCS8::PEM_encode(key);
 
-#if defined(BOTAN_HAS_ELGAMAL)
-   DL_KEY(ElGamal_PrivateKey, "modp/ietf/1024");
-   DL_KEY(ElGamal_PrivateKey, "dsa/jce/1024");
-   DL_KEY(ElGamal_PrivateKey, "dsa/botan/2048");
-   DL_KEY(ElGamal_PrivateKey, "dsa/botan/3072");
-#endif
+   try
+      {
+      Botan::DataSource_Memory input_priv(priv_pem);
+      std::unique_ptr<Botan::Private_Key> restored_priv(
+         Botan::PKCS8::load_key(input_priv, Test::rng()));
 
-#if defined(BOTAN_HAS_ECDSA)
-   EC_KEY(ECDSA_PrivateKey, "secp112r1");
-   EC_KEY(ECDSA_PrivateKey, "secp128r1");
-   EC_KEY(ECDSA_PrivateKey, "secp160r1");
-   EC_KEY(ECDSA_PrivateKey, "secp192r1");
-   EC_KEY(ECDSA_PrivateKey, "secp224r1");
-   EC_KEY(ECDSA_PrivateKey, "secp256r1");
-   EC_KEY(ECDSA_PrivateKey, "secp384r1");
-   EC_KEY(ECDSA_PrivateKey, "secp521r1");
-#endif
+      result.test_eq("recovered private key from blob", restored_priv.get(), true);
+      result.test_eq("reloaded key has same type", restored_priv->algo_name(), key.algo_name());
+      result.test_eq("private key passes checks", restored_priv->check_key(Test::rng(), false), true);
+      }
+   catch(std::exception& e)
+      {
+      result.test_failure("roundtrip private key", e.what());
+      }
 
-#if defined(BOTAN_HAS_GOST_34_10_2001)
-   EC_KEY(GOST_3410_PrivateKey, "gost_256A");
-   EC_KEY(GOST_3410_PrivateKey, "secp112r1");
-   EC_KEY(GOST_3410_PrivateKey, "secp128r1");
-   EC_KEY(GOST_3410_PrivateKey, "secp160r1");
-   EC_KEY(GOST_3410_PrivateKey, "secp192r1");
-   EC_KEY(GOST_3410_PrivateKey, "secp224r1");
-   EC_KEY(GOST_3410_PrivateKey, "secp256r1");
-   EC_KEY(GOST_3410_PrivateKey, "secp384r1");
-   EC_KEY(GOST_3410_PrivateKey, "secp521r1");
-#endif
+   const std::string passphrase = Test::random_password();
+   const std::string enc_priv_pem = Botan::PKCS8::PEM_encode(key, Test::rng(), passphrase,
+                                                             std::chrono::milliseconds(10));
+   try
+      {
+      Botan::DataSource_Memory input_priv(priv_pem);
+      std::unique_ptr<Botan::Private_Key> restored_priv(
+         Botan::PKCS8::load_key(input_priv, Test::rng(), passphrase));
 
-   test_report("PK keygen", tests, fails);
+      result.test_eq("recovered private key from encrypted blob", restored_priv.get(), true);
+      result.test_eq("reloaded key has same type", restored_priv->algo_name(), key.algo_name());
+      result.test_eq("private key passes checks", restored_priv->check_key(Test::rng(), false), true);
+      }
+   catch(std::exception& e)
+      {
+      result.test_failure("roundtrip private key", e.what());
+      }
 
-   return fails;
+   return result;
    }
 
-#else
-
-SKIP_TEST(pk_keygen);
+}
 
-#endif // BOTAN_HAS_PUBLIC_KEY_CRYPTO
+#endif