Update and consolidate the test framework.

The tests previously had used 4 to 6 different schemes internally (the vec file
reader framework, Catch, the old InSiTo Boost.Test tests, the PK/BigInt tests
which escaped the rewrite in 1.11.7, plus a number of one-offs). Converge on a
design that works everywhere, and update all the things.

Fix also a few bugs found by the test changes: SHA-512-256 name incorrect,
OpenSSL RC4 name incorrect, signature of FFI function botan_pubkey_destroy
was wrong.

1 files changed, 315 insertions, 167 deletions

diff --git a/src/tests/tests.h b/src/tests/tests.h
index 1e496ccb2..c431eb6bd 100644
--- a/src/tests/tests.h
+++ b/src/tests/tests.h
@@ -1,3 +1,4 @@
+
 /*
 * (C) 2014,2015 Jack Lloyd
 * (C) 2015 Simon Warta (Kullo GmbH)
@@ -10,105 +11,340 @@
 
 #include <botan/build.h>
 #include <botan/rng.h>
+#include <botan/hex.h>
+
+#if defined(BOTAN_HAS_BIGINT)
+  #include <botan/bigint.h>
+#endif
+
+#if defined(BOTAN_HAS_EC_CURVE_GFP)
+  #include <botan/point_gfp.h>
+#endif
+
+#include <fstream>
 #include <functional>
-#include <istream>
 #include <map>
+#include <memory>
+#include <set>
+#include <sstream>
 #include <string>
+#include <unordered_map>
 #include <vector>
-#include <iostream>
-#include <sstream>
 
-Botan::RandomNumberGenerator& test_rng();
+namespace Botan_Tests {
 
-size_t run_tests_bb(std::istream& src,
-                    const std::string& name_key,
-                    const std::string& output_key,
-                    bool clear_between_cb,
-                    std::function<size_t (std::map<std::string, std::string>)> cb);
+using Botan::byte;
 
-size_t run_tests(std::istream& src,
-                 const std::string& name_key,
-                 const std::string& output_key,
-                 bool clear_between_cb,
-                 std::function<std::string (std::map<std::string, std::string>)> cb);
+#if defined(BOTAN_HAS_BIGINT)
+using Botan::BigInt;
+#endif
 
-size_t run_tests(const std::string& filename,
-                 const std::string& name_key,
-                 const std::string& output_key,
-                 bool clear_between_cb,
-                 std::function<std::string (std::map<std::string, std::string>)> cb);
+/*
+* A generic test which retuns a set of results when run.
+* The tests may not all have the same type (for example test
+* "block" returns results for "AES-128" and "AES-256").
+*
+* For most test cases you want Text_Based_Test derived below
+*/
+class Test
+   {
+   public:
 
-size_t run_tests_in_dir(const std::string& dir, std::function<size_t (const std::string&)> fn);
+      /*
+      * Some number of test results, all associated with who()
+      */
+      class Result
+         {
+         public:
+            Result(const std::string& who = "") : m_who(who) {}
+
+            size_t tests_passed() const { return m_tests_passed; }
+            size_t tests_failed() const { return m_fail_log.size(); }
+            size_t tests_run() const { return tests_passed() + tests_failed(); }
+            bool any_results() const { return tests_run() > 0; }
+
+            const std::string& who() const { return m_who; }
+            std::string result_string() const;
+
+            static Result Failure(const std::string& who,
+                                  const std::string& what)
+               {
+               Result r(who);
+               r.test_failure(what);
+               return r;
+               }
+
+            static Result Note(const std::string& who,
+                               const std::string& what)
+               {
+               Result r(who);
+               r.test_note(what);
+               return r;
+               }
+
+            void merge(const Result& other);
+
+            void test_note(const std::string& note);
+
+            void note_missing(const std::string& thing);
+
+            bool test_success(const std::string& note = "");
+
+            bool test_failure(const std::string& err);
+
+            bool test_failure(const char* what, const char* error);
+
+            void test_failure(const char* what, const uint8_t buf[], size_t buf_len);
+
+            template<typename Alloc>
+            void test_failure(const char* what, const std::vector<uint8_t, Alloc>& buf)
+               {
+               test_failure(what, buf.data(), buf.size());
+               }
+
+            bool confirm(const char* what, bool expr)
+               {
+               return test_eq(what, expr, true);
+               }
+
+            template<typename T>
+            bool test_is_eq(const T& produced, const T& expected)
+               {
+               return test_is_eq(nullptr, produced, expected);
+               }
+
+            template<typename T>
+            bool test_is_eq(const char* what, const T& produced, const T& expected)
+               {
+               std::ostringstream out;
+               out << m_who;
+               if(what)
+                  out << " " << what;
+
+               if(produced == expected)
+                  {
+                  out << " produced expected result " << produced;
+                  return test_success(out.str());
+                  }
+               else
+                  {
+                  out << " produced unexpected result " << produced << " expected " << expected;
+                  return test_failure(out.str());
+                  }
+               }
+
+            bool test_eq(const char* what, const char* produced, const char* expected);
+            bool test_eq(const char* what, const std::string& produced, const std::string& expected);
+            bool test_eq(const char* what, bool produced, bool expected);
+
+            bool test_eq(const char* what, size_t produced, size_t expected);
+            bool test_lt(const char* what, size_t produced, size_t expected);
+            bool test_gte(const char* what, size_t produced, size_t expected);
+
+            bool test_rc_ok(const char* func, int rc);
+            bool test_rc_fail(const char* func, const char* why, int rc);
+
+#if defined(BOTAN_HAS_BIGINT)
+            bool test_eq(const char* what, const BigInt& produced, const BigInt& expected);
+            bool test_ne(const char* what, const BigInt& produced, const BigInt& expected);
+#endif
 
-// Run a list of tests
-typedef std::function<size_t ()> test_fn;
+#if defined(BOTAN_HAS_EC_CURVE_GFP)
+            bool test_eq(const char* what, const Botan::PointGFp& a, const Botan::PointGFp& b);
+#endif
 
-size_t run_tests(const std::vector<std::pair<std::string, test_fn>>& tests);
-void test_report(const std::string& name, size_t ran, size_t failed);
+            bool test_eq(const char* producer, const char* what,
+                         const uint8_t produced[], size_t produced_len,
+                         const uint8_t expected[], size_t expected_len);
+
+            bool test_ne(const char* what,
+                         const uint8_t produced[], size_t produced_len,
+                         const uint8_t expected[], size_t expected_len);
+
+            template<typename Alloc1, typename Alloc2>
+            bool test_eq(const char* what,
+                         const std::vector<uint8_t, Alloc1>& produced,
+                         const std::vector<uint8_t, Alloc2>& expected)
+               {
+               return test_eq(nullptr, what,
+                              produced.data(), produced.size(),
+                              expected.data(), expected.size());
+               }
+
+            template<typename Alloc1, typename Alloc2>
+            bool test_eq(const std::string& producer, const char* what,
+                         const std::vector<uint8_t, Alloc1>& produced,
+                         const std::vector<uint8_t, Alloc2>& expected)
+               {
+               return test_eq(producer.c_str(), what,
+                              produced.data(), produced.size(),
+                              expected.data(), expected.size());
+               }
+
+            template<typename Alloc>
+            bool test_eq(const char* what,
+                         const std::vector<uint8_t, Alloc>& produced,
+                         const char* expected_hex)
+               {
+               const std::vector<byte> expected = Botan::hex_decode(expected_hex);
+               return test_eq(nullptr, what,
+                              produced.data(), produced.size(),
+                              expected.data(), expected.size());
+               }
+
+            template<typename Alloc1, typename Alloc2>
+            bool test_ne(const char* what,
+                         const std::vector<uint8_t, Alloc1>& produced,
+                         const std::vector<uint8_t, Alloc2>& expected)
+               {
+               return test_ne(what,
+                              produced.data(), produced.size(),
+                              expected.data(), expected.size());
+               }
+
+            bool test_throws(const std::string& what, std::function<void ()> fn);
+
+            void set_ns_consumed(uint64_t ns) { m_ns_taken = ns; }
+
+         private:
+            std::string m_who;
+            uint64_t m_ns_taken = 0;
+            size_t m_tests_passed = 0;
+            std::vector<std::string> m_fail_log;
+            std::vector<std::string> m_log;
+         };
+
+      class Registration
+         {
+         public:
+            Registration(const std::string& name, Test* test);
+         };
 
-class Test_State
-   {
-   public:
-      void started(const std::string& /*msg*/) { m_tests_run++; }
+      virtual std::vector<Test::Result> run() = 0;
+      virtual ~Test() {}
+
+      static std::vector<Test::Result> run_test(const std::string& what, bool fail_if_missing);
+
+      static std::map<std::string, std::unique_ptr<Test>>& global_registry();
 
-      void test_ran(const char* msg);
+      static std::set<std::string> registered_tests();
 
-      void failure(const char* test, const std::string& what_failed)
+      static Test* get_test(const std::string& test_name);
+
+      static std::string data_dir(const std::string& what);
+      static std::string data_file(const std::string& what);
+
+      template<typename Alloc>
+      static std::vector<uint8_t, Alloc> mutate_vec(const std::vector<uint8_t, Alloc>& v, bool maybe_resize = false)
          {
-         std::cout << "FAIL " << test << " " << what_failed << "\n";
-         m_tests_failed++;
+         auto& rng = Test::rng();
+
+         std::vector<uint8_t, Alloc> r = v;
+
+         if(maybe_resize && (r.empty() || rng.next_byte() < 32))
+            {
+            // TODO: occasionally truncate, insert at random index
+            const size_t add = 1 + (rng.next_byte() % 16);
+            r.resize(r.size() + add);
+            rng.randomize(&r[r.size() - add], add);
+            }
+
+         if(r.size() > 0)
+            {
+            const size_t offset = rng.get_random<uint16_t>() % r.size();
+            r[offset] ^= rng.next_nonzero_byte();
+            }
+
+         return r;
          }
 
-      size_t ran() const { return m_tests_run; }
-      size_t failed() const { return m_tests_failed; }
+      static void setup_tests(size_t soak, bool log_succcss, Botan::RandomNumberGenerator* rng);
+
+      static size_t soak_level();
+      static bool log_success();
+
+      static Botan::RandomNumberGenerator& rng();
+      static std::string random_password();
+      static uint64_t timestamp(); // nanoseconds arbitrary epoch
+
+   private:
+      static Botan::RandomNumberGenerator* m_test_rng;
+      static size_t m_soak_level;
+      static bool m_log_success;
+   };
+
+/*
+* Register the test with the runner
+*/
+#define BOTAN_REGISTER_TEST(type, Test_Class) namespace { Test::Registration reg_ ## Test_Class ## _tests(type, new Test_Class); }
+
+/*
+* A test based on reading an input file which contains key/value pairs
+* Special note: the last value in required_key (there must be at least
+* one), is the output key. This triggers the callback.
+*
+* Calls run_one_test with the variables set. If an ini-style [header]
+* is used in the file, then header will be set to that value. This allows
+* splitting up tests between [valid] and [invalid] tests, or different
+* related algorithms tested in the same file. Use the protected get_XXX
+* functions to retrieve formatted values from the VarMap
+*
+* If most of your tests are text-based but you find yourself with a few
+* odds-and-ends tests that you want to do, override run_final_tests which
+* can test whatever it likes and returns a vector of Results.
+*/
+class Text_Based_Test : public Test
+   {
+   public:
+      Text_Based_Test(const std::string& input_file,
+                      const std::vector<std::string>& required_keys,
+                      const std::vector<std::string>& optional_keys = {});
+
+      Text_Based_Test(const std::string& algo,
+                      const std::string& input_file,
+                      const std::vector<std::string>& required_keys,
+                      const std::vector<std::string>& optional_keys = {});
+
+      virtual bool clear_between_callbacks() const { return true; }
+
+      std::vector<Test::Result> run() override;
+   protected:
+      typedef std::unordered_map<std::string, std::string> VarMap;
+      std::string get_next_line();
+
+      virtual Test::Result run_one_test(const std::string& header,
+                                        const VarMap& vars) = 0;
+
+      virtual std::vector<Test::Result> run_final_tests() { return std::vector<Test::Result>(); }
+
+      std::vector<uint8_t> get_req_bin(const VarMap& vars, const std::string& key) const;
+      std::vector<uint8_t> get_opt_bin(const VarMap& vars, const std::string& key) const;
+
+#if defined(BOTAN_HAS_BIGINT)
+      Botan::BigInt get_req_bn(const VarMap& vars, const std::string& key) const;
+#endif
+
+      std::string get_req_str(const VarMap& vars, const std::string& key) const;
+      std::string get_opt_str(const VarMap& vars, const std::string& key, const std::string& def_value) const;
+
+      size_t get_req_sz(const VarMap& vars, const std::string& key) const;
+      size_t get_opt_sz(const VarMap& vars, const std::string& key, const size_t def_value) const;
+
+      std::string algo_name() const { return m_algo; }
    private:
-      size_t m_tests_run = 0, m_tests_failed = 0;
+      std::string m_algo;
+      std::string m_data_dir;
+      std::set<std::string> m_required_keys;
+      std::set<std::string> m_optional_keys;
+      std::string m_output_key;
+      bool m_clear_between_cb = false;
+
+      bool m_first = true;
+      std::unique_ptr<std::ifstream> m_cur;
+      std::deque<std::string> m_srcs;
    };
 
-#define BOTAN_CONFIRM_NOTHROW(block) do {                              \
-   try { block }                                                        \
-   catch(std::exception& e) {                                           \
-      _test.failure(BOTAN_CURRENT_FUNCTION, e.what());  \
-   } } while(0)                                                         \
-
-#define BOTAN_TEST(lhs, rhs, msg) do {                                     \
-   _test.started(msg);                                                  \
-   BOTAN_CONFIRM_NOTHROW({                                              \
-      const auto lhs_val = lhs;                                         \
-      const auto rhs_val = rhs;                                         \
-      const bool cmp = lhs_val == rhs_val;                              \
-      if(!cmp)                                                          \
-         {                                                              \
-         std::ostringstream fmt;                                        \
-         fmt << "expr '" << #lhs << " == " << #rhs << "' false, "       \
-             << "actually " << lhs_val << " " << rhs_val                \
-             << " (" << msg << ")";                                     \
-         _test.failure(BOTAN_CURRENT_FUNCTION, fmt.str()); \
-         }                                                              \
-      });                                                               \
-   } while(0)
-
-#define BOTAN_CONFIRM(expr, msg) do {                                     \
-   _test.started(msg);                                                  \
-   BOTAN_CONFIRM_NOTHROW({                                              \
-      const bool expr_val = expr;                                         \
-      if(!expr_val)                                                     \
-         {                                                              \
-         std::ostringstream fmt;                                        \
-         fmt << "expr '" << #expr << " false (" << msg << ")";           \
-         _test.failure(BOTAN_CURRENT_FUNCTION, fmt.str());              \
-         }                                                              \
-      });                                                               \
-   } while(0)
-
-#define BOTAN_TEST_CASE(name, descr, block) size_t test_ ## name() {     \
-   Test_State _test;                                                    \
-   BOTAN_CONFIRM_NOTHROW(block);                                           \
-   test_report(descr, _test.ran(), _test.failed());                     \
-   return _test.failed();                                   \
-   }
-
-//#define TEST(expr, msg) do { if(!(expr)) { ++fails; std::cout << msg; } while(0)
+}
 
 #define TEST_DATA_DIR     "src/tests/data"
 #define TEST_DATA_DIR_PK  "src/tests/data/pubkey"
@@ -116,92 +352,4 @@ class Test_State
 
 #define TEST_OUTDATA_DIR  "src/tests/outdata"
 
-int test_main(int argc, char* argv[]);
-
-// Tests using reader framework above
-size_t test_block();
-size_t test_stream();
-size_t test_hash();
-size_t test_mac();
-size_t test_modes();
-size_t test_rngs();
-size_t test_pbkdf();
-size_t test_kdf();
-size_t test_aead();
-size_t test_transform();
-
-size_t test_rsa();
-size_t test_rw();
-size_t test_dsa();
-size_t test_nr();
-size_t test_dh();
-size_t test_dlies();
-size_t test_elgamal();
-size_t test_ecc_pointmul();
-size_t test_ecc_random();
-size_t test_ecdsa();
-size_t test_gost_3410();
-size_t test_curve25519();
-size_t test_gf2m();
-size_t test_mceliece();
-size_t test_mce();
-
-// One off tests
-size_t test_ocb();
-size_t test_keywrap();
-size_t test_bcrypt();
-size_t test_passhash9();
-size_t test_cryptobox();
-size_t test_tss();
-size_t test_rfc6979();
-
-size_t test_pk_keygen();
-
-size_t test_bigint();
-
-size_t test_ecc_unit();
-size_t test_ecc_randomized();
-size_t test_ecdsa_unit();
-size_t test_ecdh_unit();
-
-size_t test_x509();
-size_t test_x509_x509test();
-size_t test_cvc();
-
-size_t test_tls();
-
-size_t test_nist_x509();
-
-size_t test_srp6();
-size_t test_compression();
-
-size_t test_fuzzer();
-
-#define SKIP_TEST(testname) \
-   size_t test_ ## testname() {                                    \
-      std::cout << "Skipping tests: " << # testname  << std::endl; \
-      return 0; \
-   } \
-
-/*
- * Warn if a test requires loading more modules than necessary to build
- * the lib. E.g.
- *    $ ./configure.py --no-autoload --enable-modules='ocb'
- *    $ make
- *    $ ./botan-test ocb
- * warns the user whereas 
- *    $ ./configure.py --no-autoload --enable-modules='ocb,aes'
- *    $ make
- *    $ ./botan-test ocb
- * runs the test.
- */
-#define UNTESTED_WARNING(testname) \
-   size_t test_ ## testname() {                                       \
-      std::cout << "Skipping tests: " << # testname << std::endl;     \
-      std::cout << "WARNING: " << # testname << " has been compiled " \
-                << "but is not tested due to other missing modules."  \
-                << std::endl; \
-      return 0; \
-   } \
-
 #endif