CLI rewrite

The command line tools' origin as a collection of examples and test
programs glued together led to some unfortunate problems; lots of
hardcoded values, missing parameters, and obsolete crypto.

Adds a small library for writing command line programs of the sort
needed here (cli.h), which cuts the length of many of the commands in
half and makes commands more pleasant to write and extend.

Generalizes a lot of the commands also, eg previously only
signing/verification with DSA/SHA-1 was included!

Removes the fuzzer entry point since that's fairly useless outside of
an instrumented build.

Removes the in-library API for benchmarking.

4 files changed, 171 insertions, 144 deletions

diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index 7a4649fcf..c4313a4ef 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -4,6 +4,7 @@
 * Botan is released under the Simplified BSD License (see license.txt)
 */
 
+#include "../cli/cli.h"
 #include "tests.h"
 #include <iostream>
 #include <sstream>
@@ -29,23 +30,6 @@ namespace {
 
 using Botan_Tests::Test;
 
-int help(std::ostream& out, const std::string binary_name)
-   {
-   std::ostringstream err;
-
-   err << "Usage:\n"
-       << binary_name << " test1 test2 ...\n"
-       << "Available tests: ";
-
-   for(auto&& test : Test::registered_tests())
-      {
-      err << test << " ";
-      }
-
-   out << err.str() << std::endl;
-   return 1;
-   }
-
 std::string report_out(const std::vector<Test::Result>& results,
                        size_t& tests_failed,
                        size_t& tests_ran)
@@ -76,72 +60,11 @@ std::string report_out(const std::vector<Test::Result>& results,
    return out.str();
    }
 
-size_t run_tests(const std::vector<std::string>& tests_to_run,
-                 std::ostream& out,
-                 size_t threads)
-   {
-   size_t tests_ran = 0, tests_failed = 0;
-
-   if(threads <= 1)
-      {
-      for(auto&& test_name : tests_to_run)
-         {
-         std::vector<Test::Result> results = Test::run_test(test_name, false);
-         out << report_out(results, tests_failed, tests_ran) << std::flush;
-         }
-      }
-   else
-      {
-
-      /*
-      We're not doing this in a particularly nice way, and variance in time is
-      high so commonly we'll 'run dry' by blocking on the first future. But
-      plain C++11 <thread> is missing a lot of tools we'd need (like
-      wait_for_any on a set of futures) and there is no point pulling in an
-      additional dependency just for this. In any case it helps somewhat
-      (50-100% speedup) and provides a proof of concept for parallel testing.
-      */
-
-      typedef std::future<std::vector<Test::Result>> FutureResults;
-      std::deque<FutureResults> fut_results;
-
-      for(auto&& test_name : tests_to_run)
-         {
-         fut_results.push_back(std::async(std::launch::async,
-                                          [test_name]() { return Test::run_test(test_name, false); }));
-
-         while(fut_results.size() > threads)
-            {
-            out << report_out(fut_results[0].get(), tests_failed, tests_ran) << std::flush;
-            fut_results.pop_front();
-            }
-         }
-
-      while(fut_results.size() > 0)
-         {
-         out << report_out(fut_results[0].get(), tests_failed, tests_ran) << std::flush;
-         fut_results.pop_front();
-         }
-      }
-
-   out << "Tests complete ran " << tests_ran << " tests ";
-
-   if(tests_failed > 0)
-      {
-      out << tests_failed << " tests failed";
-      }
-   else if(tests_ran > 0)
-      {
-      out << "all tests ok";
-      }
-
-   out << std::endl;
-
-   return tests_failed;
-   }
-
 std::unique_ptr<Botan::RandomNumberGenerator>
-setup_tests(std::ostream& out, size_t threads, size_t soak_level, bool log_success, std::string drbg_seed)
+setup_tests(std::ostream& out, size_t threads,
+            size_t soak_level,
+            bool log_success,
+            std::string drbg_seed)
    {
    out << "Testing " << Botan::version_string() << "\n";
    out << "Starting tests";
@@ -190,60 +113,166 @@ setup_tests(std::ostream& out, size_t threads, size_t soak_level, bool log_succe
    return rng;
    }
 
-int cpp_main(const std::vector<std::string> args)
+class Test_Runner : public Botan_CLI::Command
    {
-   try
-      {
-      if(args.size() == 2 && (args[1] == "--help" || args[1] == "help"))
+   public:
+      Test_Runner() : Command("test --threads=0 --soak=5 --drbg-seed= --log-success *suites") {}
+
+      std::string help_text() const override
          {
-         return help(std::cout, args[0]);
-         }
+         std::ostringstream err;
+
+         err << "Usage: botan-test [--drbg-seed=] [--threads=N] [--log-success] "
+             << "suite suite ...\n\n"
+             << "Available suites\n"
+             << "----------------\n";
+
+         size_t line_len = 0;
+
+         for(auto&& test : Test::registered_tests())
+            {
+            err << test << " ";
+            line_len += test.size() + 1;
+
+            if(line_len > 64)
+               {
+               err << "\n";
+               line_len = 0;
+               }
+            }
 
-      size_t threads = 0;//std::thread::hardware_concurrency();
-      size_t soak = 5;
-      const std::string drbg_seed = "";
-      bool log_success = false;
+         if(line_len > 0)
+            {
+            err << "\n";
+            }
 
-      std::vector<std::string> req(args.begin()+1, args.end());
+         return err.str();
+         }
 
-      if(req.empty())
+      void go() override
          {
-         req = {"block", "stream", "hash", "mac", "modes", "aead", "kdf", "pbkdf", "hmac_drbg", "x931_rng", "util"};
+         const size_t threads = get_arg_sz("threads");
+         const size_t soak = get_arg_sz("soak");
+         const std::string drbg_seed = get_arg("drbg-seed");
+         bool log_success = flag_set("log-success");
+
+         std::vector<std::string> req = get_arg_list("suites");
 
-         std::set<std::string> all_others = Botan_Tests::Test::registered_tests();
+         if(req.empty())
+            {
+            /*
+            If nothing was requested on the command line, run everything. First
+            run the "essentials" to smoke test, then everything else in
+            alphabetical order.
+            */
+            req = {"block", "stream", "hash", "mac", "modes", "aead"
+                   "kdf", "pbkdf", "hmac_drbg", "x931_rng", "util"};
+
+            std::set<std::string> all_others = Botan_Tests::Test::registered_tests();
+
+            for(auto f : req)
+               {
+               all_others.erase(f);
+               }
+
+            req.insert(req.end(), all_others.begin(), all_others.end());
+            }
+
+         std::unique_ptr<Botan::RandomNumberGenerator> rng =
+            setup_tests(std::cout, threads, soak, log_success, drbg_seed);
 
-         for(auto f : req)
-            all_others.erase(f);
+         size_t failed = run_tests(req, std::cout, threads);
 
-         req.insert(req.end(), all_others.begin(), all_others.end());
+         // Throw so main returns an error
+         if(failed)
+            throw Botan_Tests::Test_Error("Test suite failure");
          }
+   private:
+      size_t run_tests(const std::vector<std::string>& tests_to_run,
+                       std::ostream& out,
+                       size_t threads)
+         {
+         size_t tests_ran = 0, tests_failed = 0;
 
-      std::unique_ptr<Botan::RandomNumberGenerator> rng =
-         setup_tests(std::cout, threads, soak, log_success, drbg_seed);
+         if(threads <= 1)
+            {
+            for(auto&& test_name : tests_to_run)
+               {
+               std::vector<Test::Result> results = Test::run_test(test_name, false);
+               out << report_out(results, tests_failed, tests_ran) << std::flush;
+               }
+            }
+         else
+            {
 
-      size_t failed = run_tests(req, std::cout, threads);
+            /*
+            We're not doing this in a particularly nice way, and variance in time is
+            high so commonly we'll 'run dry' by blocking on the first future. But
+            plain C++11 <thread> is missing a lot of tools we'd need (like
+            wait_for_any on a set of futures) and there is no point pulling in an
+            additional dependency just for this. In any case it helps somewhat
+            (50-100% speedup) and provides a proof of concept for parallel testing.
+            */
+
+            typedef std::future<std::vector<Test::Result>> FutureResults;
+            std::deque<FutureResults> fut_results;
+
+            for(auto&& test_name : tests_to_run)
+               {
+               fut_results.push_back(std::async(std::launch::async,
+                                                [test_name]() { return Test::run_test(test_name, false); }));
+
+               while(fut_results.size() > threads)
+                  {
+                  out << report_out(fut_results[0].get(), tests_failed, tests_ran) << std::flush;
+                  fut_results.pop_front();
+                  }
+               }
+
+            while(fut_results.size() > 0)
+               {
+               out << report_out(fut_results[0].get(), tests_failed, tests_ran) << std::flush;
+               fut_results.pop_front();
+               }
+            }
 
-      if(failed)
-         return 2;
+         out << "Tests complete ran " << tests_ran << " tests ";
 
-      return 0;
-      }
-   catch(std::exception& e)
-      {
-      std::cout << "Exception caused test abort: " << e.what() << std::endl;
-      return 3;
-      }
-   catch(...)
-      {
-      std::cout << "Unknown exception caused test abort" << std::endl;
-      return 3;
-      }
-   }
+         if(tests_failed > 0)
+            {
+            out << tests_failed << " tests failed";
+            }
+         else if(tests_ran > 0)
+            {
+            out << "all tests ok";
+            }
+
+         out << std::endl;
+
+         return tests_failed;
+         }
+
+
+   };
+
+BOTAN_REGISTER_COMMAND(Test_Runner);
 
 }
 
 int main(int argc, char* argv[])
    {
-   std::vector<std::string> args(argv, argv + argc);
-   return cpp_main(args);
+   std::cerr << Botan::runtime_version_check(BOTAN_VERSION_MAJOR,
+                                             BOTAN_VERSION_MINOR,
+                                             BOTAN_VERSION_PATCH);
+
+   Botan_CLI::Command* cmd = Botan_CLI::Command::get_cmd("test");
+
+   if(!cmd)
+      {
+      std::cout << "Unable to retrieve testing helper (program bug)\n"; // WTF
+      return 1;
+      }
+
+   std::vector<std::string> args(argv + 1, argv + argc);
+   return cmd->run(args);
    }
diff --git a/src/tests/test_stream.cpp b/src/tests/test_stream.cpp
index 618d34755..fb38ad677 100644
--- a/src/tests/test_stream.cpp
+++ b/src/tests/test_stream.cpp
@@ -17,7 +17,8 @@ namespace Botan_Tests {
 class Stream_Cipher_Tests : public Text_Based_Test
    {
    public:
-      Stream_Cipher_Tests(): Text_Based_Test(Test::data_dir("stream"), {"Key", "In", "Out"}, {"Nonce"}) {}
+      Stream_Cipher_Tests(): Text_Based_Test(Test::data_dir("stream"),
+                                             {"Key", "In", "Out"}, {"Nonce"}) {}
 
       Test::Result run_one_test(const std::string& algo, const VarMap& vars) override
          {
diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp
index 360d671cc..111622dab 100644
--- a/src/tests/tests.cpp
+++ b/src/tests/tests.cpp
@@ -11,6 +11,7 @@
 #include <botan/hex.h>
 #include <botan/internal/filesystem.h>
 #include <botan/internal/bit_ops.h>
+#include <botan/internal/stl_util.h>
 
 namespace Botan_Tests {
 
@@ -237,7 +238,8 @@ bool Test::Result::test_ne(const std::string& what, const BigInt& produced, cons
 #endif
 
 #if defined(BOTAN_HAS_EC_CURVE_GFP)
-bool Test::Result::test_eq(const std::string& what, const Botan::PointGFp& a, const Botan::PointGFp& b)
+bool Test::Result::test_eq(const std::string& what,
+                           const Botan::PointGFp& a, const Botan::PointGFp& b)
    {
    //return test_is_eq(what, a, b);
    if(a == b)
@@ -358,21 +360,6 @@ std::map<std::string, std::unique_ptr<Test>>& Test::global_registry()
    return g_test_registry;
    }
 
-namespace {
-
-template<typename K, typename V>
-std::set<K> map_keys_as_set(const std::map<K, V>& kv)
-   {
-   std::set<K> s;
-   for(auto&& i : kv)
-      {
-      s.insert(i.first);
-      }
-   return s;
-   }
-
-}
-
 //static
 uint64_t Test::timestamp()
    {
@@ -383,7 +370,7 @@ uint64_t Test::timestamp()
 //static
 std::set<std::string> Test::registered_tests()
    {
-   return map_keys_as_set(Test::global_registry());
+   return Botan::map_keys_as_set(Test::global_registry());
    }
 
 //static
@@ -730,7 +717,8 @@ std::vector<Test::Result> Text_Based_Test::run()
             }
          catch(std::exception& e)
             {
-            results.push_back(Test::Result::Failure(who, "test " + std::to_string(test_cnt) + " failed with exception '" + e.what() + "'"));
+            results.push_back(Test::Result::Failure(who, "test " + std::to_string(test_cnt) +
+                                                    " failed with exception '" + e.what() + "'"));
             }
 
          if(clear_between_callbacks())
diff --git a/src/tests/tests.h b/src/tests/tests.h
index 333cbee98..71b839363 100644
--- a/src/tests/tests.h
+++ b/src/tests/tests.h
@@ -1,4 +1,3 @@
-
 /*
 * (C) 2014,2015 Jack Lloyd
 * (C) 2015 Simon Warta (Kullo GmbH)
@@ -167,7 +166,11 @@ class Test
                }
 
             bool test_eq(const std::string& what, const char* produced, const char* expected);
-            bool test_eq(const std::string& what, const std::string& produced, const std::string& expected);
+
+            bool test_eq(const std::string& what,
+                         const std::string& produced,
+                         const std::string& expected);
+
             bool test_eq(const std::string& what, bool produced, bool expected);
 
             bool test_eq(const std::string& what, size_t produced, size_t expected);
@@ -183,7 +186,9 @@ class Test
 #endif
 
 #if defined(BOTAN_HAS_EC_CURVE_GFP)
-            bool test_eq(const std::string& what, const Botan::PointGFp& a, const Botan::PointGFp& b);
+            bool test_eq(const std::string& what,
+                         const Botan::PointGFp& a,
+                         const Botan::PointGFp& b);
 #endif
 
             bool test_eq(const char* producer, const std::string& what,
@@ -273,7 +278,8 @@ class Test
       static std::string full_path_for_output_file(const std::string& base);
 
       template<typename Alloc>
-      static std::vector<uint8_t, Alloc> mutate_vec(const std::vector<uint8_t, Alloc>& v, bool maybe_resize = false)
+      static std::vector<uint8_t, Alloc>
+      mutate_vec(const std::vector<uint8_t, Alloc>& v, bool maybe_resize = false)
          {
          auto& rng = Test::rng();
 
@@ -314,7 +320,8 @@ class Test
 /*
 * Register the test with the runner
 */
-#define BOTAN_REGISTER_TEST(type, Test_Class) namespace { Test::Registration reg_ ## Test_Class ## _tests(type, new Test_Class); }
+#define BOTAN_REGISTER_TEST(type, Test_Class) \
+   namespace { Test::Registration reg_ ## Test_Class ## _tests(type, new Test_Class); }
 
 /*
 * A test based on reading an input file which contains key/value pairs
@@ -363,7 +370,9 @@ class Text_Based_Test : public Test
 #endif
 
       std::string get_req_str(const VarMap& vars, const std::string& key) const;
-      std::string get_opt_str(const VarMap& vars, const std::string& key, const std::string& def_value) const;
+      std::string get_opt_str(const VarMap& vars,
+                              const std::string& key,
+                              const std::string& def_value) const;
 
       size_t get_req_sz(const VarMap& vars, const std::string& key) const;
       size_t get_opt_sz(const VarMap& vars, const std::string& key, const size_t def_value) const;