Run the test suite in multiple threads

Refactor areas where data was being shared

6 files changed, 149 insertions, 68 deletions

diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index 995510ec7..3cf7526f0 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -61,7 +61,7 @@ int main(int argc, char* argv[])
       const std::string arg_spec =
          "botan-test --verbose --help --data-dir= --pkcs11-lib= --provider= "
          "--log-success --abort-on-first-fail --no-avoid-undefined --skip-tests= "
-         "--run-long-tests --run-online-tests --test-runs=1 --drbg-seed= "
+         "--test-threads=1 --run-long-tests --run-online-tests --test-runs=1 --drbg-seed= "
          "*suites";
 
       Botan_CLI::Argument_Parser parser(arg_spec);
@@ -74,6 +74,15 @@ int main(int argc, char* argv[])
          return 0;
          }
 
+#if defined(BOTAN_TARGET_OS_HAS_POSIX1) && defined(BOTAN_TARGET_OS_HAS_THREADS)
+      // The mlock pool becomes a major contention point when many
+      // threads are running.
+      if(parser.get_arg_sz("test-threads") != 1)
+         {
+         ::setenv("BOTAN_MLOCK_POOL_SIZE", "0", 1);
+         }
+#endif
+
       const Botan_Tests::Test_Options opts(
          parser.get_arg_list("suites"),
          parser.get_arg_list("skip-tests"),
@@ -82,6 +91,7 @@ int main(int argc, char* argv[])
          parser.get_arg("provider"),
          parser.get_arg("drbg-seed"),
          parser.get_arg_sz("test-runs"),
+         parser.get_arg_sz("test-threads"),
          parser.flag_set("verbose"),
          parser.flag_set("log-success"),
          parser.flag_set("run-online-tests"),
diff --git a/src/tests/test_rng.cpp b/src/tests/test_rng.cpp
index a7510b650..e62f81623 100644
--- a/src/tests/test_rng.cpp
+++ b/src/tests/test_rng.cpp
@@ -60,9 +60,19 @@ class Stateful_RNG_Tests : public Test
          results.push_back(test_broken_entropy_input());
          results.push_back(test_check_nonce());
          results.push_back(test_prediction_resistance());
-         results.push_back(test_fork_safety());
          results.push_back(test_randomize_with_ts_input());
          results.push_back(test_security_level());
+
+         /*
+         * This test uses the library in both parent and child processes. But
+         * this causes a race with other threads, where if any other test thread
+         * is holding the mlock pool mutex, it is killed after the fork. Then,
+         * in the child, any attempt to allocate or free memory will cause a
+         * deadlock.
+         */
+         if(Test::options().test_threads() == 1)
+            results.push_back(test_fork_safety());
+
          return results;
          }
 
diff --git a/src/tests/test_runner.cpp b/src/tests/test_runner.cpp
index e346e9afa..86ceab1f8 100644
--- a/src/tests/test_runner.cpp
+++ b/src/tests/test_runner.cpp
@@ -12,6 +12,10 @@
 #include <botan/loadstor.h>
 #include <botan/cpuid.h>
 
+#if defined(BOTAN_HAS_THREAD_UTILS)
+   #include <botan/internal/thread_pool.h>
+#endif
+
 namespace Botan_Tests {
 
 Test_Runner::Test_Runner(std::ostream& out) : m_output(out) {}
@@ -196,7 +200,7 @@ int Test_Runner::run(const Test_Options& opts)
 
       Botan_Tests::Test::set_test_rng(std::move(rng));
 
-      const size_t failed = run_tests(req, i, opts.test_runs());
+      const size_t failed = run_tests(req, opts.test_threads(), i, opts.test_runs());
       if(failed > 0)
          return static_cast<int>(failed);
       }
@@ -236,9 +240,68 @@ std::string report_out(const std::vector<Botan_Tests::Test::Result>& results,
    return out.str();
    }
 
+std::vector<Test::Result> run_a_test(const std::string& test_name)
+   {
+   std::vector<Test::Result> results;
+
+   try
+      {
+      if(test_name == "simd_32" && Botan::CPUID::has_simd_32() == false)
+         {
+         results.push_back(Test::Result::Note(test_name, "SIMD not available on this platform"));
+         }
+      else if(std::unique_ptr<Test> test = Test::get_test(test_name))
+         {
+         std::vector<Test::Result> test_results = test->run();
+         results.insert(results.end(), test_results.begin(), test_results.end());
+         }
+      else
+         {
+         results.push_back(Test::Result::Note(test_name, "Test missing or unavailable"));
+         }
+      }
+   catch(std::exception& e)
+      {
+      results.push_back(Test::Result::Failure(test_name, e.what()));
+      }
+   catch(...)
+      {
+      results.push_back(Test::Result::Failure(test_name, "unknown exception"));
+      }
+
+   return results;
+   }
+
+std::string test_summary(size_t test_run, size_t tot_test_runs, uint64_t total_ns,
+                         size_t tests_ran, size_t tests_failed)
+   {
+   std::ostringstream oss;
+
+   if(test_run == 0 && tot_test_runs == 1)
+      oss << "Tests";
+   else
+      oss << "Test run " << (1+test_run) << "/" << tot_test_runs;
+
+   oss << " complete ran " << tests_ran << " tests in "
+            << Botan_Tests::Test::format_time(total_ns) << " ";
+
+   if(tests_failed > 0)
+      {
+      oss << tests_failed << " tests failed";
+      }
+   else if(tests_ran > 0)
+      {
+      oss << "all tests ok";
+      }
+
+   oss << "\n";
+   return oss.str();
+   }
+
 }
 
 size_t Test_Runner::run_tests(const std::vector<std::string>& tests_to_run,
+                              size_t test_threads,
                               size_t test_run,
                               size_t tot_test_runs)
    {
@@ -246,60 +309,48 @@ size_t Test_Runner::run_tests(const std::vector<std::string>& tests_to_run,
 
    const uint64_t start_time = Botan_Tests::Test::timestamp();
 
-   for(auto const& test_name : tests_to_run)
+   if(test_threads != 1)
       {
-      output() << test_name << ':' << std::endl;
+#if defined(BOTAN_HAS_THREAD_UTILS)
+      // If 0 then we let thread pool select the count
+      Botan::Thread_Pool pool(test_threads);
 
-      std::vector<Test::Result> results;
+      std::vector<std::future<std::vector<Test::Result>>> m_fut_results;
 
-      try
+      for(auto const& test_name : tests_to_run)
          {
-         if(test_name == "simd_32" && Botan::CPUID::has_simd_32() == false)
-            {
-            results.push_back(Test::Result::Note(test_name, "SIMD not available on this platform"));
-            }
-         else if(Test* test = Test::get_test(test_name))
-            {
-            std::vector<Test::Result> test_results = test->run();
-            results.insert(results.end(), test_results.begin(), test_results.end());
-            }
-         else
-            {
-            results.push_back(Test::Result::Note(test_name, "Test missing or unavailable"));
-            }
+         m_fut_results.push_back(pool.run(run_a_test, test_name));
          }
-      catch(std::exception& e)
-         {
-         results.push_back(Test::Result::Failure(test_name, e.what()));
-         }
-      catch(...)
+
+      for(size_t i = 0; i != m_fut_results.size(); ++i)
          {
-         results.push_back(Test::Result::Failure(test_name, "unknown exception"));
+         output() << tests_to_run[i] << ':' << std::endl;
+         const std::vector<Test::Result> results = m_fut_results[i].get();
+         output() << report_out(results, tests_failed, tests_ran) << std::flush;
          }
 
-      output() << report_out(results, tests_failed, tests_ran) << std::flush;
-      }
-
-   const uint64_t total_ns = Botan_Tests::Test::timestamp() - start_time;
+      pool.shutdown();
 
-   if(test_run == 0 && tot_test_runs == 1)
-      output() << "Tests";
-   else
-      output() << "Test run " << (1+test_run) << "/" << tot_test_runs;
+      const uint64_t total_ns = Botan_Tests::Test::timestamp() - start_time;
 
-   output() << " complete ran " << tests_ran << " tests in "
-            << Botan_Tests::Test::format_time(total_ns) << " ";
+      output() << test_summary(test_run, tot_test_runs, total_ns, tests_ran, tests_failed);
 
-   if(tests_failed > 0)
-      {
-      output() << tests_failed << " tests failed";
+      return tests_failed;
+#else
+      output() << "Running tests in multiple threads not enabled in this build\n";
+#endif
       }
-   else if(tests_ran > 0)
+
+   for(auto const& test_name : tests_to_run)
       {
-      output() << "all tests ok";
+      output() << test_name << ':' << std::endl;
+      const std::vector<Test::Result> results = run_a_test(test_name);
+      output() << report_out(results, tests_failed, tests_ran) << std::flush;
       }
 
-   output() << std::endl;
+   const uint64_t total_ns = Botan_Tests::Test::timestamp() - start_time;
+
+   output() << test_summary(test_run, tot_test_runs, total_ns, tests_ran, tests_failed);
 
    return tests_failed;
    }
diff --git a/src/tests/test_runner.h b/src/tests/test_runner.h
index 0bde5cc4f..47d4f1363 100644
--- a/src/tests/test_runner.h
+++ b/src/tests/test_runner.h
@@ -26,8 +26,9 @@ class Test_Runner final
       std::ostream& output() const { return m_output; }
 
       size_t run_tests(const std::vector<std::string>& tests_to_run,
+                       size_t test_threads,
                        size_t test_run,
-                       const size_t tot_test_runs);
+                       size_t tot_test_runs);
 
       std::ostream& m_output;
    };
diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp
index 2300dfaea..6b44bccd9 100644
--- a/src/tests/tests.cpp
+++ b/src/tests/tests.cpp
@@ -29,18 +29,6 @@
 
 namespace Botan_Tests {
 
-Test::Registration::Registration(const std::string& name, Test* test)
-   {
-   if(Test::global_registry().count(name) == 0)
-      {
-      Test::global_registry().insert(std::make_pair(name, std::unique_ptr<Test>(test)));
-      }
-   else
-      {
-      throw Test_Error("Duplicate registration of test '" + name + "'");
-      }
-   }
-
 void Test::Result::merge(const Result& other)
    {
    if(who() != other.who())
@@ -484,9 +472,9 @@ std::string Test::Result::result_string() const
 
 // static Test:: functions
 //static
-std::map<std::string, std::unique_ptr<Test>>& Test::global_registry()
+std::map<std::string, std::function<Test* ()>>& Test::global_registry()
    {
-   static std::map<std::string, std::unique_ptr<Test>> g_test_registry;
+   static std::map<std::string, std::function<Test* ()>> g_test_registry;
    return g_test_registry;
    }
 
@@ -504,12 +492,12 @@ std::set<std::string> Test::registered_tests()
    }
 
 //static
-Test* Test::get_test(const std::string& test_name)
+std::unique_ptr<Test> Test::get_test(const std::string& test_name)
    {
    auto i = Test::global_registry().find(test_name);
    if(i != Test::global_registry().end())
       {
-      return i->second.get();
+      return std::unique_ptr<Test>(i->second());
       }
    return nullptr;
    }
@@ -596,6 +584,14 @@ void Test::set_test_options(const Test_Options& opts)
 //static
 void Test::set_test_rng(std::unique_ptr<Botan::RandomNumberGenerator> rng)
    {
+#if defined(BOTAN_TARGET_OS_HAS_THREADS)
+   if(m_opts.test_threads() != 1)
+      {
+      m_test_rng.reset(new Botan::Serialized_RNG(rng.release()));
+      return;
+      }
+#endif
+
    m_test_rng.reset(rng.release());
    }
 
diff --git a/src/tests/tests.h b/src/tests/tests.h
index 93e972900..4bf9fb0db 100644
--- a/src/tests/tests.h
+++ b/src/tests/tests.h
@@ -60,6 +60,7 @@ class Test_Options
                    const std::string& provider,
                    const std::string& drbg_seed,
                    size_t test_runs,
+                   size_t test_threads,
                    bool verbose,
                    bool log_success,
                    bool run_online_tests,
@@ -73,6 +74,7 @@ class Test_Options
          m_provider(provider),
          m_drbg_seed(drbg_seed),
          m_test_runs(test_runs),
+         m_test_threads(test_threads),
          m_verbose(verbose),
          m_log_success(log_success),
          m_run_online_tests(run_online_tests),
@@ -98,6 +100,8 @@ class Test_Options
 
       size_t test_runs() const { return m_test_runs; }
 
+      size_t test_threads() const { return m_test_threads; }
+
       bool log_success() const { return m_log_success; }
 
       bool run_online_tests() const { return m_run_online_tests; }
@@ -126,6 +130,7 @@ class Test_Options
       std::string m_provider;
       std::string m_drbg_seed;
       size_t m_test_runs;
+      size_t m_test_threads;
       bool m_verbose;
       bool m_log_success;
       bool m_run_online_tests;
@@ -436,22 +441,30 @@ class Test
             std::vector<std::string> m_log;
          };
 
-      class Registration final
-         {
-         public:
-            Registration(const std::string& name, Test* test);
-         };
-
       virtual ~Test() = default;
       virtual std::vector<Test::Result> run() = 0;
 
       virtual std::vector<std::string> possible_providers(const std::string&);
 
-      static std::map<std::string, std::unique_ptr<Test>>& global_registry();
+      template<typename Test_Class>
+      class Registration
+         {
+         public:
+            Registration(const std::string& name)
+               {
+               if(Test::global_registry().count(name) != 0)
+                  throw Test_Error("Duplicate registration of test '" + name + "'");
+
+               auto maker = []() -> Test* { return new Test_Class; };
+               Test::global_registry().insert(std::make_pair(name, maker));
+               }
+         };
+
+      static std::map<std::string, std::function<Test* ()>>& global_registry();
 
       static std::set<std::string> registered_tests();
 
-      static Test* get_test(const std::string& test_name);
+      static std::unique_ptr<Test> get_test(const std::string& test_name);
 
       static std::string data_file(const std::string& what);
 
@@ -515,7 +528,7 @@ class Test
 * Register the test with the runner
 */
 #define BOTAN_REGISTER_TEST(type, Test_Class) \
-   Test::Registration reg_ ## Test_Class ## _tests(type, new Test_Class)
+   Test::Registration<Test_Class> reg_ ## Test_Class ## _tests(type)
 
 class VarMap
    {