aboutsummaryrefslogtreecommitdiffstats
path: root/src/tests
diff options
context:
space:
mode:
Diffstat (limited to 'src/tests')
-rw-r--r--src/tests/main.cpp267
-rw-r--r--src/tests/test_stream.cpp3
-rw-r--r--src/tests/tests.cpp24
-rw-r--r--src/tests/tests.h21
4 files changed, 171 insertions, 144 deletions
diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index 7a4649fcf..c4313a4ef 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -4,6 +4,7 @@
* Botan is released under the Simplified BSD License (see license.txt)
*/
+#include "../cli/cli.h"
#include "tests.h"
#include <iostream>
#include <sstream>
@@ -29,23 +30,6 @@ namespace {
using Botan_Tests::Test;
-int help(std::ostream& out, const std::string binary_name)
- {
- std::ostringstream err;
-
- err << "Usage:\n"
- << binary_name << " test1 test2 ...\n"
- << "Available tests: ";
-
- for(auto&& test : Test::registered_tests())
- {
- err << test << " ";
- }
-
- out << err.str() << std::endl;
- return 1;
- }
-
std::string report_out(const std::vector<Test::Result>& results,
size_t& tests_failed,
size_t& tests_ran)
@@ -76,72 +60,11 @@ std::string report_out(const std::vector<Test::Result>& results,
return out.str();
}
-size_t run_tests(const std::vector<std::string>& tests_to_run,
- std::ostream& out,
- size_t threads)
- {
- size_t tests_ran = 0, tests_failed = 0;
-
- if(threads <= 1)
- {
- for(auto&& test_name : tests_to_run)
- {
- std::vector<Test::Result> results = Test::run_test(test_name, false);
- out << report_out(results, tests_failed, tests_ran) << std::flush;
- }
- }
- else
- {
-
- /*
- We're not doing this in a particularly nice way, and variance in time is
- high so commonly we'll 'run dry' by blocking on the first future. But
- plain C++11 <thread> is missing a lot of tools we'd need (like
- wait_for_any on a set of futures) and there is no point pulling in an
- additional dependency just for this. In any case it helps somewhat
- (50-100% speedup) and provides a proof of concept for parallel testing.
- */
-
- typedef std::future<std::vector<Test::Result>> FutureResults;
- std::deque<FutureResults> fut_results;
-
- for(auto&& test_name : tests_to_run)
- {
- fut_results.push_back(std::async(std::launch::async,
- [test_name]() { return Test::run_test(test_name, false); }));
-
- while(fut_results.size() > threads)
- {
- out << report_out(fut_results[0].get(), tests_failed, tests_ran) << std::flush;
- fut_results.pop_front();
- }
- }
-
- while(fut_results.size() > 0)
- {
- out << report_out(fut_results[0].get(), tests_failed, tests_ran) << std::flush;
- fut_results.pop_front();
- }
- }
-
- out << "Tests complete ran " << tests_ran << " tests ";
-
- if(tests_failed > 0)
- {
- out << tests_failed << " tests failed";
- }
- else if(tests_ran > 0)
- {
- out << "all tests ok";
- }
-
- out << std::endl;
-
- return tests_failed;
- }
-
std::unique_ptr<Botan::RandomNumberGenerator>
-setup_tests(std::ostream& out, size_t threads, size_t soak_level, bool log_success, std::string drbg_seed)
+setup_tests(std::ostream& out, size_t threads,
+ size_t soak_level,
+ bool log_success,
+ std::string drbg_seed)
{
out << "Testing " << Botan::version_string() << "\n";
out << "Starting tests";
@@ -190,60 +113,166 @@ setup_tests(std::ostream& out, size_t threads, size_t soak_level, bool log_succe
return rng;
}
-int cpp_main(const std::vector<std::string> args)
+class Test_Runner : public Botan_CLI::Command
{
- try
- {
- if(args.size() == 2 && (args[1] == "--help" || args[1] == "help"))
+ public:
+ Test_Runner() : Command("test --threads=0 --soak=5 --drbg-seed= --log-success *suites") {}
+
+ std::string help_text() const override
{
- return help(std::cout, args[0]);
- }
+ std::ostringstream err;
+
+ err << "Usage: botan-test [--drbg-seed=] [--threads=N] [--log-success] "
+ << "suite suite ...\n\n"
+ << "Available suites\n"
+ << "----------------\n";
+
+ size_t line_len = 0;
+
+ for(auto&& test : Test::registered_tests())
+ {
+ err << test << " ";
+ line_len += test.size() + 1;
+
+ if(line_len > 64)
+ {
+ err << "\n";
+ line_len = 0;
+ }
+ }
- size_t threads = 0;//std::thread::hardware_concurrency();
- size_t soak = 5;
- const std::string drbg_seed = "";
- bool log_success = false;
+ if(line_len > 0)
+ {
+ err << "\n";
+ }
- std::vector<std::string> req(args.begin()+1, args.end());
+ return err.str();
+ }
- if(req.empty())
+ void go() override
{
- req = {"block", "stream", "hash", "mac", "modes", "aead", "kdf", "pbkdf", "hmac_drbg", "x931_rng", "util"};
+ const size_t threads = get_arg_sz("threads");
+ const size_t soak = get_arg_sz("soak");
+ const std::string drbg_seed = get_arg("drbg-seed");
+ bool log_success = flag_set("log-success");
+
+ std::vector<std::string> req = get_arg_list("suites");
- std::set<std::string> all_others = Botan_Tests::Test::registered_tests();
+ if(req.empty())
+ {
+ /*
+ If nothing was requested on the command line, run everything. First
+ run the "essentials" to smoke test, then everything else in
+ alphabetical order.
+ */
+ req = {"block", "stream", "hash", "mac", "modes", "aead"
+ "kdf", "pbkdf", "hmac_drbg", "x931_rng", "util"};
+
+ std::set<std::string> all_others = Botan_Tests::Test::registered_tests();
+
+ for(auto f : req)
+ {
+ all_others.erase(f);
+ }
+
+ req.insert(req.end(), all_others.begin(), all_others.end());
+ }
+
+ std::unique_ptr<Botan::RandomNumberGenerator> rng =
+ setup_tests(std::cout, threads, soak, log_success, drbg_seed);
- for(auto f : req)
- all_others.erase(f);
+ size_t failed = run_tests(req, std::cout, threads);
- req.insert(req.end(), all_others.begin(), all_others.end());
+ // Throw so main returns an error
+ if(failed)
+ throw Botan_Tests::Test_Error("Test suite failure");
}
+ private:
+ size_t run_tests(const std::vector<std::string>& tests_to_run,
+ std::ostream& out,
+ size_t threads)
+ {
+ size_t tests_ran = 0, tests_failed = 0;
- std::unique_ptr<Botan::RandomNumberGenerator> rng =
- setup_tests(std::cout, threads, soak, log_success, drbg_seed);
+ if(threads <= 1)
+ {
+ for(auto&& test_name : tests_to_run)
+ {
+ std::vector<Test::Result> results = Test::run_test(test_name, false);
+ out << report_out(results, tests_failed, tests_ran) << std::flush;
+ }
+ }
+ else
+ {
- size_t failed = run_tests(req, std::cout, threads);
+ /*
+ We're not doing this in a particularly nice way, and variance in time is
+ high so commonly we'll 'run dry' by blocking on the first future. But
+ plain C++11 <thread> is missing a lot of tools we'd need (like
+ wait_for_any on a set of futures) and there is no point pulling in an
+ additional dependency just for this. In any case it helps somewhat
+ (50-100% speedup) and provides a proof of concept for parallel testing.
+ */
+
+ typedef std::future<std::vector<Test::Result>> FutureResults;
+ std::deque<FutureResults> fut_results;
+
+ for(auto&& test_name : tests_to_run)
+ {
+ fut_results.push_back(std::async(std::launch::async,
+ [test_name]() { return Test::run_test(test_name, false); }));
+
+ while(fut_results.size() > threads)
+ {
+ out << report_out(fut_results[0].get(), tests_failed, tests_ran) << std::flush;
+ fut_results.pop_front();
+ }
+ }
+
+ while(fut_results.size() > 0)
+ {
+ out << report_out(fut_results[0].get(), tests_failed, tests_ran) << std::flush;
+ fut_results.pop_front();
+ }
+ }
- if(failed)
- return 2;
+ out << "Tests complete ran " << tests_ran << " tests ";
- return 0;
- }
- catch(std::exception& e)
- {
- std::cout << "Exception caused test abort: " << e.what() << std::endl;
- return 3;
- }
- catch(...)
- {
- std::cout << "Unknown exception caused test abort" << std::endl;
- return 3;
- }
- }
+ if(tests_failed > 0)
+ {
+ out << tests_failed << " tests failed";
+ }
+ else if(tests_ran > 0)
+ {
+ out << "all tests ok";
+ }
+
+ out << std::endl;
+
+ return tests_failed;
+ }
+
+
+ };
+
+BOTAN_REGISTER_COMMAND(Test_Runner);
}
int main(int argc, char* argv[])
{
- std::vector<std::string> args(argv, argv + argc);
- return cpp_main(args);
+ std::cerr << Botan::runtime_version_check(BOTAN_VERSION_MAJOR,
+ BOTAN_VERSION_MINOR,
+ BOTAN_VERSION_PATCH);
+
+ Botan_CLI::Command* cmd = Botan_CLI::Command::get_cmd("test");
+
+ if(!cmd)
+ {
+ std::cout << "Unable to retrieve testing helper (program bug)\n"; // WTF
+ return 1;
+ }
+
+ std::vector<std::string> args(argv + 1, argv + argc);
+ return cmd->run(args);
}
diff --git a/src/tests/test_stream.cpp b/src/tests/test_stream.cpp
index 618d34755..fb38ad677 100644
--- a/src/tests/test_stream.cpp
+++ b/src/tests/test_stream.cpp
@@ -17,7 +17,8 @@ namespace Botan_Tests {
class Stream_Cipher_Tests : public Text_Based_Test
{
public:
- Stream_Cipher_Tests(): Text_Based_Test(Test::data_dir("stream"), {"Key", "In", "Out"}, {"Nonce"}) {}
+ Stream_Cipher_Tests(): Text_Based_Test(Test::data_dir("stream"),
+ {"Key", "In", "Out"}, {"Nonce"}) {}
Test::Result run_one_test(const std::string& algo, const VarMap& vars) override
{
diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp
index 360d671cc..111622dab 100644
--- a/src/tests/tests.cpp
+++ b/src/tests/tests.cpp
@@ -11,6 +11,7 @@
#include <botan/hex.h>
#include <botan/internal/filesystem.h>
#include <botan/internal/bit_ops.h>
+#include <botan/internal/stl_util.h>
namespace Botan_Tests {
@@ -237,7 +238,8 @@ bool Test::Result::test_ne(const std::string& what, const BigInt& produced, cons
#endif
#if defined(BOTAN_HAS_EC_CURVE_GFP)
-bool Test::Result::test_eq(const std::string& what, const Botan::PointGFp& a, const Botan::PointGFp& b)
+bool Test::Result::test_eq(const std::string& what,
+ const Botan::PointGFp& a, const Botan::PointGFp& b)
{
//return test_is_eq(what, a, b);
if(a == b)
@@ -358,21 +360,6 @@ std::map<std::string, std::unique_ptr<Test>>& Test::global_registry()
return g_test_registry;
}
-namespace {
-
-template<typename K, typename V>
-std::set<K> map_keys_as_set(const std::map<K, V>& kv)
- {
- std::set<K> s;
- for(auto&& i : kv)
- {
- s.insert(i.first);
- }
- return s;
- }
-
-}
-
//static
uint64_t Test::timestamp()
{
@@ -383,7 +370,7 @@ uint64_t Test::timestamp()
//static
std::set<std::string> Test::registered_tests()
{
- return map_keys_as_set(Test::global_registry());
+ return Botan::map_keys_as_set(Test::global_registry());
}
//static
@@ -730,7 +717,8 @@ std::vector<Test::Result> Text_Based_Test::run()
}
catch(std::exception& e)
{
- results.push_back(Test::Result::Failure(who, "test " + std::to_string(test_cnt) + " failed with exception '" + e.what() + "'"));
+ results.push_back(Test::Result::Failure(who, "test " + std::to_string(test_cnt) +
+ " failed with exception '" + e.what() + "'"));
}
if(clear_between_callbacks())
diff --git a/src/tests/tests.h b/src/tests/tests.h
index 333cbee98..71b839363 100644
--- a/src/tests/tests.h
+++ b/src/tests/tests.h
@@ -1,4 +1,3 @@
-
/*
* (C) 2014,2015 Jack Lloyd
* (C) 2015 Simon Warta (Kullo GmbH)
@@ -167,7 +166,11 @@ class Test
}
bool test_eq(const std::string& what, const char* produced, const char* expected);
- bool test_eq(const std::string& what, const std::string& produced, const std::string& expected);
+
+ bool test_eq(const std::string& what,
+ const std::string& produced,
+ const std::string& expected);
+
bool test_eq(const std::string& what, bool produced, bool expected);
bool test_eq(const std::string& what, size_t produced, size_t expected);
@@ -183,7 +186,9 @@ class Test
#endif
#if defined(BOTAN_HAS_EC_CURVE_GFP)
- bool test_eq(const std::string& what, const Botan::PointGFp& a, const Botan::PointGFp& b);
+ bool test_eq(const std::string& what,
+ const Botan::PointGFp& a,
+ const Botan::PointGFp& b);
#endif
bool test_eq(const char* producer, const std::string& what,
@@ -273,7 +278,8 @@ class Test
static std::string full_path_for_output_file(const std::string& base);
template<typename Alloc>
- static std::vector<uint8_t, Alloc> mutate_vec(const std::vector<uint8_t, Alloc>& v, bool maybe_resize = false)
+ static std::vector<uint8_t, Alloc>
+ mutate_vec(const std::vector<uint8_t, Alloc>& v, bool maybe_resize = false)
{
auto& rng = Test::rng();
@@ -314,7 +320,8 @@ class Test
/*
* Register the test with the runner
*/
-#define BOTAN_REGISTER_TEST(type, Test_Class) namespace { Test::Registration reg_ ## Test_Class ## _tests(type, new Test_Class); }
+#define BOTAN_REGISTER_TEST(type, Test_Class) \
+ namespace { Test::Registration reg_ ## Test_Class ## _tests(type, new Test_Class); }
/*
* A test based on reading an input file which contains key/value pairs
@@ -363,7 +370,9 @@ class Text_Based_Test : public Test
#endif
std::string get_req_str(const VarMap& vars, const std::string& key) const;
- std::string get_opt_str(const VarMap& vars, const std::string& key, const std::string& def_value) const;
+ std::string get_opt_str(const VarMap& vars,
+ const std::string& key,
+ const std::string& def_value) const;
size_t get_req_sz(const VarMap& vars, const std::string& key) const;
size_t get_opt_sz(const VarMap& vars, const std::string& key, const size_t def_value) const;
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-25 11:32:49 +0000