aboutsummaryrefslogtreecommitdiffstats
path: root/src/tests
diff options
context:
space:
mode:
Diffstat (limited to 'src/tests')
-rw-r--r--src/tests/main.cpp16
-rw-r--r--src/tests/test_bigint.cpp13
-rw-r--r--src/tests/test_c25519.cpp2
-rw-r--r--src/tests/test_dl_group.cpp18
-rw-r--r--src/tests/test_mceliece.cpp36
-rw-r--r--src/tests/test_passhash.cpp4
-rw-r--r--src/tests/test_pubkey.cpp8
-rw-r--r--src/tests/test_x509_path.cpp4
-rw-r--r--src/tests/test_xmss.cpp16
-rw-r--r--src/tests/tests.cpp40
-rw-r--r--src/tests/tests.h15
-rw-r--r--src/tests/unit_ecc.cpp15
12 files changed, 130 insertions, 57 deletions
diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index 3fa6ce4ab..2ba5751aa 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -35,7 +35,7 @@ namespace {
class Test_Runner : public Botan_CLI::Command
{
public:
- Test_Runner() : Command("test --threads=0 --soak=5 --run-online-tests --drbg-seed= --data-dir= --pkcs11-lib= --log-success *suites") {}
+ Test_Runner() : Command("test --threads=0 --run-long-tests --run-online-tests --drbg-seed= --data-dir= --pkcs11-lib= --log-success *suites") {}
std::string help_text() const override
{
@@ -73,10 +73,10 @@ class Test_Runner : public Botan_CLI::Command
void go() override
{
const size_t threads = get_arg_sz("threads");
- const size_t soak_level = get_arg_sz("soak");
const std::string drbg_seed = get_arg("drbg-seed");
const bool log_success = flag_set("log-success");
const bool run_online_tests = flag_set("run-online-tests");
+ const bool run_long_tests = flag_set("run-long-tests");
const std::string data_dir = get_arg_or("data-dir", "src/tests/data");
const std::string pkcs11_lib = get_arg("pkcs11-lib");
@@ -129,9 +129,7 @@ class Test_Runner : public Botan_CLI::Command
if(threads > 1)
output() << " threads:" << threads;
- output() << " soak level:" << soak_level;
-
- if(! pkcs11_lib.empty())
+ if(!pkcs11_lib.empty())
{
output() << " pkcs11 library:" << pkcs11_lib;
}
@@ -180,7 +178,7 @@ class Test_Runner : public Botan_CLI::Command
throw Botan_Tests::Test_Error("No usable RNG enabled in build, aborting tests");
}
- Botan_Tests::Test::setup_tests(soak_level, log_success, run_online_tests,
+ Botan_Tests::Test::setup_tests(log_success, run_online_tests, run_long_tests,
data_dir, pkcs11_lib, rng.get());
const size_t failed = run_tests(req, output(), threads);
@@ -229,6 +227,8 @@ class Test_Runner : public Botan_CLI::Command
{
size_t tests_ran = 0, tests_failed = 0;
+ const uint64_t start_time = Botan_Tests::Test::timestamp();
+
if(threads <= 1)
{
for(auto&& test_name : tests_to_run)
@@ -288,7 +288,9 @@ class Test_Runner : public Botan_CLI::Command
}
}
- out << "Tests complete ran " << tests_ran << " tests ";
+ const uint64_t total_ns = Botan_Tests::Test::timestamp() - start_time;
+ out << "Tests complete ran " << tests_ran << " tests in "
+ << Botan_Tests::Test::format_time(total_ns) << " ";
if(tests_failed > 0)
{
diff --git a/src/tests/test_bigint.cpp b/src/tests/test_bigint.cpp
index 29f0203b8..2d9992abb 100644
--- a/src/tests/test_bigint.cpp
+++ b/src/tests/test_bigint.cpp
@@ -91,17 +91,14 @@ class BigInt_Unit_Tests : public Test
std::vector<size_t> min_ranges{ 0 };
std::vector<size_t> max_ranges{ 10 };
- // This gets slow quickly:
- if(Test::soak_level() > 10)
+ if(Test::run_long_tests())
{
+ // This gets slow quickly:
min_ranges.push_back(10);
max_ranges.push_back(100);
- if(Test::soak_level() > 50)
- {
- min_ranges.push_back(79);
- max_ranges.push_back(293);
- }
+ min_ranges.push_back(79);
+ max_ranges.push_back(293);
}
for(size_t range_min : min_ranges)
@@ -556,7 +553,7 @@ class DSA_ParamGen_Test : public Text_Based_Test
Test::Result result("DSA Parameter Generation");
// These tests are very slow so skip in normal runs
- if(Test::soak_level() <= 5 && p_bits > 1024)
+ if(p_bits > 1024 && Test::run_long_tests() == false)
return result;
try {
diff --git a/src/tests/test_c25519.cpp b/src/tests/test_c25519.cpp
index 6790b3c4e..f5fab60f4 100644
--- a/src/tests/test_c25519.cpp
+++ b/src/tests/test_c25519.cpp
@@ -46,7 +46,7 @@ class Curve25519_Roundtrip_Test : public Test
{
std::vector<Test::Result> results;
- for(size_t i = 0; i <= Test::soak_level(); ++i)
+ for(size_t i = 0; i < 10; ++i)
{
Test::Result result("Curve25519 roundtrip");
diff --git a/src/tests/test_dl_group.cpp b/src/tests/test_dl_group.cpp
index 985d77a1f..27757fdfe 100644
--- a/src/tests/test_dl_group.cpp
+++ b/src/tests/test_dl_group.cpp
@@ -115,13 +115,11 @@ class DL_Group_Tests : public Test
for(std::string name : dl_named)
{
+ // Confirm we can load every group we expect
Botan::DL_Group group(name);
- // These two groups fail verification because pow(g,q,p) != 1
- if(name != "modp/srp/1024" && name != "modp/srp/2048")
- {
- result.test_eq(name + " verifies", group.verify_group(rng, false), true);
- }
+ result.test_ne("DL_Group p is set", group.get_p(), 0);
+ result.test_ne("DL_Group g is set", group.get_g(), 0);
if(name.find("/srp/") == std::string::npos)
{
@@ -134,6 +132,16 @@ class DL_Group_Tests : public Test
result.test_failure("Group " + name + " has no q");
}
}
+
+ if(group.get_p().bits() < 2048 || Test::run_long_tests())
+ {
+ // These two groups fail verification because pow(g,q,p) != 1
+ if(name != "modp/srp/1024" && name != "modp/srp/2048")
+ {
+ result.test_eq(name + " verifies", group.verify_group(rng, false), true);
+ }
+ }
+
}
result.end_timer();
diff --git a/src/tests/test_mceliece.cpp b/src/tests/test_mceliece.cpp
index 22bdd95fe..c8203e474 100644
--- a/src/tests/test_mceliece.cpp
+++ b/src/tests/test_mceliece.cpp
@@ -55,12 +55,21 @@ class McEliece_Keygen_Encrypt_Test : public Text_Based_Test
const size_t keygen_n = get_req_sz(vars, "KeyN");
const size_t keygen_t = get_req_sz(vars, "KeyT");
+ Test::Result result("McEliece keygen");
+ result.start_timer();
+
+ if(Test::run_long_tests() == false && keygen_n > 3072)
+ {
+ result.test_note("Skipping because long");
+ return result;
+ }
+
+ result.start_timer();
+
Botan::HMAC_DRBG rng("SHA-384");
rng.initialize_with(keygen_seed.data(), keygen_seed.size());
Botan::McEliece_PrivateKey mce_priv(rng, keygen_n, keygen_t);
- Test::Result result("McEliece keygen");
-
result.test_eq("public key fingerprint", hash_bytes(mce_priv.public_key_bits()), fprint_pub);
result.test_eq("private key fingerprint", hash_bytes(mce_priv.private_key_bits()), fprint_priv);
@@ -85,6 +94,7 @@ class McEliece_Keygen_Encrypt_Test : public Text_Based_Test
{
}
+ result.end_timer();
return result;
}
@@ -147,8 +157,14 @@ class McEliece_Tests : public Test
for(size_t i = 0; i < sizeof(param_sets)/sizeof(param_sets[0]); ++i)
{
+ if(Test::run_long_tests() == false && param_sets[i].code_length >= 2048)
+ continue;
+
for(size_t t = param_sets[i].t_min; t <= param_sets[i].t_max; ++t)
{
+ Test::Result result("McEliece keygen");
+ result.start_timer();
+
Botan::McEliece_PrivateKey sk1(Test::rng(), param_sets[i].code_length, t);
const Botan::McEliece_PublicKey& pk1 = sk1;
@@ -158,13 +174,12 @@ class McEliece_Tests : public Test
Botan::McEliece_PublicKey pk(pk_enc);
Botan::McEliece_PrivateKey sk(sk_enc);
- Test::Result result("McEliece keygen");
-
result.test_eq("decoded public key equals original", fingerprint(pk1), fingerprint(pk));
-
result.test_eq("decoded private key equals original", fingerprint(sk1), fingerprint(sk));
-
result.test_eq("key validation passes", sk.check_key(Test::rng(), false), true);
+ result.end_timer();
+
+ result.end_timer();
results.push_back(result);
@@ -184,11 +199,13 @@ class McEliece_Tests : public Test
const Botan::McEliece_PublicKey& pk)
{
Test::Result result("McEliece KEM");
+ result.start_timer();
Botan::PK_KEM_Encryptor enc_op(pk, Test::rng(), "KDF2(SHA-256)");
Botan::PK_KEM_Decryptor dec_op(sk, Test::rng(), "KDF2(SHA-256)");
- for(size_t i = 0; i <= Test::soak_level(); i++)
+ const size_t trials = (Test::run_long_tests() ? 30 : 10);
+ for(size_t i = 0; i < trials; i++)
{
Botan::secure_vector<uint8_t> salt = Test::rng().random_vec(i);
@@ -199,6 +216,7 @@ class McEliece_Tests : public Test
result.test_eq("same key", shared_key, shared_key2);
}
+ result.end_timer();
return result;
}
@@ -207,8 +225,9 @@ class McEliece_Tests : public Test
const Botan::McEliece_PublicKey& pk)
{
Test::Result result("McEliece IES");
+ result.start_timer();
- for(size_t i = 0; i <= Test::soak_level(); ++i)
+ for(size_t i = 0; i <= 10; ++i)
{
uint8_t ad[8];
Botan::store_be(static_cast<uint64_t>(i), ad);
@@ -243,6 +262,7 @@ class McEliece_Tests : public Test
}
}
+ result.end_timer();
return result;
}
#endif
diff --git a/src/tests/test_passhash.cpp b/src/tests/test_passhash.cpp
index bfd17a443..42528f51b 100644
--- a/src/tests/test_passhash.cpp
+++ b/src/tests/test_passhash.cpp
@@ -36,7 +36,7 @@ class Bcrypt_Tests : public Text_Based_Test
Test::Result result("bcrypt");
result.test_eq("correct hash accepted", Botan::check_bcrypt(password, passhash), true);
- const size_t max_level = 1 + std::min<size_t>(Test::soak_level() / 2, 10);
+ const size_t max_level = (Test::run_long_tests() ? 18 : 11);
for(size_t level = 1; level <= max_level; ++level)
{
@@ -80,7 +80,7 @@ class Passhash9_Tests : public Text_Based_Test
}
}
- const size_t max_level = 1 + std::min<size_t>(Test::soak_level() / 2, 10);
+ const size_t max_level = (Test::run_long_tests() ? 16 : 8);
for(size_t level = 1; level <= max_level; ++level)
{
diff --git a/src/tests/test_pubkey.cpp b/src/tests/test_pubkey.cpp
index 86d04169e..712fa8df4 100644
--- a/src/tests/test_pubkey.cpp
+++ b/src/tests/test_pubkey.cpp
@@ -33,10 +33,12 @@ void check_invalid_signatures(Test::Result& result,
const std::vector<uint8_t>& message,
const std::vector<uint8_t>& signature)
{
+ const size_t tests_to_run = (Test::run_long_tests() ? 20 : 5);
+
const std::vector<uint8_t> zero_sig(signature.size());
result.test_eq("all zero signature invalid", verifier.verify_message(message, zero_sig), false);
- for(size_t i = 0; i < Test::soak_level(); ++i)
+ for(size_t i = 0; i < tests_to_run; ++i)
{
const std::vector<uint8_t> bad_sig = Test::mutate_vec(signature);
@@ -53,9 +55,11 @@ void check_invalid_ciphertexts(Test::Result& result,
const std::vector<uint8_t>& plaintext,
const std::vector<uint8_t>& ciphertext)
{
+ const size_t tests_to_run = (Test::run_long_tests() ? 20 : 5);
+
size_t ciphertext_accepted = 0, ciphertext_rejected = 0;
- for(size_t i = 0; i < Test::soak_level(); ++i)
+ for(size_t i = 0; i < tests_to_run; ++i)
{
const std::vector<uint8_t> bad_ctext = Test::mutate_vec(ciphertext);
diff --git a/src/tests/test_x509_path.cpp b/src/tests/test_x509_path.cpp
index 63a4a9070..e897d3e01 100644
--- a/src/tests/test_x509_path.cpp
+++ b/src/tests/test_x509_path.cpp
@@ -76,6 +76,7 @@ class X509test_Path_Validation_Tests : public Test
for(auto i = expected.begin(); i != expected.end(); ++i)
{
Test::Result result("X509test path validation");
+ result.start_timer();
const std::string filename = i->first;
const std::string expected_result = i->second;
@@ -94,6 +95,7 @@ class X509test_Path_Validation_Tests : public Test
path_result = Botan::Path_Validation_Result(Botan::Certificate_Status_Code::CANNOT_ESTABLISH_TRUST);
result.test_eq("test " + filename, path_result.result_string(), expected_result);
+ result.end_timer();
results.push_back(result);
}
@@ -171,6 +173,7 @@ std::vector<Test::Result> NIST_Path_Validation_Tests::run()
const std::string test_dir = nist_test_dir + "/" + test_name;
Test::Result result("NIST path validation");
+ result.start_timer();
const std::vector<std::string> all_files = Botan::get_files_recursive(test_dir);
@@ -213,6 +216,7 @@ std::vector<Test::Result> NIST_Path_Validation_Tests::run()
validation_result.result_string(),
expected_result);
+ result.end_timer();
results.push_back(result);
}
diff --git a/src/tests/test_xmss.cpp b/src/tests/test_xmss.cpp
index 73b23483e..da58057e2 100644
--- a/src/tests/test_xmss.cpp
+++ b/src/tests/test_xmss.cpp
@@ -27,6 +27,22 @@ class XMSS_Signature_Tests : public PK_Signature_Generation_Test
"Params,Msg,PrivateKey,Signature")
{}
+ bool skip_this_test(const std::string&,
+ const VarMap& vars) override
+ {
+ if(Test::run_long_tests() == false)
+ {
+ const std::string params = get_req_str(vars, "Params");
+
+ if(params == "SHAKE128_W16_H10")
+ return false;
+
+ return true;
+ }
+
+ return false;
+ }
+
std::string default_padding(const VarMap& vars) const override
{
return get_req_str(vars, "Params");
diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp
index 81da95736..a4d81135a 100644
--- a/src/tests/tests.cpp
+++ b/src/tests/tests.cpp
@@ -337,9 +337,8 @@ bool Test::Result::test_rc(const std::string& func, int expected, int rc)
return test_success();
}
-namespace {
-
-std::string format_time(uint64_t ns)
+//static
+std::string Test::format_time(uint64_t ns)
{
std::ostringstream o;
@@ -355,8 +354,6 @@ std::string format_time(uint64_t ns)
return o.str();
}
-}
-
std::string Test::Result::result_string(bool verbose) const
{
if(tests_run() == 0 && !verbose)
@@ -475,34 +472,28 @@ std::vector<Test::Result> Test::run_test(const std::string& test_name, bool fail
// static member variables of Test
Botan::RandomNumberGenerator* Test::m_test_rng = nullptr;
std::string Test::m_data_dir;
-size_t Test::m_soak_level = 0;
bool Test::m_log_success = false;
bool Test::m_run_online_tests = false;
+bool Test::m_run_long_tests = false;
std::string Test::m_pkcs11_lib;
//static
-void Test::setup_tests(size_t soak,
- bool log_success,
+void Test::setup_tests(bool log_success,
bool run_online,
+ bool run_long,
const std::string& data_dir,
const std::string& pkcs11_lib,
Botan::RandomNumberGenerator* rng)
{
m_data_dir = data_dir;
- m_soak_level = soak;
m_log_success = log_success;
m_run_online_tests = run_online;
+ m_run_long_tests = run_long;
m_test_rng = rng;
m_pkcs11_lib = pkcs11_lib;
}
//static
-size_t Test::soak_level()
- {
- return m_soak_level;
- }
-
-//static
std::string Test::data_file(const std::string& what)
{
return Test::data_dir() + "/" + what;
@@ -527,6 +518,12 @@ bool Test::run_online_tests()
}
//static
+bool Test::run_long_tests()
+ {
+ return m_run_long_tests;
+ }
+
+//static
std::string Test::pkcs11_lib()
{
return m_pkcs11_lib;
@@ -787,6 +784,12 @@ parse_cpuid_bits(const std::vector<std::string>& tok)
}
+bool Text_Based_Test::skip_this_test(const std::string& /*header*/,
+ const VarMap& /*vars*/)
+ {
+ return false;
+ }
+
std::vector<Test::Result> Text_Based_Test::run()
{
std::vector<Test::Result> results;
@@ -816,7 +819,9 @@ std::vector<Test::Result> Text_Based_Test::run()
continue;
}
else if(line[0] == '#')
+ {
throw Test_Error("Unknown test pragma '" + line + "' in " + m_cur_src_name);
+ }
if(line[0] == '[' && line[line.size()-1] == ']')
{
@@ -850,6 +855,11 @@ std::vector<Test::Result> Text_Based_Test::run()
{
try
{
+ if(skip_this_test(header, vars))
+ {
+ continue;
+ }
+
++test_cnt;
uint64_t start = Test::timestamp();
diff --git a/src/tests/tests.h b/src/tests/tests.h
index ce6431e35..dd7b3784b 100644
--- a/src/tests/tests.h
+++ b/src/tests/tests.h
@@ -328,6 +328,8 @@ class Test
static std::string data_file(const std::string& what);
+ static std::string format_time(uint64_t nanoseconds);
+
template<typename Alloc>
static std::vector<uint8_t, Alloc>
mutate_vec(const std::vector<uint8_t, Alloc>& v,
@@ -356,16 +358,16 @@ class Test
return r;
}
- static void setup_tests(size_t soak,
- bool log_succcss,
+ static void setup_tests(bool log_succcss,
bool run_online_tests,
+ bool run_long_tests,
const std::string& data_dir,
const std::string& pkcs11_lib,
Botan::RandomNumberGenerator* rng);
- static size_t soak_level();
static bool log_success();
static bool run_online_tests();
+ static bool run_long_tests();
static std::string pkcs11_lib();
static const std::string& data_dir();
@@ -377,8 +379,7 @@ class Test
private:
static std::string m_data_dir;
static Botan::RandomNumberGenerator* m_test_rng;
- static size_t m_soak_level;
- static bool m_log_success, m_run_online_tests;
+ static bool m_log_success, m_run_online_tests, m_run_long_tests;
static std::string m_pkcs11_lib;
};
@@ -421,6 +422,10 @@ class Text_Based_Test : public Test
virtual Test::Result run_one_test(const std::string& header,
const VarMap& vars) = 0;
+ // Called before run_one_test
+ virtual bool skip_this_test(const std::string& header,
+ const VarMap& vars);
+
virtual std::vector<Test::Result> run_final_tests() { return std::vector<Test::Result>(); }
bool get_req_bool(const VarMap& vars, const std::string& key) const;
diff --git a/src/tests/unit_ecc.cpp b/src/tests/unit_ecc.cpp
index 5961ca5e3..dca93f57f 100644
--- a/src/tests/unit_ecc.cpp
+++ b/src/tests/unit_ecc.cpp
@@ -134,6 +134,8 @@ std::vector<Test::Result> ECC_Randomized_Tests::run()
{
Test::Result result("ECC randomized " + group_name);
+ result.start_timer();
+
Botan::EC_Group group(group_name);
const Botan::PointGFp& base_point = group.get_base_point();
@@ -145,7 +147,8 @@ std::vector<Test::Result> ECC_Randomized_Tests::run()
try
{
- for(size_t i = 0; i <= Test::soak_level(); ++i)
+ const size_t trials = (Test::run_long_tests() ? 10 : 3);
+ for(size_t i = 0; i < trials; ++i)
{
const size_t h = 1 + (Test::rng().next_byte() % 8);
Botan::Blinded_Point_Multiply blind(base_point, group_order, h);
@@ -171,8 +174,6 @@ std::vector<Test::Result> ECC_Randomized_Tests::run()
result.test_eq("p on the curve", P.on_the_curve(), true);
result.test_eq("q on the curve", Q.on_the_curve(), true);
result.test_eq("r on the curve", R.on_the_curve(), true);
- result.test_eq("a1 on the curve", A1.on_the_curve(), true);
- result.test_eq("a2 on the curve", A2.on_the_curve(), true);
result.test_eq("P1", P1, P);
result.test_eq("Q1", Q1, Q);
@@ -183,6 +184,7 @@ std::vector<Test::Result> ECC_Randomized_Tests::run()
{
result.test_failure(group_name, e.what());
}
+ result.end_timer();
results.push_back(result);
}
@@ -220,8 +222,11 @@ class NIST_Curve_Reduction_Tests : public Test
Botan::secure_vector<Botan::word> ws;
Test::Result result("NIST " + prime_name + " reduction");
+ result.start_timer();
+
+ const size_t trials = (Test::run_long_tests() ? 128 : 16);
- for(size_t i = 0; i <= 10 * Test::soak_level(); ++i)
+ for(size_t i = 0; i <= trials; ++i)
{
const Botan::BigInt x = test_integer(Test::rng(), 2*p_bits, p2);
@@ -239,6 +244,8 @@ class NIST_Curve_Reduction_Tests : public Test
}
}
+ result.end_timer();
+
return result;
}
};