diff options
Diffstat (limited to 'src')
-rwxr-xr-x | src/scripts/ci/travis/build.sh | 2 | ||||
-rw-r--r-- | src/tests/main.cpp | 16 | ||||
-rw-r--r-- | src/tests/test_bigint.cpp | 13 | ||||
-rw-r--r-- | src/tests/test_c25519.cpp | 2 | ||||
-rw-r--r-- | src/tests/test_dl_group.cpp | 18 | ||||
-rw-r--r-- | src/tests/test_mceliece.cpp | 36 | ||||
-rw-r--r-- | src/tests/test_passhash.cpp | 4 | ||||
-rw-r--r-- | src/tests/test_pubkey.cpp | 8 | ||||
-rw-r--r-- | src/tests/test_x509_path.cpp | 4 | ||||
-rw-r--r-- | src/tests/test_xmss.cpp | 16 | ||||
-rw-r--r-- | src/tests/tests.cpp | 40 | ||||
-rw-r--r-- | src/tests/tests.h | 15 | ||||
-rw-r--r-- | src/tests/unit_ecc.cpp | 15 |
13 files changed, 131 insertions, 58 deletions
diff --git a/src/scripts/ci/travis/build.sh b/src/scripts/ci/travis/build.sh index 0accbc3db..d997a96bf 100755 --- a/src/scripts/ci/travis/build.sh +++ b/src/scripts/ci/travis/build.sh @@ -54,7 +54,7 @@ elif [ "${BUILD_MODE:0:5}" != "cross" ]; then if [ "$BUILD_MODE" = "coverage" ]; then CFG_FLAGS+=(--with-tpm) - TEST_FLAGS=(--run-online-tests --pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so) + TEST_FLAGS=(--run-long-tests --run-online-tests --pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so) fi # Avoid OpenSSL when using dynamic checkers, or on OS X where it sporadically diff --git a/src/tests/main.cpp b/src/tests/main.cpp index 3fa6ce4ab..2ba5751aa 100644 --- a/src/tests/main.cpp +++ b/src/tests/main.cpp @@ -35,7 +35,7 @@ namespace { class Test_Runner : public Botan_CLI::Command { public: - Test_Runner() : Command("test --threads=0 --soak=5 --run-online-tests --drbg-seed= --data-dir= --pkcs11-lib= --log-success *suites") {} + Test_Runner() : Command("test --threads=0 --run-long-tests --run-online-tests --drbg-seed= --data-dir= --pkcs11-lib= --log-success *suites") {} std::string help_text() const override { @@ -73,10 +73,10 @@ class Test_Runner : public Botan_CLI::Command void go() override { const size_t threads = get_arg_sz("threads"); - const size_t soak_level = get_arg_sz("soak"); const std::string drbg_seed = get_arg("drbg-seed"); const bool log_success = flag_set("log-success"); const bool run_online_tests = flag_set("run-online-tests"); + const bool run_long_tests = flag_set("run-long-tests"); const std::string data_dir = get_arg_or("data-dir", "src/tests/data"); const std::string pkcs11_lib = get_arg("pkcs11-lib"); @@ -129,9 +129,7 @@ class Test_Runner : public Botan_CLI::Command if(threads > 1) output() << " threads:" << threads; - output() << " soak level:" << soak_level; - - if(! pkcs11_lib.empty()) + if(!pkcs11_lib.empty()) { output() << " pkcs11 library:" << pkcs11_lib; } @@ -180,7 +178,7 @@ class Test_Runner : public Botan_CLI::Command throw Botan_Tests::Test_Error("No usable RNG enabled in build, aborting tests"); } - Botan_Tests::Test::setup_tests(soak_level, log_success, run_online_tests, + Botan_Tests::Test::setup_tests(log_success, run_online_tests, run_long_tests, data_dir, pkcs11_lib, rng.get()); const size_t failed = run_tests(req, output(), threads); @@ -229,6 +227,8 @@ class Test_Runner : public Botan_CLI::Command { size_t tests_ran = 0, tests_failed = 0; + const uint64_t start_time = Botan_Tests::Test::timestamp(); + if(threads <= 1) { for(auto&& test_name : tests_to_run) @@ -288,7 +288,9 @@ class Test_Runner : public Botan_CLI::Command } } - out << "Tests complete ran " << tests_ran << " tests "; + const uint64_t total_ns = Botan_Tests::Test::timestamp() - start_time; + out << "Tests complete ran " << tests_ran << " tests in " + << Botan_Tests::Test::format_time(total_ns) << " "; if(tests_failed > 0) { diff --git a/src/tests/test_bigint.cpp b/src/tests/test_bigint.cpp index 29f0203b8..2d9992abb 100644 --- a/src/tests/test_bigint.cpp +++ b/src/tests/test_bigint.cpp @@ -91,17 +91,14 @@ class BigInt_Unit_Tests : public Test std::vector<size_t> min_ranges{ 0 }; std::vector<size_t> max_ranges{ 10 }; - // This gets slow quickly: - if(Test::soak_level() > 10) + if(Test::run_long_tests()) { + // This gets slow quickly: min_ranges.push_back(10); max_ranges.push_back(100); - if(Test::soak_level() > 50) - { - min_ranges.push_back(79); - max_ranges.push_back(293); - } + min_ranges.push_back(79); + max_ranges.push_back(293); } for(size_t range_min : min_ranges) @@ -556,7 +553,7 @@ class DSA_ParamGen_Test : public Text_Based_Test Test::Result result("DSA Parameter Generation"); // These tests are very slow so skip in normal runs - if(Test::soak_level() <= 5 && p_bits > 1024) + if(p_bits > 1024 && Test::run_long_tests() == false) return result; try { diff --git a/src/tests/test_c25519.cpp b/src/tests/test_c25519.cpp index 6790b3c4e..f5fab60f4 100644 --- a/src/tests/test_c25519.cpp +++ b/src/tests/test_c25519.cpp @@ -46,7 +46,7 @@ class Curve25519_Roundtrip_Test : public Test { std::vector<Test::Result> results; - for(size_t i = 0; i <= Test::soak_level(); ++i) + for(size_t i = 0; i < 10; ++i) { Test::Result result("Curve25519 roundtrip"); diff --git a/src/tests/test_dl_group.cpp b/src/tests/test_dl_group.cpp index 985d77a1f..27757fdfe 100644 --- a/src/tests/test_dl_group.cpp +++ b/src/tests/test_dl_group.cpp @@ -115,13 +115,11 @@ class DL_Group_Tests : public Test for(std::string name : dl_named) { + // Confirm we can load every group we expect Botan::DL_Group group(name); - // These two groups fail verification because pow(g,q,p) != 1 - if(name != "modp/srp/1024" && name != "modp/srp/2048") - { - result.test_eq(name + " verifies", group.verify_group(rng, false), true); - } + result.test_ne("DL_Group p is set", group.get_p(), 0); + result.test_ne("DL_Group g is set", group.get_g(), 0); if(name.find("/srp/") == std::string::npos) { @@ -134,6 +132,16 @@ class DL_Group_Tests : public Test result.test_failure("Group " + name + " has no q"); } } + + if(group.get_p().bits() < 2048 || Test::run_long_tests()) + { + // These two groups fail verification because pow(g,q,p) != 1 + if(name != "modp/srp/1024" && name != "modp/srp/2048") + { + result.test_eq(name + " verifies", group.verify_group(rng, false), true); + } + } + } result.end_timer(); diff --git a/src/tests/test_mceliece.cpp b/src/tests/test_mceliece.cpp index 22bdd95fe..c8203e474 100644 --- a/src/tests/test_mceliece.cpp +++ b/src/tests/test_mceliece.cpp @@ -55,12 +55,21 @@ class McEliece_Keygen_Encrypt_Test : public Text_Based_Test const size_t keygen_n = get_req_sz(vars, "KeyN"); const size_t keygen_t = get_req_sz(vars, "KeyT"); + Test::Result result("McEliece keygen"); + result.start_timer(); + + if(Test::run_long_tests() == false && keygen_n > 3072) + { + result.test_note("Skipping because long"); + return result; + } + + result.start_timer(); + Botan::HMAC_DRBG rng("SHA-384"); rng.initialize_with(keygen_seed.data(), keygen_seed.size()); Botan::McEliece_PrivateKey mce_priv(rng, keygen_n, keygen_t); - Test::Result result("McEliece keygen"); - result.test_eq("public key fingerprint", hash_bytes(mce_priv.public_key_bits()), fprint_pub); result.test_eq("private key fingerprint", hash_bytes(mce_priv.private_key_bits()), fprint_priv); @@ -85,6 +94,7 @@ class McEliece_Keygen_Encrypt_Test : public Text_Based_Test { } + result.end_timer(); return result; } @@ -147,8 +157,14 @@ class McEliece_Tests : public Test for(size_t i = 0; i < sizeof(param_sets)/sizeof(param_sets[0]); ++i) { + if(Test::run_long_tests() == false && param_sets[i].code_length >= 2048) + continue; + for(size_t t = param_sets[i].t_min; t <= param_sets[i].t_max; ++t) { + Test::Result result("McEliece keygen"); + result.start_timer(); + Botan::McEliece_PrivateKey sk1(Test::rng(), param_sets[i].code_length, t); const Botan::McEliece_PublicKey& pk1 = sk1; @@ -158,13 +174,12 @@ class McEliece_Tests : public Test Botan::McEliece_PublicKey pk(pk_enc); Botan::McEliece_PrivateKey sk(sk_enc); - Test::Result result("McEliece keygen"); - result.test_eq("decoded public key equals original", fingerprint(pk1), fingerprint(pk)); - result.test_eq("decoded private key equals original", fingerprint(sk1), fingerprint(sk)); - result.test_eq("key validation passes", sk.check_key(Test::rng(), false), true); + result.end_timer(); + + result.end_timer(); results.push_back(result); @@ -184,11 +199,13 @@ class McEliece_Tests : public Test const Botan::McEliece_PublicKey& pk) { Test::Result result("McEliece KEM"); + result.start_timer(); Botan::PK_KEM_Encryptor enc_op(pk, Test::rng(), "KDF2(SHA-256)"); Botan::PK_KEM_Decryptor dec_op(sk, Test::rng(), "KDF2(SHA-256)"); - for(size_t i = 0; i <= Test::soak_level(); i++) + const size_t trials = (Test::run_long_tests() ? 30 : 10); + for(size_t i = 0; i < trials; i++) { Botan::secure_vector<uint8_t> salt = Test::rng().random_vec(i); @@ -199,6 +216,7 @@ class McEliece_Tests : public Test result.test_eq("same key", shared_key, shared_key2); } + result.end_timer(); return result; } @@ -207,8 +225,9 @@ class McEliece_Tests : public Test const Botan::McEliece_PublicKey& pk) { Test::Result result("McEliece IES"); + result.start_timer(); - for(size_t i = 0; i <= Test::soak_level(); ++i) + for(size_t i = 0; i <= 10; ++i) { uint8_t ad[8]; Botan::store_be(static_cast<uint64_t>(i), ad); @@ -243,6 +262,7 @@ class McEliece_Tests : public Test } } + result.end_timer(); return result; } #endif diff --git a/src/tests/test_passhash.cpp b/src/tests/test_passhash.cpp index bfd17a443..42528f51b 100644 --- a/src/tests/test_passhash.cpp +++ b/src/tests/test_passhash.cpp @@ -36,7 +36,7 @@ class Bcrypt_Tests : public Text_Based_Test Test::Result result("bcrypt"); result.test_eq("correct hash accepted", Botan::check_bcrypt(password, passhash), true); - const size_t max_level = 1 + std::min<size_t>(Test::soak_level() / 2, 10); + const size_t max_level = (Test::run_long_tests() ? 18 : 11); for(size_t level = 1; level <= max_level; ++level) { @@ -80,7 +80,7 @@ class Passhash9_Tests : public Text_Based_Test } } - const size_t max_level = 1 + std::min<size_t>(Test::soak_level() / 2, 10); + const size_t max_level = (Test::run_long_tests() ? 16 : 8); for(size_t level = 1; level <= max_level; ++level) { diff --git a/src/tests/test_pubkey.cpp b/src/tests/test_pubkey.cpp index 86d04169e..712fa8df4 100644 --- a/src/tests/test_pubkey.cpp +++ b/src/tests/test_pubkey.cpp @@ -33,10 +33,12 @@ void check_invalid_signatures(Test::Result& result, const std::vector<uint8_t>& message, const std::vector<uint8_t>& signature) { + const size_t tests_to_run = (Test::run_long_tests() ? 20 : 5); + const std::vector<uint8_t> zero_sig(signature.size()); result.test_eq("all zero signature invalid", verifier.verify_message(message, zero_sig), false); - for(size_t i = 0; i < Test::soak_level(); ++i) + for(size_t i = 0; i < tests_to_run; ++i) { const std::vector<uint8_t> bad_sig = Test::mutate_vec(signature); @@ -53,9 +55,11 @@ void check_invalid_ciphertexts(Test::Result& result, const std::vector<uint8_t>& plaintext, const std::vector<uint8_t>& ciphertext) { + const size_t tests_to_run = (Test::run_long_tests() ? 20 : 5); + size_t ciphertext_accepted = 0, ciphertext_rejected = 0; - for(size_t i = 0; i < Test::soak_level(); ++i) + for(size_t i = 0; i < tests_to_run; ++i) { const std::vector<uint8_t> bad_ctext = Test::mutate_vec(ciphertext); diff --git a/src/tests/test_x509_path.cpp b/src/tests/test_x509_path.cpp index 63a4a9070..e897d3e01 100644 --- a/src/tests/test_x509_path.cpp +++ b/src/tests/test_x509_path.cpp @@ -76,6 +76,7 @@ class X509test_Path_Validation_Tests : public Test for(auto i = expected.begin(); i != expected.end(); ++i) { Test::Result result("X509test path validation"); + result.start_timer(); const std::string filename = i->first; const std::string expected_result = i->second; @@ -94,6 +95,7 @@ class X509test_Path_Validation_Tests : public Test path_result = Botan::Path_Validation_Result(Botan::Certificate_Status_Code::CANNOT_ESTABLISH_TRUST); result.test_eq("test " + filename, path_result.result_string(), expected_result); + result.end_timer(); results.push_back(result); } @@ -171,6 +173,7 @@ std::vector<Test::Result> NIST_Path_Validation_Tests::run() const std::string test_dir = nist_test_dir + "/" + test_name; Test::Result result("NIST path validation"); + result.start_timer(); const std::vector<std::string> all_files = Botan::get_files_recursive(test_dir); @@ -213,6 +216,7 @@ std::vector<Test::Result> NIST_Path_Validation_Tests::run() validation_result.result_string(), expected_result); + result.end_timer(); results.push_back(result); } diff --git a/src/tests/test_xmss.cpp b/src/tests/test_xmss.cpp index 73b23483e..da58057e2 100644 --- a/src/tests/test_xmss.cpp +++ b/src/tests/test_xmss.cpp @@ -27,6 +27,22 @@ class XMSS_Signature_Tests : public PK_Signature_Generation_Test "Params,Msg,PrivateKey,Signature") {} + bool skip_this_test(const std::string&, + const VarMap& vars) override + { + if(Test::run_long_tests() == false) + { + const std::string params = get_req_str(vars, "Params"); + + if(params == "SHAKE128_W16_H10") + return false; + + return true; + } + + return false; + } + std::string default_padding(const VarMap& vars) const override { return get_req_str(vars, "Params"); diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp index 81da95736..a4d81135a 100644 --- a/src/tests/tests.cpp +++ b/src/tests/tests.cpp @@ -337,9 +337,8 @@ bool Test::Result::test_rc(const std::string& func, int expected, int rc) return test_success(); } -namespace { - -std::string format_time(uint64_t ns) +//static +std::string Test::format_time(uint64_t ns) { std::ostringstream o; @@ -355,8 +354,6 @@ std::string format_time(uint64_t ns) return o.str(); } -} - std::string Test::Result::result_string(bool verbose) const { if(tests_run() == 0 && !verbose) @@ -475,34 +472,28 @@ std::vector<Test::Result> Test::run_test(const std::string& test_name, bool fail // static member variables of Test Botan::RandomNumberGenerator* Test::m_test_rng = nullptr; std::string Test::m_data_dir; -size_t Test::m_soak_level = 0; bool Test::m_log_success = false; bool Test::m_run_online_tests = false; +bool Test::m_run_long_tests = false; std::string Test::m_pkcs11_lib; //static -void Test::setup_tests(size_t soak, - bool log_success, +void Test::setup_tests(bool log_success, bool run_online, + bool run_long, const std::string& data_dir, const std::string& pkcs11_lib, Botan::RandomNumberGenerator* rng) { m_data_dir = data_dir; - m_soak_level = soak; m_log_success = log_success; m_run_online_tests = run_online; + m_run_long_tests = run_long; m_test_rng = rng; m_pkcs11_lib = pkcs11_lib; } //static -size_t Test::soak_level() - { - return m_soak_level; - } - -//static std::string Test::data_file(const std::string& what) { return Test::data_dir() + "/" + what; @@ -527,6 +518,12 @@ bool Test::run_online_tests() } //static +bool Test::run_long_tests() + { + return m_run_long_tests; + } + +//static std::string Test::pkcs11_lib() { return m_pkcs11_lib; @@ -787,6 +784,12 @@ parse_cpuid_bits(const std::vector<std::string>& tok) } +bool Text_Based_Test::skip_this_test(const std::string& /*header*/, + const VarMap& /*vars*/) + { + return false; + } + std::vector<Test::Result> Text_Based_Test::run() { std::vector<Test::Result> results; @@ -816,7 +819,9 @@ std::vector<Test::Result> Text_Based_Test::run() continue; } else if(line[0] == '#') + { throw Test_Error("Unknown test pragma '" + line + "' in " + m_cur_src_name); + } if(line[0] == '[' && line[line.size()-1] == ']') { @@ -850,6 +855,11 @@ std::vector<Test::Result> Text_Based_Test::run() { try { + if(skip_this_test(header, vars)) + { + continue; + } + ++test_cnt; uint64_t start = Test::timestamp(); diff --git a/src/tests/tests.h b/src/tests/tests.h index ce6431e35..dd7b3784b 100644 --- a/src/tests/tests.h +++ b/src/tests/tests.h @@ -328,6 +328,8 @@ class Test static std::string data_file(const std::string& what); + static std::string format_time(uint64_t nanoseconds); + template<typename Alloc> static std::vector<uint8_t, Alloc> mutate_vec(const std::vector<uint8_t, Alloc>& v, @@ -356,16 +358,16 @@ class Test return r; } - static void setup_tests(size_t soak, - bool log_succcss, + static void setup_tests(bool log_succcss, bool run_online_tests, + bool run_long_tests, const std::string& data_dir, const std::string& pkcs11_lib, Botan::RandomNumberGenerator* rng); - static size_t soak_level(); static bool log_success(); static bool run_online_tests(); + static bool run_long_tests(); static std::string pkcs11_lib(); static const std::string& data_dir(); @@ -377,8 +379,7 @@ class Test private: static std::string m_data_dir; static Botan::RandomNumberGenerator* m_test_rng; - static size_t m_soak_level; - static bool m_log_success, m_run_online_tests; + static bool m_log_success, m_run_online_tests, m_run_long_tests; static std::string m_pkcs11_lib; }; @@ -421,6 +422,10 @@ class Text_Based_Test : public Test virtual Test::Result run_one_test(const std::string& header, const VarMap& vars) = 0; + // Called before run_one_test + virtual bool skip_this_test(const std::string& header, + const VarMap& vars); + virtual std::vector<Test::Result> run_final_tests() { return std::vector<Test::Result>(); } bool get_req_bool(const VarMap& vars, const std::string& key) const; diff --git a/src/tests/unit_ecc.cpp b/src/tests/unit_ecc.cpp index 5961ca5e3..dca93f57f 100644 --- a/src/tests/unit_ecc.cpp +++ b/src/tests/unit_ecc.cpp @@ -134,6 +134,8 @@ std::vector<Test::Result> ECC_Randomized_Tests::run() { Test::Result result("ECC randomized " + group_name); + result.start_timer(); + Botan::EC_Group group(group_name); const Botan::PointGFp& base_point = group.get_base_point(); @@ -145,7 +147,8 @@ std::vector<Test::Result> ECC_Randomized_Tests::run() try { - for(size_t i = 0; i <= Test::soak_level(); ++i) + const size_t trials = (Test::run_long_tests() ? 10 : 3); + for(size_t i = 0; i < trials; ++i) { const size_t h = 1 + (Test::rng().next_byte() % 8); Botan::Blinded_Point_Multiply blind(base_point, group_order, h); @@ -171,8 +174,6 @@ std::vector<Test::Result> ECC_Randomized_Tests::run() result.test_eq("p on the curve", P.on_the_curve(), true); result.test_eq("q on the curve", Q.on_the_curve(), true); result.test_eq("r on the curve", R.on_the_curve(), true); - result.test_eq("a1 on the curve", A1.on_the_curve(), true); - result.test_eq("a2 on the curve", A2.on_the_curve(), true); result.test_eq("P1", P1, P); result.test_eq("Q1", Q1, Q); @@ -183,6 +184,7 @@ std::vector<Test::Result> ECC_Randomized_Tests::run() { result.test_failure(group_name, e.what()); } + result.end_timer(); results.push_back(result); } @@ -220,8 +222,11 @@ class NIST_Curve_Reduction_Tests : public Test Botan::secure_vector<Botan::word> ws; Test::Result result("NIST " + prime_name + " reduction"); + result.start_timer(); + + const size_t trials = (Test::run_long_tests() ? 128 : 16); - for(size_t i = 0; i <= 10 * Test::soak_level(); ++i) + for(size_t i = 0; i <= trials; ++i) { const Botan::BigInt x = test_integer(Test::rng(), 2*p_bits, p2); @@ -239,6 +244,8 @@ class NIST_Curve_Reduction_Tests : public Test } } + result.end_timer(); + return result; } }; |