Add test option --run-long-tests

Previously longer tests were hidden behind higher 'soak levels'
but these arbitrary cutoffs are confusing compared to a simple
short tests/long tests split.

13 files changed, 131 insertions, 58 deletions

diff --git a/src/scripts/ci/travis/build.sh b/src/scripts/ci/travis/build.sh
index 0accbc3db..d997a96bf 100755
--- a/src/scripts/ci/travis/build.sh
+++ b/src/scripts/ci/travis/build.sh
@@ -54,7 +54,7 @@ elif [ "${BUILD_MODE:0:5}" != "cross" ]; then
 
     if [ "$BUILD_MODE" = "coverage" ]; then
         CFG_FLAGS+=(--with-tpm)
-        TEST_FLAGS=(--run-online-tests --pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so)
+        TEST_FLAGS=(--run-long-tests --run-online-tests --pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so)
     fi
 
     # Avoid OpenSSL when using dynamic checkers, or on OS X where it sporadically
diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index 3fa6ce4ab..2ba5751aa 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -35,7 +35,7 @@ namespace {
 class Test_Runner : public Botan_CLI::Command
    {
    public:
-      Test_Runner() : Command("test --threads=0 --soak=5 --run-online-tests --drbg-seed= --data-dir= --pkcs11-lib= --log-success *suites") {}
+      Test_Runner() : Command("test --threads=0 --run-long-tests --run-online-tests --drbg-seed= --data-dir= --pkcs11-lib= --log-success *suites") {}
 
       std::string help_text() const override
          {
@@ -73,10 +73,10 @@ class Test_Runner : public Botan_CLI::Command
       void go() override
          {
          const size_t threads = get_arg_sz("threads");
-         const size_t soak_level = get_arg_sz("soak");
          const std::string drbg_seed = get_arg("drbg-seed");
          const bool log_success = flag_set("log-success");
          const bool run_online_tests = flag_set("run-online-tests");
+         const bool run_long_tests = flag_set("run-long-tests");
          const std::string data_dir = get_arg_or("data-dir", "src/tests/data");
          const std::string pkcs11_lib = get_arg("pkcs11-lib");
 
@@ -129,9 +129,7 @@ class Test_Runner : public Botan_CLI::Command
          if(threads > 1)
             output() << " threads:" << threads;
 
-         output() << " soak level:" << soak_level;
-
-         if(! pkcs11_lib.empty())
+         if(!pkcs11_lib.empty())
             {
             output() << " pkcs11 library:" << pkcs11_lib;
             }
@@ -180,7 +178,7 @@ class Test_Runner : public Botan_CLI::Command
              throw Botan_Tests::Test_Error("No usable RNG enabled in build, aborting tests");
              }
 
-         Botan_Tests::Test::setup_tests(soak_level, log_success, run_online_tests,
+         Botan_Tests::Test::setup_tests(log_success, run_online_tests, run_long_tests,
                                         data_dir, pkcs11_lib, rng.get());
 
          const size_t failed = run_tests(req, output(), threads);
@@ -229,6 +227,8 @@ class Test_Runner : public Botan_CLI::Command
          {
          size_t tests_ran = 0, tests_failed = 0;
 
+         const uint64_t start_time = Botan_Tests::Test::timestamp();
+
          if(threads <= 1)
             {
             for(auto&& test_name : tests_to_run)
@@ -288,7 +288,9 @@ class Test_Runner : public Botan_CLI::Command
                }
             }
 
-         out << "Tests complete ran " << tests_ran << " tests ";
+         const uint64_t total_ns = Botan_Tests::Test::timestamp() - start_time;
+         out << "Tests complete ran " << tests_ran << " tests in "
+             << Botan_Tests::Test::format_time(total_ns) << " ";
 
          if(tests_failed > 0)
             {
diff --git a/src/tests/test_bigint.cpp b/src/tests/test_bigint.cpp
index 29f0203b8..2d9992abb 100644
--- a/src/tests/test_bigint.cpp
+++ b/src/tests/test_bigint.cpp
@@ -91,17 +91,14 @@ class BigInt_Unit_Tests : public Test
          std::vector<size_t> min_ranges{ 0 };
          std::vector<size_t> max_ranges{ 10 };
 
-         // This gets slow quickly:
-         if(Test::soak_level() > 10)
+         if(Test::run_long_tests())
             {
+            // This gets slow quickly:
             min_ranges.push_back(10);
             max_ranges.push_back(100);
 
-            if(Test::soak_level() > 50)
-               {
-               min_ranges.push_back(79);
-               max_ranges.push_back(293);
-               }
+            min_ranges.push_back(79);
+            max_ranges.push_back(293);
             }
 
          for(size_t range_min : min_ranges)
@@ -556,7 +553,7 @@ class DSA_ParamGen_Test : public Text_Based_Test
          Test::Result result("DSA Parameter Generation");
 
          // These tests are very slow so skip in normal runs
-         if(Test::soak_level() <= 5 && p_bits > 1024)
+         if(p_bits > 1024 && Test::run_long_tests() == false)
             return result;
 
          try {
diff --git a/src/tests/test_c25519.cpp b/src/tests/test_c25519.cpp
index 6790b3c4e..f5fab60f4 100644
--- a/src/tests/test_c25519.cpp
+++ b/src/tests/test_c25519.cpp
@@ -46,7 +46,7 @@ class Curve25519_Roundtrip_Test : public Test
          {
          std::vector<Test::Result> results;
 
-         for(size_t i = 0; i <= Test::soak_level(); ++i)
+         for(size_t i = 0; i < 10; ++i)
             {
             Test::Result result("Curve25519 roundtrip");
 
diff --git a/src/tests/test_dl_group.cpp b/src/tests/test_dl_group.cpp
index 985d77a1f..27757fdfe 100644
--- a/src/tests/test_dl_group.cpp
+++ b/src/tests/test_dl_group.cpp
@@ -115,13 +115,11 @@ class DL_Group_Tests : public Test
 
          for(std::string name : dl_named)
             {
+            // Confirm we can load every group we expect
             Botan::DL_Group group(name);
 
-            // These two groups fail verification because pow(g,q,p) != 1
-            if(name != "modp/srp/1024" && name != "modp/srp/2048")
-               {
-               result.test_eq(name + " verifies", group.verify_group(rng, false), true);
-               }
+            result.test_ne("DL_Group p is set", group.get_p(), 0);
+            result.test_ne("DL_Group g is set", group.get_g(), 0);
 
             if(name.find("/srp/") == std::string::npos)
                {
@@ -134,6 +132,16 @@ class DL_Group_Tests : public Test
                   result.test_failure("Group " + name + " has no q");
                   }
                }
+
+            if(group.get_p().bits() < 2048 || Test::run_long_tests())
+               {
+               // These two groups fail verification because pow(g,q,p) != 1
+               if(name != "modp/srp/1024" && name != "modp/srp/2048")
+                  {
+                  result.test_eq(name + " verifies", group.verify_group(rng, false), true);
+                  }
+               }
+
             }
          result.end_timer();
 
diff --git a/src/tests/test_mceliece.cpp b/src/tests/test_mceliece.cpp
index 22bdd95fe..c8203e474 100644
--- a/src/tests/test_mceliece.cpp
+++ b/src/tests/test_mceliece.cpp
@@ -55,12 +55,21 @@ class McEliece_Keygen_Encrypt_Test : public Text_Based_Test
          const size_t keygen_n = get_req_sz(vars, "KeyN");
          const size_t keygen_t = get_req_sz(vars, "KeyT");
 
+         Test::Result result("McEliece keygen");
+         result.start_timer();
+
+         if(Test::run_long_tests() == false && keygen_n > 3072)
+            {
+            result.test_note("Skipping because long");
+            return result;
+            }
+
+         result.start_timer();
+
          Botan::HMAC_DRBG rng("SHA-384");
          rng.initialize_with(keygen_seed.data(), keygen_seed.size());
          Botan::McEliece_PrivateKey mce_priv(rng, keygen_n, keygen_t);
 
-         Test::Result result("McEliece keygen");
-
          result.test_eq("public key fingerprint", hash_bytes(mce_priv.public_key_bits()), fprint_pub);
          result.test_eq("private key fingerprint", hash_bytes(mce_priv.private_key_bits()), fprint_priv);
 
@@ -85,6 +94,7 @@ class McEliece_Keygen_Encrypt_Test : public Text_Based_Test
             {
             }
 
+         result.end_timer();
          return result;
          }
 
@@ -147,8 +157,14 @@ class McEliece_Tests : public Test
 
          for(size_t i = 0; i < sizeof(param_sets)/sizeof(param_sets[0]); ++i)
             {
+            if(Test::run_long_tests() == false && param_sets[i].code_length >= 2048)
+               continue;
+
             for(size_t t = param_sets[i].t_min; t <= param_sets[i].t_max; ++t)
                {
+               Test::Result result("McEliece keygen");
+               result.start_timer();
+
                Botan::McEliece_PrivateKey sk1(Test::rng(), param_sets[i].code_length, t);
                const Botan::McEliece_PublicKey& pk1 = sk1;
 
@@ -158,13 +174,12 @@ class McEliece_Tests : public Test
                Botan::McEliece_PublicKey pk(pk_enc);
                Botan::McEliece_PrivateKey sk(sk_enc);
 
-               Test::Result result("McEliece keygen");
-
                result.test_eq("decoded public key equals original", fingerprint(pk1), fingerprint(pk));
-
                result.test_eq("decoded private key equals original", fingerprint(sk1), fingerprint(sk));
-
                result.test_eq("key validation passes", sk.check_key(Test::rng(), false), true);
+               result.end_timer();
+
+               result.end_timer();
 
                results.push_back(result);
 
@@ -184,11 +199,13 @@ class McEliece_Tests : public Test
                             const Botan::McEliece_PublicKey& pk)
          {
          Test::Result result("McEliece KEM");
+         result.start_timer();
 
          Botan::PK_KEM_Encryptor enc_op(pk, Test::rng(), "KDF2(SHA-256)");
          Botan::PK_KEM_Decryptor dec_op(sk, Test::rng(), "KDF2(SHA-256)");
 
-         for(size_t i = 0; i <= Test::soak_level(); i++)
+         const size_t trials = (Test::run_long_tests() ? 30 : 10);
+         for(size_t i = 0; i < trials; i++)
             {
             Botan::secure_vector<uint8_t> salt = Test::rng().random_vec(i);
 
@@ -199,6 +216,7 @@ class McEliece_Tests : public Test
 
             result.test_eq("same key", shared_key, shared_key2);
             }
+         result.end_timer();
          return result;
          }
 
@@ -207,8 +225,9 @@ class McEliece_Tests : public Test
                                const Botan::McEliece_PublicKey& pk)
          {
          Test::Result result("McEliece IES");
+         result.start_timer();
 
-         for(size_t i = 0; i <= Test::soak_level(); ++i)
+         for(size_t i = 0; i <= 10; ++i)
             {
             uint8_t ad[8];
             Botan::store_be(static_cast<uint64_t>(i), ad);
@@ -243,6 +262,7 @@ class McEliece_Tests : public Test
                }
             }
 
+         result.end_timer();
          return result;
          }
 #endif
diff --git a/src/tests/test_passhash.cpp b/src/tests/test_passhash.cpp
index bfd17a443..42528f51b 100644
--- a/src/tests/test_passhash.cpp
+++ b/src/tests/test_passhash.cpp
@@ -36,7 +36,7 @@ class Bcrypt_Tests : public Text_Based_Test
          Test::Result result("bcrypt");
          result.test_eq("correct hash accepted", Botan::check_bcrypt(password, passhash), true);
 
-         const size_t max_level = 1 + std::min<size_t>(Test::soak_level() / 2, 10);
+         const size_t max_level = (Test::run_long_tests() ? 18 : 11);
 
          for(size_t level = 1; level <= max_level; ++level)
             {
@@ -80,7 +80,7 @@ class Passhash9_Tests : public Text_Based_Test
                }
             }
 
-         const size_t max_level = 1 + std::min<size_t>(Test::soak_level() / 2, 10);
+         const size_t max_level = (Test::run_long_tests() ? 16 : 8);
 
          for(size_t level = 1; level <= max_level; ++level)
             {
diff --git a/src/tests/test_pubkey.cpp b/src/tests/test_pubkey.cpp
index 86d04169e..712fa8df4 100644
--- a/src/tests/test_pubkey.cpp
+++ b/src/tests/test_pubkey.cpp
@@ -33,10 +33,12 @@ void check_invalid_signatures(Test::Result& result,
                               const std::vector<uint8_t>& message,
                               const std::vector<uint8_t>& signature)
    {
+   const size_t tests_to_run = (Test::run_long_tests() ? 20 : 5);
+
    const std::vector<uint8_t> zero_sig(signature.size());
    result.test_eq("all zero signature invalid", verifier.verify_message(message, zero_sig), false);
 
-   for(size_t i = 0; i < Test::soak_level(); ++i)
+   for(size_t i = 0; i < tests_to_run; ++i)
       {
       const std::vector<uint8_t> bad_sig = Test::mutate_vec(signature);
 
@@ -53,9 +55,11 @@ void check_invalid_ciphertexts(Test::Result& result,
                                const std::vector<uint8_t>& plaintext,
                                const std::vector<uint8_t>& ciphertext)
    {
+   const size_t tests_to_run = (Test::run_long_tests() ? 20 : 5);
+
    size_t ciphertext_accepted = 0, ciphertext_rejected = 0;
 
-   for(size_t i = 0; i < Test::soak_level(); ++i)
+   for(size_t i = 0; i < tests_to_run; ++i)
       {
       const std::vector<uint8_t> bad_ctext = Test::mutate_vec(ciphertext);
 
diff --git a/src/tests/test_x509_path.cpp b/src/tests/test_x509_path.cpp
index 63a4a9070..e897d3e01 100644
--- a/src/tests/test_x509_path.cpp
+++ b/src/tests/test_x509_path.cpp
@@ -76,6 +76,7 @@ class X509test_Path_Validation_Tests : public Test
          for(auto i = expected.begin(); i != expected.end(); ++i)
             {
             Test::Result result("X509test path validation");
+            result.start_timer();
             const std::string filename = i->first;
             const std::string expected_result = i->second;
 
@@ -94,6 +95,7 @@ class X509test_Path_Validation_Tests : public Test
                path_result = Botan::Path_Validation_Result(Botan::Certificate_Status_Code::CANNOT_ESTABLISH_TRUST);
 
             result.test_eq("test " + filename, path_result.result_string(), expected_result);
+            result.end_timer();
             results.push_back(result);
             }
 
@@ -171,6 +173,7 @@ std::vector<Test::Result> NIST_Path_Validation_Tests::run()
       const std::string test_dir = nist_test_dir + "/" + test_name;
 
       Test::Result result("NIST path validation");
+      result.start_timer();
 
       const std::vector<std::string> all_files = Botan::get_files_recursive(test_dir);
 
@@ -213,6 +216,7 @@ std::vector<Test::Result> NIST_Path_Validation_Tests::run()
                      validation_result.result_string(),
                      expected_result);
 
+      result.end_timer();
       results.push_back(result);
       }
 
diff --git a/src/tests/test_xmss.cpp b/src/tests/test_xmss.cpp
index 73b23483e..da58057e2 100644
--- a/src/tests/test_xmss.cpp
+++ b/src/tests/test_xmss.cpp
@@ -27,6 +27,22 @@ class XMSS_Signature_Tests : public PK_Signature_Generation_Test
          "Params,Msg,PrivateKey,Signature")
          {}
 
+      bool skip_this_test(const std::string&,
+                          const VarMap& vars) override
+         {
+         if(Test::run_long_tests() == false)
+            {
+            const std::string params = get_req_str(vars, "Params");
+
+            if(params == "SHAKE128_W16_H10")
+               return false;
+
+            return true;
+            }
+
+         return false;
+         }
+
       std::string default_padding(const VarMap& vars) const override
          {
          return get_req_str(vars, "Params");
diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp
index 81da95736..a4d81135a 100644
--- a/src/tests/tests.cpp
+++ b/src/tests/tests.cpp
@@ -337,9 +337,8 @@ bool Test::Result::test_rc(const std::string& func, int expected, int rc)
    return test_success();
    }
 
-namespace {
-
-std::string format_time(uint64_t ns)
+//static
+std::string Test::format_time(uint64_t ns)
    {
    std::ostringstream o;
 
@@ -355,8 +354,6 @@ std::string format_time(uint64_t ns)
    return o.str();
    }
 
-}
-
 std::string Test::Result::result_string(bool verbose) const
    {
    if(tests_run() == 0 && !verbose)
@@ -475,34 +472,28 @@ std::vector<Test::Result> Test::run_test(const std::string& test_name, bool fail
 // static member variables of Test
 Botan::RandomNumberGenerator* Test::m_test_rng = nullptr;
 std::string Test::m_data_dir;
-size_t Test::m_soak_level = 0;
 bool Test::m_log_success = false;
 bool Test::m_run_online_tests = false;
+bool Test::m_run_long_tests = false;
 std::string Test::m_pkcs11_lib;
 
 //static
-void Test::setup_tests(size_t soak,
-                       bool log_success,
+void Test::setup_tests(bool log_success,
                        bool run_online,
+                       bool run_long,
                        const std::string& data_dir,
                        const std::string& pkcs11_lib,
                        Botan::RandomNumberGenerator* rng)
    {
    m_data_dir = data_dir;
-   m_soak_level = soak;
    m_log_success = log_success;
    m_run_online_tests = run_online;
+   m_run_long_tests = run_long;
    m_test_rng = rng;
    m_pkcs11_lib = pkcs11_lib;
    }
 
 //static
-size_t Test::soak_level()
-   {
-   return m_soak_level;
-   }
-
-//static
 std::string Test::data_file(const std::string& what)
    {
    return Test::data_dir() + "/" + what;
@@ -527,6 +518,12 @@ bool Test::run_online_tests()
    }
 
 //static
+bool Test::run_long_tests()
+   {
+   return m_run_long_tests;
+   }
+
+//static
 std::string Test::pkcs11_lib()
    {
    return m_pkcs11_lib;
@@ -787,6 +784,12 @@ parse_cpuid_bits(const std::vector<std::string>& tok)
 
 }
 
+bool Text_Based_Test::skip_this_test(const std::string& /*header*/,
+                                     const VarMap& /*vars*/)
+   {
+   return false;
+   }
+
 std::vector<Test::Result> Text_Based_Test::run()
    {
    std::vector<Test::Result> results;
@@ -816,7 +819,9 @@ std::vector<Test::Result> Text_Based_Test::run()
          continue;
          }
       else if(line[0] == '#')
+         {
          throw Test_Error("Unknown test pragma '" + line + "' in " + m_cur_src_name);
+         }
 
       if(line[0] == '[' && line[line.size()-1] == ']')
          {
@@ -850,6 +855,11 @@ std::vector<Test::Result> Text_Based_Test::run()
          {
          try
             {
+            if(skip_this_test(header, vars))
+               {
+               continue;
+               }
+
             ++test_cnt;
 
             uint64_t start = Test::timestamp();
diff --git a/src/tests/tests.h b/src/tests/tests.h
index ce6431e35..dd7b3784b 100644
--- a/src/tests/tests.h
+++ b/src/tests/tests.h
@@ -328,6 +328,8 @@ class Test
 
       static std::string data_file(const std::string& what);
 
+      static std::string format_time(uint64_t nanoseconds);
+
       template<typename Alloc>
       static std::vector<uint8_t, Alloc>
       mutate_vec(const std::vector<uint8_t, Alloc>& v,
@@ -356,16 +358,16 @@ class Test
          return r;
          }
 
-      static void setup_tests(size_t soak,
-                              bool log_succcss,
+      static void setup_tests(bool log_succcss,
                               bool run_online_tests,
+                              bool run_long_tests,
                               const std::string& data_dir,
                               const std::string& pkcs11_lib,
                               Botan::RandomNumberGenerator* rng);
 
-      static size_t soak_level();
       static bool log_success();
       static bool run_online_tests();
+      static bool run_long_tests();
       static std::string pkcs11_lib();
 
       static const std::string& data_dir();
@@ -377,8 +379,7 @@ class Test
    private:
       static std::string m_data_dir;
       static Botan::RandomNumberGenerator* m_test_rng;
-      static size_t m_soak_level;
-      static bool m_log_success, m_run_online_tests;
+      static bool m_log_success, m_run_online_tests, m_run_long_tests;
       static std::string m_pkcs11_lib;
    };
 
@@ -421,6 +422,10 @@ class Text_Based_Test : public Test
       virtual Test::Result run_one_test(const std::string& header,
                                         const VarMap& vars) = 0;
 
+      // Called before run_one_test
+      virtual bool skip_this_test(const std::string& header,
+                                  const VarMap& vars);
+
       virtual std::vector<Test::Result> run_final_tests() { return std::vector<Test::Result>(); }
 
       bool get_req_bool(const VarMap& vars, const std::string& key) const;
diff --git a/src/tests/unit_ecc.cpp b/src/tests/unit_ecc.cpp
index 5961ca5e3..dca93f57f 100644
--- a/src/tests/unit_ecc.cpp
+++ b/src/tests/unit_ecc.cpp
@@ -134,6 +134,8 @@ std::vector<Test::Result> ECC_Randomized_Tests::run()
       {
       Test::Result result("ECC randomized " + group_name);
 
+      result.start_timer();
+
       Botan::EC_Group group(group_name);
 
       const Botan::PointGFp& base_point = group.get_base_point();
@@ -145,7 +147,8 @@ std::vector<Test::Result> ECC_Randomized_Tests::run()
 
       try
          {
-         for(size_t i = 0; i <= Test::soak_level(); ++i)
+         const size_t trials = (Test::run_long_tests() ? 10 : 3);
+         for(size_t i = 0; i < trials; ++i)
             {
             const size_t h = 1 + (Test::rng().next_byte() % 8);
             Botan::Blinded_Point_Multiply blind(base_point, group_order, h);
@@ -171,8 +174,6 @@ std::vector<Test::Result> ECC_Randomized_Tests::run()
             result.test_eq("p on the curve", P.on_the_curve(), true);
             result.test_eq("q on the curve", Q.on_the_curve(), true);
             result.test_eq("r on the curve", R.on_the_curve(), true);
-            result.test_eq("a1 on the curve", A1.on_the_curve(), true);
-            result.test_eq("a2 on the curve", A2.on_the_curve(), true);
 
             result.test_eq("P1", P1, P);
             result.test_eq("Q1", Q1, Q);
@@ -183,6 +184,7 @@ std::vector<Test::Result> ECC_Randomized_Tests::run()
          {
          result.test_failure(group_name, e.what());
          }
+      result.end_timer();
       results.push_back(result);
       }
 
@@ -220,8 +222,11 @@ class NIST_Curve_Reduction_Tests : public Test
          Botan::secure_vector<Botan::word> ws;
 
          Test::Result result("NIST " + prime_name + " reduction");
+         result.start_timer();
+
+         const size_t trials = (Test::run_long_tests() ? 128 : 16);
 
-         for(size_t i = 0; i <= 10 * Test::soak_level(); ++i)
+         for(size_t i = 0; i <= trials; ++i)
             {
             const Botan::BigInt x = test_integer(Test::rng(), 2*p_bits, p2);
 
@@ -239,6 +244,8 @@ class NIST_Curve_Reduction_Tests : public Test
                }
             }
 
+         result.end_timer();
+
          return result;
          }
    };