Merge GH #955 Add ability to limit tests to single provider

9 files changed, 96 insertions, 24 deletions

diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index cb105db18..c90aaec7b 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -35,7 +35,7 @@ namespace {
 class Test_Runner : public Botan_CLI::Command
    {
    public:
-      Test_Runner() : Command("test --threads=0 --run-long-tests --run-online-tests --test-runs=1 --drbg-seed= --data-dir= --pkcs11-lib= --log-success *suites") {}
+      Test_Runner() : Command("test --threads=0 --run-long-tests --run-online-tests --test-runs=1 --drbg-seed= --data-dir= --pkcs11-lib= --provider= --log-success *suites") {}
 
       std::string help_text() const override
          {
@@ -79,6 +79,7 @@ class Test_Runner : public Botan_CLI::Command
          const bool run_long_tests = flag_set("run-long-tests");
          const std::string data_dir = get_arg_or("data-dir", "src/tests/data");
          const std::string pkcs11_lib = get_arg("pkcs11-lib");
+         const std::string provider = get_arg("provider");
          const size_t runs = get_arg_sz("test-runs");
 
          std::vector<std::string> req = get_arg_list("suites");
@@ -146,6 +147,13 @@ class Test_Runner : public Botan_CLI::Command
             output() << " pkcs11 library:" << pkcs11_lib;
             }
 
+         Botan_Tests::Provider_Filter pf;
+         if(!provider.empty())
+            {
+            output() << " provider:" << provider;
+            pf.set(provider);
+            }
+
          std::unique_ptr<Botan::RandomNumberGenerator> rng;
 
 #if defined(BOTAN_HAS_HMAC_DRBG) && defined(BOTAN_HAS_SHA2_64)
@@ -186,7 +194,7 @@ class Test_Runner : public Botan_CLI::Command
          output() << "\n";
 
          Botan_Tests::Test::setup_tests(log_success, run_online_tests, run_long_tests,
-                                        data_dir, pkcs11_lib, rng.get());
+                                        data_dir, pkcs11_lib, pf, rng.get());
 
          for(size_t i = 0; i != runs; ++i)
             {
diff --git a/src/tests/test_block.cpp b/src/tests/test_block.cpp
index 20d887794..62f32c21c 100644
--- a/src/tests/test_block.cpp
+++ b/src/tests/test_block.cpp
@@ -14,6 +14,11 @@ class Block_Cipher_Tests : public Text_Based_Test
    public:
       Block_Cipher_Tests() : Text_Based_Test("block", "Key,In,Out") {}
 
+      std::vector<std::string> possible_providers(const std::string& algo) override
+         {
+         return provider_filter(Botan::BlockCipher::providers(algo));
+         }
+
       Test::Result run_one_test(const std::string& algo, const VarMap& vars) override
          {
          const std::vector<uint8_t> key      = get_req_bin(vars, "Key");
@@ -22,7 +27,7 @@ class Block_Cipher_Tests : public Text_Based_Test
 
          Test::Result result(algo);
 
-         const std::vector<std::string> providers = Botan::BlockCipher::providers(algo);
+         const std::vector<std::string> providers = possible_providers(algo);
 
          if(providers.empty())
             {
diff --git a/src/tests/test_hash.cpp b/src/tests/test_hash.cpp
index fd255131b..6584a20d1 100644
--- a/src/tests/test_hash.cpp
+++ b/src/tests/test_hash.cpp
@@ -17,6 +17,11 @@ class Hash_Function_Tests : public Text_Based_Test
    public:
       Hash_Function_Tests() : Text_Based_Test("hash", "In,Out") {}
 
+      std::vector<std::string> possible_providers(const std::string& algo) override
+         {
+         return provider_filter(Botan::HashFunction::providers(algo));
+         }
+
       Test::Result run_one_test(const std::string& algo, const VarMap& vars) override
          {
          const std::vector<uint8_t> input    = get_req_bin(vars, "In");
@@ -24,7 +29,7 @@ class Hash_Function_Tests : public Text_Based_Test
 
          Test::Result result(algo);
 
-         const std::vector<std::string> providers = Botan::HashFunction::providers(algo);
+         const std::vector<std::string> providers = possible_providers(algo);
 
          if(providers.empty())
             {
diff --git a/src/tests/test_mac.cpp b/src/tests/test_mac.cpp
index 8fb3b38c2..2e12ee6ed 100644
--- a/src/tests/test_mac.cpp
+++ b/src/tests/test_mac.cpp
@@ -22,6 +22,11 @@ class Message_Auth_Tests : public Text_Based_Test
    public:
       Message_Auth_Tests() : Text_Based_Test("mac", "Key,In,Out", "IV") {}
 
+      std::vector<std::string> possible_providers(const std::string& algo) override
+         {
+         return provider_filter(Botan::MessageAuthenticationCode::providers(algo));
+         }
+
       Test::Result run_one_test(const std::string& algo, const VarMap& vars) override
          {
          const std::vector<uint8_t> key      = get_req_bin(vars, "Key");
@@ -31,7 +36,7 @@ class Message_Auth_Tests : public Text_Based_Test
 
          Test::Result result(algo);
 
-         const std::vector<std::string> providers = Botan::MessageAuthenticationCode::providers(algo);
+         const std::vector<std::string> providers = possible_providers(algo);
 
          if(providers.empty())
             {
diff --git a/src/tests/test_pubkey.cpp b/src/tests/test_pubkey.cpp
index 9a8c5d2e7..d392437c9 100644
--- a/src/tests/test_pubkey.cpp
+++ b/src/tests/test_pubkey.cpp
@@ -19,15 +19,6 @@
 
 namespace Botan_Tests {
 
-namespace {
-
-std::vector<std::string> possible_pk_providers()
-   {
-   return { "base", "openssl", "tpm" };
-   }
-
-}
-
 void check_invalid_signatures(Test::Result& result,
                               Botan::PK_Verifier& verifier,
                               const std::vector<uint8_t>& message,
@@ -91,6 +82,11 @@ void check_invalid_ciphertexts(Test::Result& result,
                     " invalid ciphertexts, rejected " + std::to_string(ciphertext_rejected));
    }
 
+std::vector<std::string> PK_Test::possible_providers(const std::string&)
+   {
+   return Test::provider_filter({ "base", "openssl", "tpm" });
+   }
+
 Test::Result
 PK_Signature_Generation_Test::run_one_test(const std::string&, const VarMap& vars)
    {
@@ -115,7 +111,7 @@ PK_Signature_Generation_Test::run_one_test(const std::string&, const VarMap& var
 
    std::vector<std::unique_ptr<Botan::PK_Verifier>> verifiers;
 
-   for(std::string verify_provider : possible_pk_providers())
+   for(std::string verify_provider : possible_providers(algo_name()))
       {
       std::unique_ptr<Botan::PK_Verifier> verifier;
 
@@ -135,7 +131,7 @@ PK_Signature_Generation_Test::run_one_test(const std::string&, const VarMap& var
       verifiers.push_back(std::move(verifier));
       }
 
-   for(auto&& sign_provider : possible_pk_providers())
+   for(auto&& sign_provider : possible_providers(algo_name()))
       {
       std::unique_ptr<Botan::RandomNumberGenerator> rng;
       if(vars.count("Nonce"))
@@ -189,7 +185,7 @@ PK_Signature_Verification_Test::run_one_test(const std::string&, const VarMap& v
 
    Test::Result result(algo_name() + "/" + padding + " signature verification");
 
-   for(auto&& verify_provider : possible_pk_providers())
+   for(auto&& verify_provider : possible_providers(algo_name()))
       {
       std::unique_ptr<Botan::PK_Verifier> verifier;
 
@@ -219,7 +215,7 @@ PK_Signature_NonVerification_Test::run_one_test(const std::string&, const VarMap
 
    Test::Result result(algo_name() + "/" + padding + " verify invalid signature");
 
-   for(auto&& verify_provider : possible_pk_providers())
+   for(auto&& verify_provider : possible_providers(algo_name()))
       {
       std::unique_ptr<Botan::PK_Verifier> verifier;
 
@@ -255,7 +251,7 @@ PK_Encryption_Decryption_Test::run_one_test(const std::string&, const VarMap& va
 
    std::vector<std::unique_ptr<Botan::PK_Decryptor>> decryptors;
 
-   for(auto&& dec_provider : possible_pk_providers())
+   for(auto&& dec_provider : possible_providers(algo_name()))
       {
       std::unique_ptr<Botan::PK_Decryptor> decryptor;
 
@@ -273,7 +269,7 @@ PK_Encryption_Decryption_Test::run_one_test(const std::string&, const VarMap& va
       }
 
 
-   for(auto&& enc_provider : possible_pk_providers())
+   for(auto&& enc_provider : possible_providers(algo_name()))
       {
       std::unique_ptr<Botan::PK_Encryptor> encryptor;
 
@@ -388,7 +384,7 @@ Test::Result PK_Key_Agreement_Test::run_one_test(const std::string& header, cons
 
    const size_t key_len = get_opt_sz(vars, "OutLen", 0);
 
-   for(auto&& provider : possible_pk_providers())
+   for(auto&& provider : possible_providers(algo_name()))
       {
       std::unique_ptr<Botan::PK_Key_Agreement> kas;
 
diff --git a/src/tests/test_pubkey.h b/src/tests/test_pubkey.h
index cbcc38f9d..7709f51c7 100644
--- a/src/tests/test_pubkey.h
+++ b/src/tests/test_pubkey.h
@@ -30,6 +30,9 @@ class PK_Test : public Text_Based_Test
 
       std::string algo_name() const { return m_algo; }
 
+   protected:
+      std::vector<std::string> possible_providers(const std::string&) override;
+
    private:
       std::string m_algo;
    };
diff --git a/src/tests/test_stream.cpp b/src/tests/test_stream.cpp
index 9250f4e2f..c2c7ce901 100644
--- a/src/tests/test_stream.cpp
+++ b/src/tests/test_stream.cpp
@@ -32,7 +32,8 @@ class Stream_Cipher_Tests : public Text_Based_Test
 
          Test::Result result(algo);
 
-         const std::vector<std::string> providers = Botan::StreamCipher::providers(algo);
+         const std::vector<std::string> providers =
+            provider_filter(Botan::StreamCipher::providers(algo));
 
          if(providers.empty())
             {
diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp
index e4a755831..fe97a2a7a 100644
--- a/src/tests/tests.cpp
+++ b/src/tests/tests.cpp
@@ -417,6 +417,22 @@ std::string Test::Result::result_string(bool verbose) const
    return report.str();
    }
 
+std::vector<std::string> Provider_Filter::filter(const std::vector<std::string> &in) const
+   {
+      if(m_provider.empty())
+         {
+         return in;
+         }
+      for(auto&& provider : in)
+         {
+         if(provider == m_provider)
+            {
+            return std::vector<std::string> { provider };
+            }
+         }
+      return std::vector<std::string> {};
+   }
+
 // static Test:: functions
 //static
 std::map<std::string, std::unique_ptr<Test>>& Test::global_registry()
@@ -488,6 +504,7 @@ bool Test::m_log_success = false;
 bool Test::m_run_online_tests = false;
 bool Test::m_run_long_tests = false;
 std::string Test::m_pkcs11_lib;
+Botan_Tests::Provider_Filter Test::m_provider_filter;
 
 //static
 void Test::setup_tests(bool log_success,
@@ -495,6 +512,7 @@ void Test::setup_tests(bool log_success,
                        bool run_long,
                        const std::string& data_dir,
                        const std::string& pkcs11_lib,
+                       const Botan_Tests::Provider_Filter& pf,
                        Botan::RandomNumberGenerator* rng)
    {
    m_data_dir = data_dir;
@@ -503,6 +521,7 @@ void Test::setup_tests(bool log_success,
    m_run_long_tests = run_long;
    m_test_rng = rng;
    m_pkcs11_lib = pkcs11_lib;
+   m_provider_filter = pf;
    }
 
 //static
@@ -542,6 +561,12 @@ std::string Test::pkcs11_lib()
    }
 
 //static
+std::vector<std::string> Test::provider_filter(const std::vector<std::string>& in)
+   {
+   return m_provider_filter.filter(in);
+   }
+
+//static
 Botan::RandomNumberGenerator& Test::rng()
    {
    if(!m_test_rng)
@@ -824,6 +849,11 @@ parse_cpuid_bits(const std::vector<std::string>& tok)
 
 }
 
+std::vector<std::string> Text_Based_Test::possible_providers(const std::string&)
+   {
+   return Test::provider_filter({ "base" });
+   }
+
 bool Text_Based_Test::skip_this_test(const std::string& /*header*/,
                                      const VarMap& /*vars*/)
    {
@@ -895,7 +925,8 @@ std::vector<Test::Result> Text_Based_Test::run()
          {
          try
             {
-            if(skip_this_test(header, vars))
+            if(possible_providers(header).empty() ||
+               skip_this_test(header, vars))
                {
                continue;
                }
@@ -938,6 +969,11 @@ std::vector<Test::Result> Text_Based_Test::run()
          }
       }
 
+   if(results.empty())
+      {
+      return results;
+      }
+
    try
       {
       std::vector<Test::Result> final_tests = run_final_tests();
diff --git a/src/tests/tests.h b/src/tests/tests.h
index f90e0037b..406cff57a 100644
--- a/src/tests/tests.h
+++ b/src/tests/tests.h
@@ -46,6 +46,16 @@ class Test_Error : public Botan::Exception
       explicit Test_Error(const std::string& what) : Exception("Test error", what) {}
    };
 
+class Provider_Filter
+   {
+   public:
+      Provider_Filter() {}
+      void set(const std::string& provider) { m_provider = provider; }
+      std::vector<std::string> filter(const std::vector<std::string> &) const;
+   private:
+      std::string m_provider;
+   };
+
 /*
 * A generic test which returns a set of results when run.
 * The tests may not all have the same type (for example test
@@ -370,12 +380,14 @@ class Test
                               bool run_long_tests,
                               const std::string& data_dir,
                               const std::string& pkcs11_lib,
+                              const Botan_Tests::Provider_Filter& pf,
                               Botan::RandomNumberGenerator* rng);
 
       static bool log_success();
       static bool run_online_tests();
       static bool run_long_tests();
       static std::string pkcs11_lib();
+      static std::vector<std::string> provider_filter(const std::vector<std::string>&);
 
       static const std::string& data_dir();
 
@@ -388,6 +400,7 @@ class Test
       static Botan::RandomNumberGenerator* m_test_rng;
       static bool m_log_success, m_run_online_tests, m_run_long_tests;
       static std::string m_pkcs11_lib;
+      static Botan_Tests::Provider_Filter m_provider_filter;
    };
 
 /*
@@ -428,8 +441,8 @@ class Text_Based_Test : public Test
 
       virtual Test::Result run_one_test(const std::string& header,
                                         const VarMap& vars) = 0;
-
       // Called before run_one_test
+      virtual std::vector<std::string> possible_providers(const std::string&);
       virtual bool skip_this_test(const std::string& header,
                                   const VarMap& vars);