Update and consolidate the test framework.

The tests previously had used 4 to 6 different schemes internally (the vec file
reader framework, Catch, the old InSiTo Boost.Test tests, the PK/BigInt tests
which escaped the rewrite in 1.11.7, plus a number of one-offs). Converge on a
design that works everywhere, and update all the things.

Fix also a few bugs found by the test changes: SHA-512-256 name incorrect,
OpenSSL RC4 name incorrect, signature of FFI function botan_pubkey_destroy
was wrong.

1 files changed, 371 insertions, 0 deletions

diff --git a/src/tests/test_x509_path.cpp b/src/tests/test_x509_path.cpp
new file mode 100644
index 000000000..62051d021
--- /dev/null
+++ b/src/tests/test_x509_path.cpp
@@ -0,0 +1,371 @@
+/*
+* (C) 2006,2011,2012,2014,2015 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include "tests.h"
+
+#if defined(BOTAN_HAS_X509_CERTIFICATES)
+  #include <botan/x509path.h>
+  #include <botan/internal/filesystem.h>
+#endif
+
+#include <algorithm>
+#include <fstream>
+#include <iomanip>
+#include <string>
+#include <vector>
+#include <map>
+#include <cstdlib>
+
+namespace Botan_Tests {
+
+namespace {
+
+#if defined(BOTAN_HAS_X509_CERTIFICATES)
+
+class X509test_Path_Validation_Tests : public Test
+   {
+   public:
+      std::vector<Test::Result> run() override
+         {
+         std::vector<Test::Result> results;
+
+         // Test certs generated by https://github.com/yymax/x509test
+         const std::string test_dir = "src/tests/data/x509test";
+
+         std::map<std::string, std::string> expected = read_results(test_dir + "/expected.txt");
+
+         const Botan::Path_Validation_Restrictions default_restrictions;
+
+         Botan::X509_Certificate root(test_dir + "/root.pem");
+         Botan::Certificate_Store_In_Memory trusted;
+         trusted.add_certificate(root);
+
+         for(auto i = expected.begin(); i != expected.end(); ++i)
+            {
+            Test::Result result("X509test path validation");
+            const std::string fsname = i->first;
+            const std::string expected = i->second;
+
+            std::vector<Botan::X509_Certificate> certs = load_cert_file(test_dir + "/" + fsname);
+
+            if(certs.empty())
+               throw std::runtime_error("Failed to read certs from " + fsname);
+
+            Botan::Path_Validation_Result path_result = Botan::x509_path_validate(
+               certs, default_restrictions, trusted,
+               "www.tls.test", Botan::Usage_Type::TLS_SERVER_AUTH);
+
+            if(path_result.successful_validation() && path_result.trust_root() != root)
+               path_result = Botan::Path_Validation_Result(Botan::Certificate_Status_Code::CANNOT_ESTABLISH_TRUST);
+
+            result.test_eq("validation result", path_result.result_string(), expected);
+            results.push_back(result);
+            }
+
+         return results;
+         }
+
+   private:
+
+      std::vector<Botan::X509_Certificate> load_cert_file(const std::string& filename)
+         {
+         Botan::DataSource_Stream in(filename);
+
+         std::vector<Botan::X509_Certificate> certs;
+         while(!in.end_of_data())
+            {
+            try {
+              certs.emplace_back(in);
+            }
+            catch(Botan::Decoding_Error&) {}
+            }
+
+         return certs;
+         }
+
+      std::map<std::string, std::string> read_results(const std::string& results_file)
+         {
+         std::ifstream in(results_file);
+         if(!in.good())
+            throw std::runtime_error("Failed reading " + results_file);
+
+         std::map<std::string, std::string> m;
+         std::string line;
+         while(in.good())
+            {
+            std::getline(in, line);
+            if(line == "")
+               continue;
+            if(line[0] == '#')
+               continue;
+
+            std::vector<std::string> parts = Botan::split_on(line, ':');
+
+            if(parts.size() != 2)
+               throw std::runtime_error("Invalid line " + line);
+
+            m[parts[0]] = parts[1];
+            }
+
+         return m;
+         }
+   };
+
+BOTAN_REGISTER_TEST("x509_path_x509test", X509test_Path_Validation_Tests);
+
+class NIST_Path_Validation_Tests : public Test
+   {
+   public:
+      std::vector<Test::Result> run() override;
+
+   private:
+      std::map<size_t, Botan::Path_Validation_Result::Code> get_expected();
+   };
+
+/*
+  The expected results are essentially the error codes that best coorespond
+  to the problem described in the testing documentation.
+
+  There are a few cases where the tests say there should or should not be an
+  error, and I disagree. A few of the tests have test results different from
+  what they "should" be: these changes are marked as such, and have comments
+  explaining the problem at hand.
+*/
+std::map<size_t, Botan::Path_Validation_Result::Code> NIST_Path_Validation_Tests::get_expected()
+   {
+   using namespace Botan;
+
+   std::map<size_t, Path_Validation_Result::Code> expected_results;
+
+
+   // TODO read from a file
+   expected_results[1] = Certificate_Status_Code::VERIFIED;
+   expected_results[2] = Certificate_Status_Code::SIGNATURE_ERROR;
+   expected_results[3] = Certificate_Status_Code::SIGNATURE_ERROR;
+   expected_results[4] = Certificate_Status_Code::VERIFIED;
+   expected_results[5] = Certificate_Status_Code::CERT_NOT_YET_VALID;
+   expected_results[6] = Certificate_Status_Code::CERT_NOT_YET_VALID;
+   expected_results[7] = Certificate_Status_Code::VERIFIED;
+   expected_results[8] = Certificate_Status_Code::CERT_NOT_YET_VALID;
+   expected_results[9] = Certificate_Status_Code::CERT_HAS_EXPIRED;
+   expected_results[10] = Certificate_Status_Code::CERT_HAS_EXPIRED;
+   expected_results[11] = Certificate_Status_Code::CERT_HAS_EXPIRED;
+   expected_results[12] = Certificate_Status_Code::VERIFIED;
+   expected_results[13] = Certificate_Status_Code::CERT_ISSUER_NOT_FOUND;
+
+   expected_results[14] = Certificate_Status_Code::CERT_ISSUER_NOT_FOUND;
+   expected_results[15] = Certificate_Status_Code::VERIFIED;
+   expected_results[16] = Certificate_Status_Code::VERIFIED;
+   expected_results[17] = Certificate_Status_Code::VERIFIED;
+   expected_results[18] = Certificate_Status_Code::VERIFIED;
+
+   expected_results[19] = Certificate_Status_Code::NO_REVOCATION_DATA;
+   expected_results[20] = Certificate_Status_Code::CERT_IS_REVOKED;
+   expected_results[21] = Certificate_Status_Code::CERT_IS_REVOKED;
+
+   expected_results[22] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
+   expected_results[23] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
+   expected_results[24] = Certificate_Status_Code::VERIFIED;
+   expected_results[25] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
+   expected_results[26] = Certificate_Status_Code::VERIFIED;
+   expected_results[27] = Certificate_Status_Code::VERIFIED;
+   expected_results[28] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
+   expected_results[29] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
+   expected_results[30] = Certificate_Status_Code::VERIFIED;
+
+   expected_results[31] = Certificate_Status_Code::CA_CERT_NOT_FOR_CRL_ISSUER;
+   expected_results[32] = Certificate_Status_Code::CA_CERT_NOT_FOR_CRL_ISSUER;
+   expected_results[33] = Certificate_Status_Code::VERIFIED;
+
+   /*
+    Policy tests: a little trickier because there are other inputs
+    which affect the result.
+
+    In the case of the tests currently in the suite, the default
+    method (with acceptable policy being "any-policy" and with no
+    explicit policy required), will almost always result in a verified
+    status. This is not particularly helpful. So, we should do several
+    different tests for each test set:
+
+       1) With the user policy as any-policy and no explicit policy
+       2) With the user policy as any-policy and an explicit policy required
+       3) With the user policy as test-policy-1 (2.16.840.1.101.3.1.48.1) and
+          an explicit policy required
+       4) With the user policy as either test-policy-1 or test-policy-2 and an
+          explicit policy required
+
+     This provides reasonably good coverage of the possible outcomes.
+   */
+
+   expected_results[34] = Certificate_Status_Code::VERIFIED;
+   expected_results[35] = Certificate_Status_Code::VERIFIED;
+   expected_results[36] = Certificate_Status_Code::VERIFIED;
+   expected_results[37] = Certificate_Status_Code::VERIFIED;
+   expected_results[38] = Certificate_Status_Code::VERIFIED;
+   expected_results[39] = Certificate_Status_Code::VERIFIED;
+   expected_results[40] = Certificate_Status_Code::VERIFIED;
+   expected_results[41] = Certificate_Status_Code::VERIFIED;
+   expected_results[42] = Certificate_Status_Code::VERIFIED;
+   expected_results[43] = Certificate_Status_Code::VERIFIED;
+   expected_results[44] = Certificate_Status_Code::VERIFIED;
+
+   //expected_results[45] = Certificate_Status_Code::EXPLICIT_POLICY_REQUIRED;
+   //expected_results[46] = Certificate_Status_Code::ACCEPT;
+   //expected_results[47] = Certificate_Status_Code::EXPLICIT_POLICY_REQUIRED;
+
+   expected_results[48] = Certificate_Status_Code::VERIFIED;
+   expected_results[49] = Certificate_Status_Code::VERIFIED;
+   expected_results[50] = Certificate_Status_Code::VERIFIED;
+   expected_results[51] = Certificate_Status_Code::VERIFIED;
+   expected_results[52] = Certificate_Status_Code::VERIFIED;
+   expected_results[53] = Certificate_Status_Code::VERIFIED;
+
+   expected_results[54] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
+   expected_results[55] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
+   expected_results[56] = Certificate_Status_Code::VERIFIED;
+   expected_results[57] = Certificate_Status_Code::VERIFIED;
+   expected_results[58] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
+   expected_results[59] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
+   expected_results[60] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
+   expected_results[61] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
+   expected_results[62] = Certificate_Status_Code::VERIFIED;
+   expected_results[63] = Certificate_Status_Code::VERIFIED;
+
+   expected_results[64] = Certificate_Status_Code::CRL_BAD_SIGNATURE;
+
+   expected_results[65] = Certificate_Status_Code::NO_REVOCATION_DATA;
+   expected_results[66] = Certificate_Status_Code::NO_REVOCATION_DATA;
+
+   expected_results[67] = Certificate_Status_Code::VERIFIED;
+
+   expected_results[68] = Certificate_Status_Code::CERT_IS_REVOKED;
+   expected_results[69] = Certificate_Status_Code::CERT_IS_REVOKED;
+   expected_results[70] = Certificate_Status_Code::CERT_IS_REVOKED;
+   expected_results[71] = Certificate_Status_Code::CERT_IS_REVOKED;
+   expected_results[72] = Certificate_Status_Code::CRL_HAS_EXPIRED;
+   expected_results[73] = Certificate_Status_Code::CRL_HAS_EXPIRED;
+   expected_results[74] = Certificate_Status_Code::VERIFIED;
+
+   /* These tests use weird CRL extensions which aren't supported yet */
+   //expected_results[75] = ;
+   //expected_results[76] = ;
+
+   return expected_results;
+   }
+
+std::vector<Test::Result> NIST_Path_Validation_Tests::run()
+   {
+   std::vector<Test::Result> results;
+
+   /**
+   * Code to run the X.509v3 processing tests described in "Conformance
+   *  Testing of Relying Party Client Certificate Path Proccessing Logic",
+   *  which is available on NIST's web site.
+   *
+   * Known Failures/Problems:
+   *  - Policy extensions are not implemented, so we skip tests #34-#53.
+   *  - Tests #75 and #76 are skipped as they make use of relatively
+   *    obscure CRL extensions which are not supported.
+   */
+   const std::string root_test_dir = "src/tests/data/nist_x509/";
+
+   try
+      {
+      // Do nothing, just test filesystem access
+      Botan::get_files_recursive(root_test_dir);
+      }
+   catch(Botan::No_Filesystem_Access)
+      {
+      Test::Result result("NIST path validation");
+      result.test_note("Skipping due to missing filesystem access");
+      results.push_back(result);
+      return results;
+      }
+
+   const size_t total_tests = 76;
+   std::map<size_t, Botan::Path_Validation_Result::Code> expected_results = get_expected();
+
+   for(size_t test_no = 1; test_no <= total_tests; ++test_no)
+      {
+      try
+         {
+         Test::Result result("NIST path validation");
+         const std::string test_dir = root_test_dir + "/test" + (test_no <= 9 ? "0" : "") + std::to_string(test_no);
+
+         const std::vector<std::string> all_files = Botan::get_files_recursive(test_dir);
+         if (all_files.empty())
+            {
+            result.test_failure("No test files found in " + test_dir);
+            continue;
+            }
+
+         std::vector<std::string> certs, crls;
+         std::string root_cert, to_verify;
+
+         for(const auto &current : all_files)
+            {
+            if(current.find("int") != std::string::npos && current.find(".crt") != std::string::npos)
+               certs.push_back(current);
+            else if(current.find("root.crt") != std::string::npos)
+               root_cert = current;
+            else if(current.find("end.crt") != std::string::npos)
+               to_verify = current;
+            else if(current.find(".crl") != std::string::npos)
+               crls.push_back(current);
+            }
+
+         if(expected_results.find(test_no) == expected_results.end())
+            {
+            result.test_note("Skipping test");
+            continue;
+            }
+
+         Botan::Certificate_Store_In_Memory store;
+
+         store.add_certificate(Botan::X509_Certificate(root_cert));
+
+         Botan::X509_Certificate end_user(to_verify);
+
+         for(size_t i = 0; i != certs.size(); i++)
+            store.add_certificate(Botan::X509_Certificate(certs[i]));
+
+         for(size_t i = 0; i != crls.size(); i++)
+            {
+            Botan::DataSource_Stream in(crls[i], true);
+            Botan::X509_CRL crl(in);
+            store.add_crl(crl);
+         }
+
+         Botan::Path_Validation_Restrictions restrictions(true);
+
+         Botan::Path_Validation_Result validation_result =
+            Botan::x509_path_validate(end_user,
+                                      restrictions,
+                                      store);
+
+         auto expected = expected_results[test_no];
+
+         result.test_eq("path validation result",
+                        validation_result.result_string(),
+                        Botan::Path_Validation_Result::status_string(expected));
+
+         results.push_back(result);
+         }
+      catch(std::exception& e)
+         {
+         results.push_back(Test::Result::Failure("NIST X509 " + std::to_string(test_no), e.what()));
+         }
+      }
+   return results;
+   }
+
+BOTAN_REGISTER_TEST("x509_path_nist", NIST_Path_Validation_Tests);
+
+#endif
+
+}
+
+}