Update NIST X509 tests to handle --data-dir and read expected results from a file

2 files changed, 148 insertions, 223 deletions

diff --git a/src/tests/data/nist_x509/expected.txt b/src/tests/data/nist_x509/expected.txt
new file mode 100644
index 000000000..0b2535fca
--- /dev/null
+++ b/src/tests/data/nist_x509/expected.txt
@@ -0,0 +1,76 @@
+test01:Verified
+test02:Signature error
+test03:Signature error
+test04:Verified
+test05:Certificate is not yet valid
+test06:Certificate is not yet valid
+test07:Verified
+test08:Certificate is not yet valid
+test09:Certificate has expired
+test10:Certificate has expired
+test11:Certificate has expired
+test12:Verified
+test13:Certificate issuer not found
+test14:Certificate issuer not found
+test15:Verified
+test16:Verified
+test17:Verified
+test18:Verified
+test19:No revocation data
+test20:Certificate is revoked
+test21:Certificate is revoked
+test22:CA certificate not allowed to issue certs
+test23:CA certificate not allowed to issue certs
+test24:Verified
+test25:CA certificate not allowed to issue certs
+test26:Verified
+test27:Verified
+test28:CA certificate not allowed to issue certs
+test29:CA certificate not allowed to issue certs
+test30:Verified
+test31:CA certificate not allowed to issue CRLs
+test32:CA certificate not allowed to issue CRLs
+test33:Verified
+test34:Verified
+test35:Verified
+test36:Verified
+test37:Verified
+test38:Verified
+test39:Verified
+test40:Verified
+test41:Verified
+test42:Verified
+test43:Verified
+test44:Verified
+#test45:Explicit policy required
+#test46:
+#test47:Explicit policy required
+test48:Verified
+test49:Verified
+test50:Verified
+test51:Verified
+test52:Verified
+test53:Verified
+test54:Certificate chain too long
+test55:Certificate chain too long
+test56:Verified
+test57:Verified
+test58:Certificate chain too long
+test59:Certificate chain too long
+test60:Certificate chain too long
+test61:Certificate chain too long
+test62:Verified
+test63:Verified
+test64:CRL bad signature
+test65:No revocation data
+test66:No revocation data
+test67:Verified
+test68:Certificate is revoked
+test69:Certificate is revoked
+test70:Certificate is revoked
+test71:Certificate is revoked
+test72:CRL has expired
+test73:CRL has expired
+test74:Verified
+#test75:
+#test76:
diff --git a/src/tests/test_x509_path.cpp b/src/tests/test_x509_path.cpp
index a7da0ee18..646f573a4 100644
--- a/src/tests/test_x509_path.cpp
+++ b/src/tests/test_x509_path.cpp
@@ -25,6 +25,33 @@ namespace {
 
 #if defined(BOTAN_HAS_X509_CERTIFICATES)
 
+std::map<std::string, std::string> read_results(const std::string& results_file)
+   {
+   std::ifstream in(results_file);
+   if(!in.good())
+      throw Test_Error("Failed reading " + results_file);
+
+   std::map<std::string, std::string> m;
+   std::string line;
+   while(in.good())
+      {
+      std::getline(in, line);
+      if(line == "")
+         continue;
+      if(line[0] == '#')
+         continue;
+
+      std::vector<std::string> parts = Botan::split_on(line, ':');
+
+      if(parts.size() != 2)
+         throw Test_Error("Invalid line " + line);
+
+      m[parts[0]] = parts[1];
+      }
+
+   return m;
+   }
+
 class X509test_Path_Validation_Tests : public Test
    {
    public:
@@ -34,11 +61,12 @@ class X509test_Path_Validation_Tests : public Test
 
          // Test certs generated by https://github.com/yymax/x509test
 
-         std::map<std::string, std::string> expected = read_results(data_file("x509test/expected.txt"));
+         std::map<std::string, std::string> expected =
+            read_results(Test::data_file("x509test/expected.txt"));
 
          const Botan::Path_Validation_Restrictions default_restrictions;
 
-         Botan::X509_Certificate root(data_file("x509test/root.pem"));
+         Botan::X509_Certificate root(Test::data_file("x509test/root.pem"));
          Botan::Certificate_Store_In_Memory trusted;
          trusted.add_certificate(root);
 
@@ -46,9 +74,10 @@ class X509test_Path_Validation_Tests : public Test
             {
             Test::Result result("X509test path validation");
             const std::string filename = i->first;
-            const std::string expected = i->second;
+            const std::string expected_result = i->second;
 
-            std::vector<Botan::X509_Certificate> certs = load_cert_file(data_file("x509test/" + filename));
+            std::vector<Botan::X509_Certificate> certs =
+               load_cert_file(Test::data_file("x509test/" + filename));
 
             if(certs.empty())
                throw Test_Error("Failed to read certs from " + filename);
@@ -60,7 +89,7 @@ class X509test_Path_Validation_Tests : public Test
             if(path_result.successful_validation() && path_result.trust_root() != root)
                path_result = Botan::Path_Validation_Result(Botan::Certificate_Status_Code::CANNOT_ESTABLISH_TRUST);
 
-            result.test_eq("validation result", path_result.result_string(), expected);
+            result.test_eq("validation result", path_result.result_string(), expected_result);
             results.push_back(result);
             }
 
@@ -85,32 +114,6 @@ class X509test_Path_Validation_Tests : public Test
          return certs;
          }
 
-      std::map<std::string, std::string> read_results(const std::string& results_file)
-         {
-         std::ifstream in(results_file);
-         if(!in.good())
-            throw Test_Error("Failed reading " + results_file);
-
-         std::map<std::string, std::string> m;
-         std::string line;
-         while(in.good())
-            {
-            std::getline(in, line);
-            if(line == "")
-               continue;
-            if(line[0] == '#')
-               continue;
-
-            std::vector<std::string> parts = Botan::split_on(line, ':');
-
-            if(parts.size() != 2)
-               throw Test_Error("Invalid line " + line);
-
-            m[parts[0]] = parts[1];
-            }
-
-         return m;
-         }
    };
 
 BOTAN_REGISTER_TEST("x509_path_x509test", X509test_Path_Validation_Tests);
@@ -119,142 +122,8 @@ class NIST_Path_Validation_Tests : public Test
    {
    public:
       std::vector<Test::Result> run() override;
-
-   private:
-      std::map<size_t, Botan::Path_Validation_Result::Code> get_expected();
    };
 
-/*
-  The expected results are essentially the error codes that best coorespond
-  to the problem described in the testing documentation.
-
-  There are a few cases where the tests say there should or should not be an
-  error, and I disagree. A few of the tests have test results different from
-  what they "should" be: these changes are marked as such, and have comments
-  explaining the problem at hand.
-*/
-std::map<size_t, Botan::Path_Validation_Result::Code> NIST_Path_Validation_Tests::get_expected()
-   {
-   using namespace Botan;
-
-   std::map<size_t, Path_Validation_Result::Code> expected_results;
-
-
-   // TODO read from a file
-   expected_results[1] = Certificate_Status_Code::VERIFIED;
-   expected_results[2] = Certificate_Status_Code::SIGNATURE_ERROR;
-   expected_results[3] = Certificate_Status_Code::SIGNATURE_ERROR;
-   expected_results[4] = Certificate_Status_Code::VERIFIED;
-   expected_results[5] = Certificate_Status_Code::CERT_NOT_YET_VALID;
-   expected_results[6] = Certificate_Status_Code::CERT_NOT_YET_VALID;
-   expected_results[7] = Certificate_Status_Code::VERIFIED;
-   expected_results[8] = Certificate_Status_Code::CERT_NOT_YET_VALID;
-   expected_results[9] = Certificate_Status_Code::CERT_HAS_EXPIRED;
-   expected_results[10] = Certificate_Status_Code::CERT_HAS_EXPIRED;
-   expected_results[11] = Certificate_Status_Code::CERT_HAS_EXPIRED;
-   expected_results[12] = Certificate_Status_Code::VERIFIED;
-   expected_results[13] = Certificate_Status_Code::CERT_ISSUER_NOT_FOUND;
-
-   expected_results[14] = Certificate_Status_Code::CERT_ISSUER_NOT_FOUND;
-   expected_results[15] = Certificate_Status_Code::VERIFIED;
-   expected_results[16] = Certificate_Status_Code::VERIFIED;
-   expected_results[17] = Certificate_Status_Code::VERIFIED;
-   expected_results[18] = Certificate_Status_Code::VERIFIED;
-
-   expected_results[19] = Certificate_Status_Code::NO_REVOCATION_DATA;
-   expected_results[20] = Certificate_Status_Code::CERT_IS_REVOKED;
-   expected_results[21] = Certificate_Status_Code::CERT_IS_REVOKED;
-
-   expected_results[22] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
-   expected_results[23] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
-   expected_results[24] = Certificate_Status_Code::VERIFIED;
-   expected_results[25] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
-   expected_results[26] = Certificate_Status_Code::VERIFIED;
-   expected_results[27] = Certificate_Status_Code::VERIFIED;
-   expected_results[28] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
-   expected_results[29] = Certificate_Status_Code::CA_CERT_NOT_FOR_CERT_ISSUER;
-   expected_results[30] = Certificate_Status_Code::VERIFIED;
-
-   expected_results[31] = Certificate_Status_Code::CA_CERT_NOT_FOR_CRL_ISSUER;
-   expected_results[32] = Certificate_Status_Code::CA_CERT_NOT_FOR_CRL_ISSUER;
-   expected_results[33] = Certificate_Status_Code::VERIFIED;
-
-   /*
-    Policy tests: a little trickier because there are other inputs
-    which affect the result.
-
-    In the case of the tests currently in the suite, the default
-    method (with acceptable policy being "any-policy" and with no
-    explicit policy required), will almost always result in a verified
-    status. This is not particularly helpful. So, we should do several
-    different tests for each test set:
-
-       1) With the user policy as any-policy and no explicit policy
-       2) With the user policy as any-policy and an explicit policy required
-       3) With the user policy as test-policy-1 (2.16.840.1.101.3.1.48.1) and
-          an explicit policy required
-       4) With the user policy as either test-policy-1 or test-policy-2 and an
-          explicit policy required
-
-     This provides reasonably good coverage of the possible outcomes.
-   */
-
-   expected_results[34] = Certificate_Status_Code::VERIFIED;
-   expected_results[35] = Certificate_Status_Code::VERIFIED;
-   expected_results[36] = Certificate_Status_Code::VERIFIED;
-   expected_results[37] = Certificate_Status_Code::VERIFIED;
-   expected_results[38] = Certificate_Status_Code::VERIFIED;
-   expected_results[39] = Certificate_Status_Code::VERIFIED;
-   expected_results[40] = Certificate_Status_Code::VERIFIED;
-   expected_results[41] = Certificate_Status_Code::VERIFIED;
-   expected_results[42] = Certificate_Status_Code::VERIFIED;
-   expected_results[43] = Certificate_Status_Code::VERIFIED;
-   expected_results[44] = Certificate_Status_Code::VERIFIED;
-
-   //expected_results[45] = Certificate_Status_Code::EXPLICIT_POLICY_REQUIRED;
-   //expected_results[46] = Certificate_Status_Code::ACCEPT;
-   //expected_results[47] = Certificate_Status_Code::EXPLICIT_POLICY_REQUIRED;
-
-   expected_results[48] = Certificate_Status_Code::VERIFIED;
-   expected_results[49] = Certificate_Status_Code::VERIFIED;
-   expected_results[50] = Certificate_Status_Code::VERIFIED;
-   expected_results[51] = Certificate_Status_Code::VERIFIED;
-   expected_results[52] = Certificate_Status_Code::VERIFIED;
-   expected_results[53] = Certificate_Status_Code::VERIFIED;
-
-   expected_results[54] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
-   expected_results[55] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
-   expected_results[56] = Certificate_Status_Code::VERIFIED;
-   expected_results[57] = Certificate_Status_Code::VERIFIED;
-   expected_results[58] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
-   expected_results[59] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
-   expected_results[60] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
-   expected_results[61] = Certificate_Status_Code::CERT_CHAIN_TOO_LONG;
-   expected_results[62] = Certificate_Status_Code::VERIFIED;
-   expected_results[63] = Certificate_Status_Code::VERIFIED;
-
-   expected_results[64] = Certificate_Status_Code::CRL_BAD_SIGNATURE;
-
-   expected_results[65] = Certificate_Status_Code::NO_REVOCATION_DATA;
-   expected_results[66] = Certificate_Status_Code::NO_REVOCATION_DATA;
-
-   expected_results[67] = Certificate_Status_Code::VERIFIED;
-
-   expected_results[68] = Certificate_Status_Code::CERT_IS_REVOKED;
-   expected_results[69] = Certificate_Status_Code::CERT_IS_REVOKED;
-   expected_results[70] = Certificate_Status_Code::CERT_IS_REVOKED;
-   expected_results[71] = Certificate_Status_Code::CERT_IS_REVOKED;
-   expected_results[72] = Certificate_Status_Code::CRL_HAS_EXPIRED;
-   expected_results[73] = Certificate_Status_Code::CRL_HAS_EXPIRED;
-   expected_results[74] = Certificate_Status_Code::VERIFIED;
-
-   /* These tests use weird CRL extensions which aren't supported yet */
-   //expected_results[75] = ;
-   //expected_results[76] = ;
-
-   return expected_results;
-   }
-
 std::vector<Test::Result> NIST_Path_Validation_Tests::run()
    {
    std::vector<Test::Result> results;
@@ -269,12 +138,12 @@ std::vector<Test::Result> NIST_Path_Validation_Tests::run()
    *  - Tests #75 and #76 are skipped as they make use of relatively
    *    obscure CRL extensions which are not supported.
    */
-   const std::string root_test_dir = "src/tests/data/nist_x509/";
+   const std::string nist_test_dir = Test::data_dir() + "/nist_x509";
 
    try
       {
       // Do nothing, just test filesystem access
-      Botan::get_files_recursive(root_test_dir);
+      Botan::get_files_recursive(nist_test_dir);
       }
    catch(Botan::No_Filesystem_Access)
       {
@@ -284,80 +153,60 @@ std::vector<Test::Result> NIST_Path_Validation_Tests::run()
       return results;
       }
 
-   const size_t total_tests = 76;
-   std::map<size_t, Botan::Path_Validation_Result::Code> expected_results = get_expected();
+   std::map<std::string, std::string> expected =
+      read_results(Test::data_file("nist_x509/expected.txt"));
 
-   for(size_t test_no = 1; test_no <= total_tests; ++test_no)
+   for(auto i = expected.begin(); i != expected.end(); ++i)
       {
-      try
-         {
-         Test::Result result("NIST path validation");
-         const std::string test_dir = root_test_dir + "/test" + (test_no <= 9 ? "0" : "") + std::to_string(test_no);
+      const std::string test_name = i->first;
+      const std::string expected_result = i->second;
+      printf("%s %s\n", test_name.c_str(), expected_result.c_str());
 
-         const std::vector<std::string> all_files = Botan::get_files_recursive(test_dir);
-         if (all_files.empty())
-            {
-            result.test_failure("No test files found in " + test_dir);
-            continue;
-            }
-
-         std::vector<std::string> certs, crls;
-         std::string root_cert, to_verify;
+      const std::string test_dir = nist_test_dir + "/" + test_name;
 
-         for(const auto &current : all_files)
-            {
-            if(current.find("int") != std::string::npos && current.find(".crt") != std::string::npos)
-               certs.push_back(current);
-            else if(current.find("root.crt") != std::string::npos)
-               root_cert = current;
-            else if(current.find("end.crt") != std::string::npos)
-               to_verify = current;
-            else if(current.find(".crl") != std::string::npos)
-               crls.push_back(current);
-            }
-
-         if(expected_results.find(test_no) == expected_results.end())
-            {
-            result.test_note("Skipping test");
-            continue;
-            }
-
-         Botan::Certificate_Store_In_Memory store;
+      Test::Result result("NIST path validation");
 
-         store.add_certificate(Botan::X509_Certificate(root_cert));
+      const std::vector<std::string> all_files = Botan::get_files_recursive(test_dir);
 
-         Botan::X509_Certificate end_user(to_verify);
+      if(all_files.empty())
+         {
+         result.test_failure("No test files found in " + test_dir);
+         results.push_back(result);
+         continue;
+         }
 
-         for(size_t i = 0; i != certs.size(); i++)
-            store.add_certificate(Botan::X509_Certificate(certs[i]));
+      Botan::Certificate_Store_In_Memory store;
 
-         for(size_t i = 0; i != crls.size(); i++)
+      for(auto&& file : all_files)
+         {
+         if(file.find(".crt") != std::string::npos && file != "end.crt")
             {
-            Botan::DataSource_Stream in(crls[i], true);
+            store.add_certificate(Botan::X509_Certificate(file));
+            }
+         else if(file.find(".crl") != std::string::npos)
+            {
+            Botan::DataSource_Stream in(file, true);
             Botan::X509_CRL crl(in);
             store.add_crl(crl);
+            }
          }
 
-         Botan::Path_Validation_Restrictions restrictions(true);
+      Botan::X509_Certificate end_user(test_dir + "/end.crt");
 
-         Botan::Path_Validation_Result validation_result =
-            Botan::x509_path_validate(end_user,
-                                      restrictions,
-                                      store);
+      Botan::Path_Validation_Restrictions restrictions(true);
 
-         auto expected = expected_results[test_no];
+      Botan::Path_Validation_Result validation_result =
+         Botan::x509_path_validate(end_user,
+                                   restrictions,
+                                   store);
 
-         result.test_eq("path validation result",
-                        validation_result.result_string(),
-                        Botan::Path_Validation_Result::status_string(expected));
+      result.test_eq(test_name + " path validation result",
+                     validation_result.result_string(),
+                     expected_result);
 
-         results.push_back(result);
-         }
-      catch(std::exception& e)
-         {
-         results.push_back(Test::Result::Failure("NIST X509 " + std::to_string(test_no), e.what()));
-         }
+      results.push_back(result);
       }
+
    return results;
    }