Clean up and finish off the Python version of the NIST X.509 test suite.

Expected results are read from a file (results.txt), and pass/fails are
reported. Also change validate to return the result normally, rather than
throwing an exception; while it did save some code in validate() itself,
with the addition of all the code to support it, it was a bit longer, as
well as being somewhat hard to follow.

2 files changed, 93 insertions, 23 deletions

diff --git a/misc/python/nisttest.py b/misc/python/nisttest.py
index c070f76a9..c1131d2b0 100755
--- a/misc/python/nisttest.py
+++ b/misc/python/nisttest.py
@@ -3,43 +3,59 @@
 import sys, os, botan
 from os.path import join;
 
-class TestResult(Exception):
-    def __init__(self, r):
-        self.result = r
-    def __str__(self):
-        return repr(self.result).replace('botan._botan.verify_result.', '')
-
-def raise_unless_ok(r):
-    if r != botan.verify_result.verified:
-        raise TestResult(r)
-
 def validate(ca_certs, certs, crls, ee_certs):
     store = botan.X509_Store()
-
     for cert in certs:
         if cert not in ee_certs:
             store.add_cert(botan.X509_Certificate(cert), cert in ca_certs)
 
     for crl in crls:
-        raise_unless_ok(store.add_crl(botan.X509_CRL(crl)))
+        r = store.add_crl(botan.X509_CRL(crl))
+        if r != botan.verify_result.verified:
+            return r
 
     for ee in ee_certs:
-        raise_unless_ok(store.validate(botan.X509_Certificate(ee)))
+        r = store.validate(botan.X509_Certificate(ee))
+        if r != botan.verify_result.verified:
+            return r
+
+    return botan.verify_result.verified
 
-    raise TestResult(botan.verify_result.verified)
+def run_test(files, rootdir, testname, expected):
+    crls = [join(rootdir,x) for x in files if x.endswith(".crl")]
+    certs = [join(rootdir,x) for x in files if x.endswith(".crt")]
+    end_entity = [x for x in certs if x.find("End Cert") != -1]
+    ca_certs = [x for x in certs if x.find("Trust Anchor") != -1]
+
+    print "Running", testname, "...",
+
+    result = validate(ca_certs, certs, crls, end_entity)
+    result = repr(result).replace('botan._botan.verify_result.', '')
+    
+    if result != expected:
+        print "FAILED: got", result, "expected", expected
+    else:
+        print "passed"
 
 def main():
+    def load_results(file):
+        results = {}
+        for line in open(file, 'r'):
+            line = line[0:line.find('#')].strip()
+            if line:
+                test,result = line.split(' ')
+                results[test] = result
+        return results
+
+    results = load_results('results.txt')
+
     for root, dirs, files in os.walk('../nist_tests/tests'):
         if files:
-            crls = [join(root,x) for x in files if x.endswith(".crl")]
-            certs = [join(root,x) for x in files if x.endswith(".crt")]
-            end_entity = [x for x in certs if x.find("End Cert") != -1]
-            ca_certs = [x for x in certs if x.find("Trust Anchor") != -1]
-
-            try:
-                validate(ca_certs, certs, crls, end_entity)
-            except TestResult, result:
-                print result
+            thistest = root[root.rfind('/')+1:]
+            if thistest in results:
+                run_test(files, root, thistest, results[thistest])
+            else:
+                print "Skipping", thistest, "- no expected result set"
         
 if __name__ == "__main__":
     sys.exit(main())
diff --git a/misc/python/results.txt b/misc/python/results.txt
new file mode 100644
index 000000000..01a5f5982
--- /dev/null
+++ b/misc/python/results.txt
@@ -0,0 +1,54 @@
+test01 verified
+test02 verified
+test03 signature_error
+test04 verified
+test05 cert_not_yet_valid
+test06 cert_not_yet_valid
+test07 verified
+test08 cert_not_yet_valid
+test09 cert_has_expired
+test10 cert_has_expired
+test11 cert_has_expired
+test12 verified
+test13 cert_issuer_not_found
+test14 cert_issuer_not_found
+test15 verified
+test16 verified
+test17 verified
+test18 verified
+test19 verified
+test20 cert_is_revoked
+test21 cert_is_revoked
+test22 ca_cert_not_for_cert_issuer
+test23 ca_cert_not_for_cert_issuer
+test24 verified
+test25 ca_cert_not_for_cert_issuer
+test26 verified
+test27 verified
+test28 ca_cert_not_for_cert_issuer
+test29 ca_cert_not_for_cert_issuer
+test30 verified
+test31 ca_cert_not_for_crl_issuer
+test32 ca_cert_not_for_crl_issuer
+test33 verified
+test54 cert_chain_too_long
+test55 cert_chain_too_long
+test56 verified
+test57 verified
+test58 cert_chain_too_long
+test59 cert_chain_too_long
+test60 cert_chain_too_long
+test61 cert_chain_too_long
+test62 verified
+test63 verified
+test64 signature_error
+test65 verified
+test66 crl_issuer_not_found
+test67 crl_issuer_not_found
+test68 cert_is_revoked
+test69 cert_is_revoked
+test70 cert_is_revoked
+test71 cert_is_revoked
+test72 crl_has_expired
+test73 crl_has_expired
+test74 verified