Test report is more useful now

12 files changed, 472 insertions, 431 deletions

diff --git a/checks/bigint.cpp b/checks/bigint.cpp
index d6619fe87..10f46900f 100644
--- a/checks/bigint.cpp
+++ b/checks/bigint.cpp
@@ -5,6 +5,7 @@
 */
 
 #include "tests.h"
+#include "common.h"
 
 #include <vector>
 #include <string>
@@ -12,43 +13,41 @@
 #include <iostream>
 #include <cstdlib>
 
+#include <botan/auto_rng.h>
 #include <botan/bigint.h>
 #include <botan/exceptn.h>
 #include <botan/numthry.h>
 using namespace Botan;
 
-#include "common.h"
-#include "validate.h"
-
-#define DEBUG 0
-
-u32bit check_add(const std::vector<std::string>&);
-u32bit check_sub(const std::vector<std::string>&);
-u32bit check_mul(const std::vector<std::string>&);
-u32bit check_sqr(const std::vector<std::string>&);
-u32bit check_div(const std::vector<std::string>&);
-u32bit check_mod(const std::vector<std::string>&,
+size_t check_add(const std::vector<std::string>&);
+size_t check_sub(const std::vector<std::string>&);
+size_t check_mul(const std::vector<std::string>&);
+size_t check_sqr(const std::vector<std::string>&);
+size_t check_div(const std::vector<std::string>&);
+size_t check_mod(const std::vector<std::string>&,
                  Botan::RandomNumberGenerator& rng);
-u32bit check_shr(const std::vector<std::string>&);
-u32bit check_shl(const std::vector<std::string>&);
+size_t check_shr(const std::vector<std::string>&);
+size_t check_shl(const std::vector<std::string>&);
 
-u32bit check_powmod(const std::vector<std::string>&);
-u32bit check_primetest(const std::vector<std::string>&,
+size_t check_powmod(const std::vector<std::string>&);
+size_t check_primetest(const std::vector<std::string>&,
                        Botan::RandomNumberGenerator&);
 
-u32bit do_bigint_tests(const std::string& filename,
-                       Botan::RandomNumberGenerator& rng)
+size_t test_bigint()
    {
+   const std::string filename = "checks/mp_valid.dat";
    std::ifstream test_data(filename.c_str());
 
    if(!test_data)
       throw Botan::Stream_IO_Error("Couldn't open test file " + filename);
 
-   u32bit total_errors = 0;
-   u32bit errors = 0, alg_count = 0;
+   size_t total_errors = 0;
+   size_t errors = 0, alg_count = 0;
    std::string algorithm;
    bool first = true;
-   u32bit counter = 0;
+   size_t counter = 0;
+
+   AutoSeeded_RNG rng;
 
    while(!test_data.eof())
       {
@@ -95,7 +94,7 @@ u32bit do_bigint_tests(const std::string& filename,
       std::cout << "Testing: " << algorithm << std::endl;
 #endif
 
-      u32bit new_errors = 0;
+      size_t new_errors = 0;
       if(algorithm.find("Addition") != std::string::npos)
          new_errors = check_add(substr);
       else if(algorithm.find("Subtraction") != std::string::npos)
@@ -134,7 +133,7 @@ u32bit do_bigint_tests(const std::string& filename,
 namespace {
 
 // c==expected, d==a op b, e==a op= b
-u32bit results(std::string op,
+size_t results(std::string op,
                const BigInt& a, const BigInt& b,
                const BigInt& c, const BigInt& d, const BigInt& e)
    {
@@ -167,7 +166,7 @@ u32bit results(std::string op,
 
 }
 
-u32bit check_add(const std::vector<std::string>& args)
+size_t check_add(const std::vector<std::string>& args)
    {
    BigInt a(args[0]);
    BigInt b(args[1]);
@@ -187,7 +186,7 @@ u32bit check_add(const std::vector<std::string>& args)
    return results("+", a, b, c, d, e);
    }
 
-u32bit check_sub(const std::vector<std::string>& args)
+size_t check_sub(const std::vector<std::string>& args)
    {
    BigInt a(args[0]);
    BigInt b(args[1]);
@@ -200,7 +199,7 @@ u32bit check_sub(const std::vector<std::string>& args)
    return results("-", a, b, c, d, e);
    }
 
-u32bit check_mul(const std::vector<std::string>& args)
+size_t check_mul(const std::vector<std::string>& args)
    {
    BigInt a(args[0]);
    BigInt b(args[1]);
@@ -231,7 +230,7 @@ u32bit check_mul(const std::vector<std::string>& args)
    return results("*", a, b, c, d, e);
    }
 
-u32bit check_sqr(const std::vector<std::string>& args)
+size_t check_sqr(const std::vector<std::string>& args)
    {
    BigInt a(args[0]);
    BigInt b(args[1]);
@@ -245,7 +244,7 @@ u32bit check_sqr(const std::vector<std::string>& args)
    return results("sqr", a, a, b, c, d);
    }
 
-u32bit check_div(const std::vector<std::string>& args)
+size_t check_div(const std::vector<std::string>& args)
    {
    BigInt a(args[0]);
    BigInt b(args[1]);
@@ -258,7 +257,7 @@ u32bit check_div(const std::vector<std::string>& args)
    return results("/", a, b, c, d, e);
    }
 
-u32bit check_mod(const std::vector<std::string>& args,
+size_t check_mod(const std::vector<std::string>& args,
                  Botan::RandomNumberGenerator& rng)
    {
    BigInt a(args[0]);
@@ -269,7 +268,7 @@ u32bit check_mod(const std::vector<std::string>& args,
    BigInt e = a;
    e %= b;
 
-   u32bit got = results("%", a, b, c, d, e);
+   size_t got = results("%", a, b, c, d, e);
 
    if(got) return got;
 
@@ -277,7 +276,7 @@ u32bit check_mod(const std::vector<std::string>& args,
 
    /* Won't work for us, just pick one at random */
    while(b_word == 0)
-      for(u32bit j = 0; j != 2*sizeof(word); j++)
+      for(size_t j = 0; j != 2*sizeof(word); j++)
          b_word = (b_word << 4) ^ rng.next_byte();
 
    b = b_word;
@@ -291,10 +290,10 @@ u32bit check_mod(const std::vector<std::string>& args,
    return results("%(word)", a, b, c, d2, e);
    }
 
-u32bit check_shl(const std::vector<std::string>& args)
+size_t check_shl(const std::vector<std::string>& args)
    {
    BigInt a(args[0]);
-   u32bit b = std::atoi(args[1].c_str());
+   size_t b = std::atoi(args[1].c_str());
    BigInt c(args[2]);
 
    BigInt d = a << b;
@@ -304,10 +303,10 @@ u32bit check_shl(const std::vector<std::string>& args)
    return results("<<", a, b, c, d, e);
    }
 
-u32bit check_shr(const std::vector<std::string>& args)
+size_t check_shr(const std::vector<std::string>& args)
    {
    BigInt a(args[0]);
-   u32bit b = std::atoi(args[1].c_str());
+   size_t b = std::atoi(args[1].c_str());
    BigInt c(args[2]);
 
    BigInt d = a >> b;
@@ -318,7 +317,7 @@ u32bit check_shr(const std::vector<std::string>& args)
    }
 
 /* Make sure that (a^b)%m == r */
-u32bit check_powmod(const std::vector<std::string>& args)
+size_t check_powmod(const std::vector<std::string>& args)
    {
    BigInt a(args[0]);
    BigInt b(args[1]);
@@ -341,7 +340,7 @@ u32bit check_powmod(const std::vector<std::string>& args)
    }
 
 /* Make sure that n is prime or not prime, according to should_be_prime */
-u32bit check_primetest(const std::vector<std::string>& args,
+size_t check_primetest(const std::vector<std::string>& args,
                        Botan::RandomNumberGenerator& rng)
    {
    BigInt n(args[0]);
diff --git a/checks/check.cpp b/checks/check.cpp
index 751dd02b6..c45256e89 100644
--- a/checks/check.cpp
+++ b/checks/check.cpp
@@ -29,12 +29,9 @@ using namespace Botan;
 
 #include "getopt.h"
 #include "bench.h"
-#include "validate.h"
 #include "common.h"
 #include "tests.h"
 
-int run_test_suite(RandomNumberGenerator& rng);
-
 namespace {
 
 template<typename T>
@@ -92,6 +89,35 @@ void test_types()
       std::cout << "Typedefs in include/types.h may be incorrect!\n";
    }
 
+int run_tests()
+   {
+   size_t errors = 0;
+
+   try
+      {
+      errors += run_all_tests();
+      }
+   catch(std::exception& e)
+      {
+      std::cout << "Exception in test suite " << e.what() << std::endl;
+      ++errors;
+      }
+   catch(...)
+      {
+      std::cout << "Unknown exception caught\n";
+      ++errors;
+      }
+
+   if(errors)
+      {
+      std::cout << errors << " test"  << ((errors == 1) ? "" : "s")
+                << " failed." << std::endl;
+      return 1;
+      }
+
+   return 0;
+   }
+
 }
 
 int main(int argc, char* argv[])
@@ -151,7 +177,7 @@ int main(int argc, char* argv[])
 
       if(opts.is_set("validate") || opts.is_set("test"))
          {
-         return run_test_suite(rng);
+         return run_tests();
          }
       if(opts.is_set("bench-algo") ||
          opts.is_set("benchmark") ||
@@ -213,40 +239,3 @@ int main(int argc, char* argv[])
    return 0;
    }
 
-int run_test_suite(RandomNumberGenerator& rng)
-   {
-   u32bit errors = 0;
-   try
-      {
-      const std::string BIGINT_VALIDATION_FILE = "checks/mp_valid.dat";
-      const std::string PK_VALIDATION_FILE = "checks/pk_valid.dat";
-
-      errors += run_all_tests();
-      //errors += do_validation_tests(VALIDATION_FILE, rng);
-      //errors += do_validation_tests(EXPECTED_FAIL_FILE, rng, false);
-      errors += do_bigint_tests(BIGINT_VALIDATION_FILE, rng);
-      errors += do_pk_validation_tests(PK_VALIDATION_FILE, rng);
-      do_x509_tests(rng);
-      do_tls_tests(rng);
-      //errors += do_cvc_tests(rng);
-      }
-   catch(std::exception& e)
-      {
-      std::cout << "Exception: " << e.what() << std::endl;
-      return 1;
-      }
-   catch(...)
-      {
-      std::cout << "Unknown exception caught." << std::endl;
-      return 1;
-      }
-
-   if(errors)
-      {
-      std::cout << errors << " test"  << ((errors == 1) ? "" : "s")
-                << " failed." << std::endl;
-      return 1;
-      }
-
-   return 0;
-   }
diff --git a/checks/cvc_tests.cpp b/checks/cvc_tests.cpp
index 9dc56610c..b4d9ea033 100644
--- a/checks/cvc_tests.cpp
+++ b/checks/cvc_tests.cpp
@@ -5,7 +5,7 @@
 *     2008 Jack Lloyd
 */
 
-#include "validate.h"
+#include "tests.h"
 #include <botan/build.h>
 
 #if defined(BOTAN_HAS_CARD_VERIFIABLE_CERTIFICATES)
@@ -77,8 +77,6 @@ bool helper_files_equal(std::string const& file_path1, std::string const& file_p
 
 void test_enc_gen_selfsigned(RandomNumberGenerator& rng)
    {
-   std::cout << '.' << std::flush;
-
    EAC1_1_CVC_Options opts;
    //opts.cpi = 0;
    opts.chr = ASN1_Chr("my_opt_chr"); // not used
@@ -190,8 +188,6 @@ void test_enc_gen_selfsigned(RandomNumberGenerator& rng)
 
 void test_enc_gen_req(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
-
    EAC1_1_CVC_Options opts;
 
    //opts.cpi = 0;
@@ -221,8 +217,6 @@ void test_enc_gen_req(RandomNumberGenerator& rng)
 
 void test_cvc_req_ext(RandomNumberGenerator&)
    {
-   std::cout << "." << std::flush;
-
    EAC1_1_Req req_in(TEST_DATA_DIR "/DE1_flen_chars_cvcRequest_ECDSA.der");
    EC_Group dom_pars(OID("1.3.36.3.3.2.8.1.1.5")); // "german curve"
    //req_in.set_domain_parameters(dom_pars);
@@ -236,8 +230,6 @@ void test_cvc_req_ext(RandomNumberGenerator&)
 
 void test_cvc_ado_ext(RandomNumberGenerator&)
    {
-   std::cout << "." << std::flush;
-
    EAC1_1_ADO req_in(TEST_DATA_DIR "/ado.cvcreq");
    EC_Group dom_pars(OID("1.3.36.3.3.2.8.1.1.5")); // "german curve"
    //cout << "car = " << req_in.get_car().value() << std::endl;
@@ -246,8 +238,6 @@ void test_cvc_ado_ext(RandomNumberGenerator&)
 
 void test_cvc_ado_creation(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
-
    EAC1_1_CVC_Options opts;
    //opts.cpi = 0;
    opts.chr = ASN1_Chr("my_opt_chr");
@@ -290,8 +280,6 @@ void test_cvc_ado_creation(RandomNumberGenerator& rng)
 
 void test_cvc_ado_comparison(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
-
    EAC1_1_CVC_Options opts;
    //opts.cpi = 0;
    opts.chr = ASN1_Chr("my_opt_chr");
@@ -346,8 +334,6 @@ void test_cvc_ado_comparison(RandomNumberGenerator& rng)
 
 void test_eac_time(RandomNumberGenerator&)
    {
-   std::cout << "." << std::flush;
-
    EAC_Time time(std::chrono::system_clock::now());
    //     std::cout << "time as std::string = " << time.as_string() << std::endl;
    EAC_Time sooner("", ASN1_Tag(99));
@@ -383,8 +369,6 @@ void test_eac_time(RandomNumberGenerator&)
 
 void test_ver_cvca(RandomNumberGenerator&)
    {
-   std::cout << "." << std::flush;
-
    EAC1_1_CVC req_in(TEST_DATA_DIR "/cvca01.cv.crt");
 
    bool exc = false;
@@ -407,8 +391,6 @@ void test_ver_cvca(RandomNumberGenerator&)
 
 void test_copy_and_assignment(RandomNumberGenerator&)
    {
-   std::cout << "." << std::flush;
-
    EAC1_1_CVC cert_in(TEST_DATA_DIR "/cvca01.cv.crt");
    EAC1_1_CVC cert_cp(cert_in);
    EAC1_1_CVC cert_ass = cert_in;
@@ -432,8 +414,6 @@ void test_copy_and_assignment(RandomNumberGenerator&)
 
 void test_eac_str_illegal_values(RandomNumberGenerator&)
    {
-   std::cout << "." << std::flush;
-
    bool exc = false;
    try
       {
@@ -461,8 +441,6 @@ void test_eac_str_illegal_values(RandomNumberGenerator&)
 
 void test_tmp_eac_str_enc(RandomNumberGenerator&)
    {
-   std::cout << "." << std::flush;
-
    bool exc = false;
    try
       {
@@ -482,8 +460,6 @@ void test_tmp_eac_str_enc(RandomNumberGenerator&)
 
 void test_cvc_chain(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
-
    EC_Group dom_pars(OID("1.3.36.3.3.2.8.1.1.5")); // "german curve"
    ECDSA_PrivateKey cvca_privk(rng, dom_pars);
    std::string hash("SHA-224");
@@ -573,9 +549,9 @@ void test_cvc_chain(RandomNumberGenerator& rng)
 
 }
 
-u32bit do_cvc_tests(Botan::RandomNumberGenerator& rng)
+size_t test_cvc()
    {
-   std::cout << "Testing CVC: " << std::flush;
+   AutoSeeded_RNG rng;
 
    test_enc_gen_selfsigned(rng);
    test_enc_gen_req(rng);
@@ -589,10 +565,9 @@ u32bit do_cvc_tests(Botan::RandomNumberGenerator& rng)
    test_eac_str_illegal_values(rng);
    test_tmp_eac_str_enc(rng);
    test_cvc_chain(rng);
-   std::cout << std::endl;
 
    return 0;
    }
 #else
-u32bit do_cvc_tests(Botan::RandomNumberGenerator&) { return 0; }
+size_t test_cvc() { return 0; }
 #endif
diff --git a/checks/ec_tests.cpp b/checks/ec_tests.cpp
index 0f2a268d8..ba01d0a79 100644
--- a/checks/ec_tests.cpp
+++ b/checks/ec_tests.cpp
@@ -4,11 +4,12 @@
 * Distributed under the terms of the Botan license
 */
 
+#include "tests.h"
 
-#include <botan/rng.h>
-
-#if defined(BOTAN_HAS_ECC_GROUP)
+#include "getopt.h"
+#include "common.h"
 
+#include <botan/auto_rng.h>
 #include <botan/bigint.h>
 #include <botan/numthry.h>
 #include <botan/curve_gfp.h>
@@ -16,19 +17,13 @@
 #include <botan/ec_group.h>
 #include <botan/reducer.h>
 #include <botan/oids.h>
-
-using namespace Botan;
-
 #include <iostream>
 #include <memory>
 
-#include "getopt.h"
-#include "validate.h"
-#include "common.h"
-
+using namespace Botan;
 
-#define CHECK_MESSAGE(expr, print) try { if(!(expr)) std::cout << print << "\n"; } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
-#define CHECK(expr) try { if(!(expr)) std::cout << #expr << "\n"; } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
+#define CHECK_MESSAGE(expr, print) try { if(!(expr)) { ++fails; std::cout << print << "\n"; }} catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
+#define CHECK(expr) try { if(!(expr)) { ++fails; std::cout << #expr << "\n"; } } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
 
 namespace {
 
@@ -62,9 +57,9 @@ PointGFp create_random_point(RandomNumberGenerator& rng,
       }
    }
 
-void test_point_turn_on_sp_red_mul()
+size_t test_point_turn_on_sp_red_mul()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    // setting up expected values
    BigInt exp_Qx(std::string("466448783855397898016055842232266600516272889280"));
@@ -125,11 +120,12 @@ void test_point_turn_on_sp_red_mul()
    r2 += p_G2;
 
    CHECK_MESSAGE(r1 == r2, "error with op+= after extra turn on sp red mul for both operands");
+   return fails;
    }
 
-void test_coordinates()
+size_t test_coordinates()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    BigInt exp_affine_x(std::string("16984103820118642236896513183038186009872590470"));
    BigInt exp_affine_y(std::string("1373093393927139016463695321221277758035357890939"));
@@ -157,6 +153,7 @@ void test_coordinates()
 
    CHECK_MESSAGE( p1.get_affine_x() == exp_affine_x, " p1_x = " << p1.get_affine_x() << "\n" << "exp_x = " << exp_affine_x << "\n");
    CHECK_MESSAGE( p1.get_affine_y() == exp_affine_y, " p1_y = " << p1.get_affine_y() << "\n" << "exp_y = " << exp_affine_y << "\n");
+   return fails;
    }
 
 
@@ -172,9 +169,9 @@ Section 2.1.2
 --------
 */
 
-void test_point_transformation ()
+size_t test_point_transformation ()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    // get a vailid point
    EC_Group dom_pars(OID("1.3.132.0.8"));
@@ -185,11 +182,12 @@ void test_point_transformation ()
 
    CHECK_MESSAGE( p.get_affine_x() == q.get_affine_x(), "affine_x changed during copy");
    CHECK_MESSAGE( p.get_affine_y() == q.get_affine_y(), "affine_y changed during copy");
+   return fails;
    }
 
-void test_point_mult ()
+size_t test_point_mult ()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    EC_Group secp160r1(OIDS::lookup("secp160r1"));
 
@@ -204,11 +202,12 @@ void test_point_mult ()
 
    CHECK(Q_U.get_affine_x() == BigInt("466448783855397898016055842232266600516272889280"));
    CHECK(Q_U.get_affine_y() == BigInt("1110706324081757720403272427311003102474457754220"));
+   return fails;
    }
 
-void test_point_negative()
+size_t test_point_negative()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    // performing calculation to test
    std::string p_secp = "ffffffffffffffffffffffffffffffff7fffffff";
@@ -234,12 +233,12 @@ void test_point_negative()
 
    CHECK(p1_neg.get_affine_x() == BigInt("16984103820118642236896513183038186009872590470"));
    CHECK(p1_neg.get_affine_y() == BigInt("88408243403763901739989511495005261618427168388"));
+   return fails;
    }
 
-void test_zeropoint()
+size_t test_zeropoint()
    {
-   std::cout << "." << std::flush;
-
+   size_t fails = 0;
 
    std::string G_secp_comp = "024a96b5688ef573284664698968c38bb913cbfc82";
    std::vector<byte> sv_G_secp_comp = hex_decode ( G_secp_comp );
@@ -257,12 +256,12 @@ void test_zeropoint()
    p1 -= p1;
 
    CHECK_MESSAGE(  p1.is_zero(), "p - q with q = p is not zero!");
+   return fails;
    }
 
-void test_zeropoint_enc_dec()
+size_t test_zeropoint_enc_dec()
    {
-   std::cout << "." << std::flush;
-
+   size_t fails = 0;
 
    BigInt bi_p_secp("0xffffffffffffffffffffffffffffffff7fffffff");
    BigInt bi_a_secp("0xffffffffffffffffffffffffffffffff7ffffffc");
@@ -284,11 +283,12 @@ void test_zeropoint_enc_dec()
    sv_p = unlock(EC2OSP(p, PointGFp::HYBRID));
    p_encdec = OS2ECP(sv_p, curve);
    CHECK_MESSAGE(  p == p_encdec, "encoded-decoded (hybrid) point is not equal the original!");
+   return fails;
    }
 
-void test_calc_with_zeropoint()
+size_t test_calc_with_zeropoint()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    std::string G_secp_comp = "024a96b5688ef573284664698968c38bb913cbfc82";
    std::vector<byte> sv_G_secp_comp = hex_decode ( G_secp_comp );
@@ -316,11 +316,12 @@ void test_calc_with_zeropoint()
 
    res = zero * 32432243;
    CHECK_MESSAGE(  res.is_zero(), "zeropoint * skalar is not a zero-point!");
+   return fails;
    }
 
-void test_add_point()
+size_t test_add_point()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    // precalculation
    std::string p_secp = "ffffffffffffffffffffffffffffffff7fffffff";
@@ -347,11 +348,12 @@ void test_add_point()
                      BigInt("1147993098458695153857594941635310323215433166682"));
 
    CHECK(p1 == expected);
+   return fails;
    }
 
-void test_sub_point()
+size_t test_sub_point()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    //Setting up expected values
    BigInt exp_sub_x(std::string("112913490230515010376958384252467223283065196552"));
@@ -383,11 +385,12 @@ void test_sub_point()
                      BigInt("203520114162904107873991457957346892027982641970"));
 
    CHECK(p1 == expected);
+   return fails;
    }
 
-void test_mult_point()
+size_t test_mult_point()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    //Setting up expected values
    BigInt exp_mult_x(std::string("967697346845926834906555988570157345422864716250"));
@@ -416,11 +419,12 @@ void test_mult_point()
    PointGFp expected(secp160r1, exp_mult_x, exp_mult_y);
 
    CHECK(p1 == expected);
+   return fails;
    }
 
-void test_basic_operations()
+size_t test_basic_operations()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    // precalculation
    std::string p_secp = "ffffffffffffffffffffffffffffffff7fffffff";
@@ -476,12 +480,12 @@ void test_basic_operations()
 
    CHECK(p0.get_affine_x() == BigInt("425826231723888350446541592701409065913635568770"));
    CHECK(p0.get_affine_y() == BigInt("203520114162904107873991457957346892027982641970"));
+   return fails;
    }
 
-void test_enc_dec_compressed_160()
+size_t test_enc_dec_compressed_160()
    {
-   std::cout << "." << std::flush;
-
+   size_t fails = 0;
 
    // Test for compressed conversion (02/03) 160bit
    std::string p_secp = "ffffffffffffffffffffffffffffffff7fffffff";
@@ -505,12 +509,12 @@ void test_enc_dec_compressed_160()
    std::vector<byte> sv_result = unlock(EC2OSP(p_G, PointGFp::COMPRESSED));
 
    CHECK( sv_result == sv_G_secp_comp);
+   return fails;
    }
 
-void test_enc_dec_compressed_256()
+size_t test_enc_dec_compressed_256()
    {
-   std::cout << "." << std::flush;
-
+   size_t fails = 0;
 
    // Test for compressed conversion (02/03) 256bit
    std::string p_secp = "ffffffff00000001000000000000000000000000ffffffffffffffffffffffff";
@@ -534,13 +538,13 @@ void test_enc_dec_compressed_256()
    std::vector<byte> sv_result = unlock(EC2OSP(p_G, PointGFp::COMPRESSED));
 
    CHECK( sv_result == sv_G_secp_comp);
+   return fails;
    }
 
 
-void test_enc_dec_uncompressed_112()
+size_t test_enc_dec_uncompressed_112()
    {
-   std::cout << "." << std::flush;
-
+   size_t fails = 0;
 
    // Test for uncompressed conversion (04) 112bit
 
@@ -565,12 +569,12 @@ void test_enc_dec_uncompressed_112()
    std::vector<byte> sv_result = unlock(EC2OSP(p_G, PointGFp::UNCOMPRESSED));
 
    CHECK( sv_result == sv_G_secp_uncomp);
+   return fails;
    }
 
-void test_enc_dec_uncompressed_521()
+size_t test_enc_dec_uncompressed_521()
    {
-   std::cout << "." << std::flush;
-
+   size_t fails = 0;
 
    // Test for uncompressed conversion(04) with big values(521 bit)
    std::string p_secp = "01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff";
@@ -597,12 +601,12 @@ void test_enc_dec_uncompressed_521()
    std::string exp_result = hex_encode(&sv_G_secp_uncomp[0], sv_G_secp_uncomp.size());
 
    CHECK_MESSAGE( sv_result == sv_G_secp_uncomp, "\ncalc. result = " << result << "\nexp. result = " << exp_result << "\n");
+   return fails;
    }
 
-void test_enc_dec_uncompressed_521_prime_too_large()
+size_t test_enc_dec_uncompressed_521_prime_too_large()
    {
-   std::cout << "." << std::flush;
-
+   size_t fails = 0;
 
    // Test for uncompressed conversion(04) with big values(521 bit)
    std::string p_secp = "01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"; // length increased by "ff"
@@ -635,11 +639,12 @@ void test_enc_dec_uncompressed_521_prime_too_large()
       }
 
    CHECK_MESSAGE(exc, "attempt of creation of point on curve with too high prime did not throw an exception");
+   return fails;
    }
 
-void test_gfp_store_restore()
+size_t test_gfp_store_restore()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    // generate point
    //EC_Group dom_pars = global_config().get_ec_dompar("1.3.132.0.8");
@@ -652,13 +657,14 @@ void test_gfp_store_restore()
    PointGFp new_p = OS2ECP(sv_mes, dom_pars.get_curve());
 
    CHECK_MESSAGE( p == new_p, "original and restored point are different!");
+   return fails;
    }
 
 
 // maybe move this test
-void test_cdc_curve_33()
+size_t test_cdc_curve_33()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    std::string G_secp_uncomp = "04081523d03d4f12cd02879dea4bf6a4f3a7df26ed888f10c5b2235a1274c386a2f218300dee6ed217841164533bcdc903f07a096f9fbf4ee95bac098a111f296f5830fe5c35b3e344d5df3a2256985f64fbe6d0edcc4c61d18bef681dd399df3d0194c5a4315e012e0245ecea56365baa9e8be1f7";
 
@@ -681,11 +687,13 @@ void test_cdc_curve_33()
       exc = true;
       }
    CHECK(!exc);
+   return fails;
    }
 
-void test_more_zeropoint()
+size_t test_more_zeropoint()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
+
    // by Falko
 
    std::string G = "024a96b5688ef573284664698968c38bb913cbfc82";
@@ -722,11 +730,12 @@ void test_more_zeropoint()
    CHECK_MESSAGE(p1 + zero == p1, "addition of zero modified point");
 
    CHECK_MESSAGE(  shouldBeZero.is_zero(), "p - q with q = p is not zero!");
+   return fails;
    }
 
-void test_mult_by_order()
+size_t test_mult_by_order()
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    // generate point
    EC_Group dom_pars(OID("1.3.132.0.8"));
@@ -734,11 +743,13 @@ void test_mult_by_order()
    PointGFp shouldBeZero = p * dom_pars.get_order();
 
    CHECK_MESSAGE(shouldBeZero.is_zero(), "G * order != O");
+   return fails;
    }
 
-void test_point_swap(RandomNumberGenerator& rng)
+size_t test_point_swap(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
+
 
    EC_Group dom_pars(OID("1.3.132.0.8"));
 
@@ -752,20 +763,21 @@ void test_point_swap(RandomNumberGenerator& rng)
    d.swap(c);
    CHECK(a == d);
    CHECK(b == c);
+   return fails;
    }
 
 /**
 * This test verifies that the side channel attack resistant multiplication function
 * yields the same result as the normal (insecure) multiplication via operator*=
 */
-void test_mult_sec_mass(RandomNumberGenerator& rng)
+size_t test_mult_sec_mass(RandomNumberGenerator& rng)
    {
+   size_t fails = 0;
+
 
    EC_Group dom_pars(OID("1.3.132.0.8"));
    for(int i = 0; i<50; i++)
       {
-      std::cout << "." << std::flush;
-      std::cout.flush();
       PointGFp a(create_random_point(rng, dom_pars.get_curve()));
       BigInt scal(BigInt(rng, 40));
       PointGFp b = a * scal;
@@ -774,54 +786,59 @@ void test_mult_sec_mass(RandomNumberGenerator& rng)
       c *= scal;
       CHECK(b == c);
       }
+   return fails;
    }
 
-void test_curve_cp_ctor()
+size_t test_curve_cp_ctor()
    {
-   std::cout << "." << std::flush;
-
-   EC_Group dom_pars(OID("1.3.132.0.8"));
-   CurveGFp curve(dom_pars.get_curve());
-   }
+   try
+      {
+      EC_Group dom_pars(OID("1.3.132.0.8"));
+      CurveGFp curve(dom_pars.get_curve());
+      }
+   catch(...)
+      {
+      return 1;
 
-}
+      }
 
-void do_ec_tests(RandomNumberGenerator& rng)
-   {
-   std::cout << "Testing ECC: " << std::flush;
-
-   test_point_turn_on_sp_red_mul();
-   test_coordinates();
-   test_point_transformation ();
-   test_point_mult ();
-   test_point_negative();
-   test_zeropoint();
-   test_zeropoint_enc_dec();
-   test_calc_with_zeropoint();
-   test_add_point();
-   test_sub_point();
-   test_mult_point();
-   test_basic_operations();
-   test_enc_dec_compressed_160();
-   test_enc_dec_compressed_256();
-   test_enc_dec_uncompressed_112();
-   test_enc_dec_uncompressed_521();
-   test_enc_dec_uncompressed_521_prime_too_large();
-   test_gfp_store_restore();
-   test_cdc_curve_33();
-   test_more_zeropoint();
-   test_mult_by_order();
-   test_point_swap(rng);
-   test_mult_sec_mass(rng);
-   test_curve_cp_ctor();
-
-   std::cout << std::endl;
+   return 0;
    }
 
-#else
+}
 
-void do_ec_tests(Botan::RandomNumberGenerator& rng)
+size_t test_ecc()
    {
+   AutoSeeded_RNG rng;
+
+   size_t fails = 0;
+
+   fails += test_point_turn_on_sp_red_mul();
+   fails += test_coordinates();
+   fails += test_point_transformation ();
+   fails += test_point_mult ();
+   fails += test_point_negative();
+   fails += test_zeropoint();
+   fails += test_zeropoint_enc_dec();
+   fails += test_calc_with_zeropoint();
+   fails += test_add_point();
+   fails += test_sub_point();
+   fails += test_mult_point();
+   fails += test_basic_operations();
+   fails += test_enc_dec_compressed_160();
+   fails += test_enc_dec_compressed_256();
+   fails += test_enc_dec_uncompressed_112();
+   fails += test_enc_dec_uncompressed_521();
+   fails += test_enc_dec_uncompressed_521_prime_too_large();
+   fails += test_gfp_store_restore();
+   fails += test_cdc_curve_33();
+   fails += test_more_zeropoint();
+   fails += test_mult_by_order();
+   fails += test_point_swap(rng);
+   fails += test_mult_sec_mass(rng);
+   fails += test_curve_cp_ctor();
+
+   test_report("ECC", 0, fails);
+
+   return fails;
    }
-
-#endif
diff --git a/checks/ecdh.cpp b/checks/ecdh.cpp
index 178282047..30139b27e 100644
--- a/checks/ecdh.cpp
+++ b/checks/ecdh.cpp
@@ -7,16 +7,14 @@
 * Distributed under the terms of the Botan license
 */
 
-#include <botan/build.h>
+#include "tests.h"
 
-#include "validate.h"
 #include "common.h"
 
-#if defined(BOTAN_HAS_ECDH) &&  defined(BOTAN_HAS_X509)
-
 #include <iostream>
 #include <fstream>
 
+#include <botan/auto_rng.h>
 #include <botan/pubkey.h>
 #include <botan/ecdh.h>
 #include <botan/x509self.h>
@@ -24,14 +22,14 @@
 
 using namespace Botan;
 
-#define CHECK_MESSAGE(expr, print) try { if(!(expr)) std::cout << print << "\n"; } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
-#define CHECK(expr) try { if(!(expr)) std::cout << #expr << "\n"; } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
+#define CHECK_MESSAGE(expr, print) try { if(!(expr)) { ++fails; std::cout << print << "\n"; } } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
+#define CHECK(expr) try { if(!(expr)) { ++fails; std::cout << #expr << "\n"; } } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
 
 namespace {
 
-void test_ecdh_normal_derivation(RandomNumberGenerator& rng)
+size_t test_ecdh_normal_derivation(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    EC_Group dom_pars(OID("1.3.132.0.8"));
 
@@ -50,11 +48,16 @@ void test_ecdh_normal_derivation(RandomNumberGenerator& rng)
       std::cout << "The two keys didn't match!\n";
       std::cout << "Alice's key was: " << alice_key.as_string() << "\n";
       std::cout << "Bob's key was: " << bob_key.as_string() << "\n";
+      ++fails;
       }
+
+   return fails;
    }
 
-void test_ecdh_some_dp(RandomNumberGenerator& rng)
+size_t test_ecdh_some_dp(RandomNumberGenerator& rng)
    {
+   size_t fails = 0;
+
    std::vector<std::string> oids;
    oids.push_back("1.2.840.10045.3.1.7");
    oids.push_back("1.3.132.0.8");
@@ -62,8 +65,6 @@ void test_ecdh_some_dp(RandomNumberGenerator& rng)
 
    for(u32bit i = 0; i< oids.size(); i++)
       {
-      std::cout << "." << std::flush;
-
       OID oid(oids[i]);
       EC_Group dom_pars(oid);
 
@@ -79,10 +80,13 @@ void test_ecdh_some_dp(RandomNumberGenerator& rng)
       CHECK_MESSAGE(alice_key == bob_key, "different keys - " << "Alice's key was: " << alice_key.as_string() << ", Bob's key was: " << bob_key.as_string());
       }
 
+   return fails;
    }
 
-void test_ecdh_der_derivation(RandomNumberGenerator& rng)
+size_t test_ecdh_der_derivation(RandomNumberGenerator& rng)
    {
+   size_t fails = 0;
+
    std::vector<std::string> oids;
    oids.push_back("1.2.840.10045.3.1.7");
    oids.push_back("1.3.132.0.8");
@@ -106,25 +110,25 @@ void test_ecdh_der_derivation(RandomNumberGenerator& rng)
       SymmetricKey bob_key = kb.derive_key(32, key_a);
 
       CHECK_MESSAGE(alice_key == bob_key, "different keys - " << "Alice's key was: " << alice_key.as_string() << ", Bob's key was: " << bob_key.as_string());
-      //cout << "key: " << alice_key.as_string() << endl;
+
       }
+
+   return fails;
    }
 
 }
 
-u32bit do_ecdh_tests(RandomNumberGenerator& rng)
+size_t test_ecdh()
    {
-   std::cout << "Testing ECDH (InSiTo unit tests): ";
+   size_t fails = 0;
 
-   test_ecdh_normal_derivation(rng);
-   test_ecdh_some_dp(rng);
-   test_ecdh_der_derivation(rng);
+   AutoSeeded_RNG rng;
 
-   std::cout << std::endl;
+   fails += test_ecdh_normal_derivation(rng);
+   fails += test_ecdh_some_dp(rng);
+   fails += test_ecdh_der_derivation(rng);
 
-   return 0;
-   }
+   test_report("ECDH", 3, fails);
 
-#else
-u32bit do_ecdh_tests(RandomNumberGenerator&) { return 0; }
-#endif
+   return fails;
+   }
diff --git a/checks/ecdsa.cpp b/checks/ecdsa.cpp
index 2506e1389..dccb17813 100644
--- a/checks/ecdsa.cpp
+++ b/checks/ecdsa.cpp
@@ -6,9 +6,7 @@
 *     2008 Jack Lloyd                                 *
 ******************************************************/
 
-#include "validate.h"
-
-#if defined(BOTAN_HAS_ECDSA) && defined(BOTAN_HAS_X509)
+#include "tests.h"
 
 #include <botan/botan.h>
 #include <botan/pubkey.h>
@@ -27,8 +25,8 @@ using namespace Botan;
 
 #define TEST_DATA_DIR "checks/ecc_testdata"
 
-#define CHECK_MESSAGE(expr, print) try { if(!(expr)) std::cout << print << "\n"; } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
-#define CHECK(expr) try { if(!(expr)) std::cout << #expr << "\n"; } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
+#define CHECK_MESSAGE(expr, print) try { if(!(expr)) { ++fails; std::cout << print << "\n"; } } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
+#define CHECK(expr) try { if(!(expr)) { ++fails; std::cout << #expr << "\n"; } } catch(std::exception& e) { std::cout << __FUNCTION__ << ": " << e.what() << "\n"; }
 
 namespace {
 
@@ -44,14 +42,13 @@ std::string to_hex(const std::vector<byte>& bin)
 * value) is larger than n, the order of the base point.  Tests the
 * signing function of the pk signer object */
 
-void test_hash_larger_than_n(RandomNumberGenerator& rng)
+size_t test_hash_larger_than_n(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
-
    EC_Group dom_pars(OID("1.3.132.0.8")); // secp160r1
    // n = 0x0100000000000000000001f4c8f927aed3ca752257 (21 bytes)
    // -> shouldn't work with SHA224 which outputs 28 bytes
 
+   size_t fails = 0;
    ECDSA_PrivateKey priv_key(rng, dom_pars);
 
    std::vector<byte> message(20);
@@ -90,14 +87,19 @@ void test_hash_larger_than_n(RandomNumberGenerator& rng)
 
    // verify against EMSA1_BSI
    if(pk_verifier.verify_message(message, signature))
+      {
       std::cout << "Corrupt ECDSA signature verified, should not have\n";
+      ++fails;
+      }
+
+   return fails;
    }
 
-void test_decode_ecdsa_X509()
+size_t test_decode_ecdsa_X509()
    {
-   std::cout << "." << std::flush;
-
    X509_Certificate cert(TEST_DATA_DIR "/CSCA.CSCA.csca-germany.1.crt");
+   size_t fails = 0;
+
    CHECK_MESSAGE(OIDS::lookup(cert.signature_algorithm().oid) == "ECDSA/EMSA1(SHA-224)", "error reading signature algorithm from x509 ecdsa certificate");
 
    CHECK_MESSAGE(to_hex(cert.serial_number()) == "01", "error reading serial from x509 ecdsa certificate");
@@ -107,39 +109,40 @@ void test_decode_ecdsa_X509()
    std::unique_ptr<X509_PublicKey> pubkey(cert.subject_public_key());
    bool ver_ec = cert.check_signature(*pubkey);
    CHECK_MESSAGE(ver_ec, "could not positively verify correct selfsigned x509-ecdsa certificate");
+
+   return fails;
    }
 
-void test_decode_ver_link_SHA256()
+size_t test_decode_ver_link_SHA256()
    {
-   std::cout << "." << std::flush;
-
    X509_Certificate root_cert(TEST_DATA_DIR "/root2_SHA256.cer");
    X509_Certificate link_cert(TEST_DATA_DIR "/link_SHA256.cer");
 
+   size_t fails = 0;
    std::unique_ptr<X509_PublicKey> pubkey(root_cert.subject_public_key());
    bool ver_ec = link_cert.check_signature(*pubkey);
    CHECK_MESSAGE(ver_ec, "could not positively verify correct SHA256 link x509-ecdsa certificate");
-
+   return fails;
    }
-void test_decode_ver_link_SHA1()
-   {
-   std::cout << "." << std::flush;
 
+size_t test_decode_ver_link_SHA1()
+   {
    X509_Certificate root_cert(TEST_DATA_DIR "/root_SHA1.163.crt");
    X509_Certificate link_cert(TEST_DATA_DIR "/link_SHA1.166.crt");
 
+   size_t fails = 0;
    std::unique_ptr<X509_PublicKey> pubkey(root_cert.subject_public_key());
    bool ver_ec = link_cert.check_signature(*pubkey);
    CHECK_MESSAGE(ver_ec, "could not positively verify correct SHA1 link x509-ecdsa certificate");
+   return fails;
    }
 
-void test_sign_then_ver(RandomNumberGenerator& rng)
+size_t test_sign_then_ver(RandomNumberGenerator& rng)
    {
-   std::cout << '.' << std::flush;
-
    EC_Group dom_pars(OID("1.3.132.0.8"));
    ECDSA_PrivateKey ecdsa(rng, dom_pars);
 
+   size_t fails = 0;
    PK_Signer signer(ecdsa, "EMSA1(SHA-1)");
 
    auto msg = hex_decode("12345678901234567890abcdef12");
@@ -150,18 +153,26 @@ void test_sign_then_ver(RandomNumberGenerator& rng)
    bool ok = verifier.verify_message(msg, sig);
 
    if(!ok)
+      {
       std::cout << "ERROR: Could not verify ECDSA signature\n";
+      fails++;
+      }
 
    sig[0]++;
    ok = verifier.verify_message(msg, sig);
 
    if(ok)
+      {
       std::cout << "ERROR: Bogus ECDSA signature verified anyway\n";
+      fails++;
+      }
+
+   return fails;
    }
 
-bool test_ec_sign(RandomNumberGenerator& rng)
+size_t test_ec_sign(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    try
       {
@@ -181,7 +192,7 @@ bool test_ec_sign(RandomNumberGenerator& rng)
       if(!verifier.check_signature(sig))
          {
          std::cout << "ECDSA self-test failed!";
-         return false;
+         ++fails;
          }
 
       // now check valid signature, different input
@@ -191,7 +202,7 @@ bool test_ec_sign(RandomNumberGenerator& rng)
       if(verifier.check_signature(sig))
          {
          std::cout << "ECDSA with bad input passed validation";
-         return false;
+         ++fails;
          }
 
       // now check with original input, modified signature
@@ -203,21 +214,22 @@ bool test_ec_sign(RandomNumberGenerator& rng)
       if(verifier.check_signature(sig))
          {
          std::cout << "ECDSA with bad signature passed validation";
-         return false;
+         ++fails;
          }
       }
    catch (std::exception& e)
       {
       std::cout << "Exception in test_ec_sign - " << e.what() << "\n";
-      return false;
+      ++fails;
       }
-   return true;
+
+   return fails;
    }
 
 
-void test_create_pkcs8(RandomNumberGenerator& rng)
+size_t test_create_pkcs8(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    try
       {
@@ -239,12 +251,15 @@ void test_create_pkcs8(RandomNumberGenerator& rng)
    catch (std::exception& e)
       {
       std::cout << "Exception: " << e.what() << std::endl;
+      ++fails;
       }
+
+   return fails;
    }
 
-void test_create_and_verify(RandomNumberGenerator& rng)
+size_t test_create_and_verify(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
+   size_t fails = 0;
 
    EC_Group dom_pars(OID("1.3.132.0.8"));
    ECDSA_PrivateKey key(rng, dom_pars);
@@ -294,10 +309,13 @@ void test_create_and_verify(RandomNumberGenerator& rng)
    if(!dynamic_cast<ECDSA_PrivateKey*>(loaded_key.get()))
       {
       std::cout << "Failed to reload an ECDSA key with unusual parameter set\n";
+      ++fails;
       }
+
+   return fails;
    }
 
-void test_curve_registry(RandomNumberGenerator& rng)
+size_t test_curve_registry(RandomNumberGenerator& rng)
    {
    std::vector<std::string> oids;
    oids.push_back("1.3.132.0.8");
@@ -320,7 +338,7 @@ void test_curve_registry(RandomNumberGenerator& rng)
    oids.push_back("1.3.132.0.10");
    oids.push_back("1.3.132.0.34");
    oids.push_back("1.3.132.0.35");
-   oids.push_back("1.3.6.1.4.1.8301.3.1.2.9.0.38");
+   //oids.push_back("1.3.6.1.4.1.8301.3.1.2.9.0.38");
    oids.push_back("1.3.36.3.3.2.8.1.1.1");
    oids.push_back("1.3.36.3.3.2.8.1.1.3");
    oids.push_back("1.3.36.3.3.2.8.1.1.5");
@@ -329,11 +347,11 @@ void test_curve_registry(RandomNumberGenerator& rng)
    oids.push_back("1.3.36.3.3.2.8.1.1.11");
    oids.push_back("1.3.36.3.3.2.8.1.1.13");
 
+   size_t fails = 0;
+
    unsigned int i;
    for (i = 0; i < oids.size(); i++)
       {
-      std::cout << "." << std::flush;
-
       try
          {
          OID oid(oids[i]);
@@ -347,21 +365,24 @@ void test_curve_registry(RandomNumberGenerator& rng)
          std::vector<byte> sig = signer.sign_message(msg, rng);
 
          if(!verifier.verify_message(msg, sig))
+            {
             std::cout << "Failed testing ECDSA sig for curve " << oids[i] << "\n";
+            ++fails;
+            }
          }
       catch(Invalid_Argument& e)
          {
          std::cout << "Error testing curve " << oids[i] << " - " << e.what() << "\n";
+         ++fails;
          }
       }
-   //  std::cout << "test_curve_registry finished" << endl;
+   return fails;
    }
 
-void test_read_pkcs8(RandomNumberGenerator& rng)
+size_t test_read_pkcs8(RandomNumberGenerator& rng)
    {
-   std::cout << "." << std::flush;
-
    auto msg = hex_decode("12345678901234567890abcdef12");
+   size_t fails = 0;
 
    try
       {
@@ -380,6 +401,7 @@ void test_read_pkcs8(RandomNumberGenerator& rng)
       }
    catch (std::exception& e)
       {
+      ++fails;
       std::cout << "Exception in test_read_pkcs8 - " << e.what() << "\n";
       }
 
@@ -404,56 +426,66 @@ void test_read_pkcs8(RandomNumberGenerator& rng)
             PKCS8::load_key(TEST_DATA_DIR "/withdompar_private.pkcs8.pem", rng));
 
          std::cout << "Unexpected success: loaded key with unknown OID\n";
+         ++fails;
          }
       catch (std::exception) { /* OK */ }
       }
    catch (std::exception& e)
       {
       std::cout << "Exception in test_read_pkcs8 - " << e.what() << "\n";
+      ++fails;
       }
+
+   return fails;
    }
 
-void test_ecc_key_with_rfc5915_extensions(RandomNumberGenerator& rng)
+size_t test_ecc_key_with_rfc5915_extensions(RandomNumberGenerator& rng)
    {
    const std::string pw = "G3bz1L1gmB5ULietOZdoLPu63D7uwTLMEk";
 
+   size_t fails = 0;
+
    try
       {
       std::unique_ptr<PKCS8_PrivateKey> pkcs8(
          PKCS8::load_key(TEST_DATA_DIR "/ecc_private_with_rfc5915_ext.pem", rng, pw));
 
       if(!dynamic_cast<ECDSA_PrivateKey*>(pkcs8.get()))
+         {
          std::cout << "Loaded RFC 5915 key, but got something other than an ECDSA key\n";
+         ++fails;
+         }
       }
    catch(std::exception& e)
       {
       std::cout << "Exception in " << __func__ << " - " << e.what() << "\n";
+      ++fails;
       }
+
+   return fails;
    }
 
 }
 
-u32bit do_ecdsa_tests(Botan::RandomNumberGenerator& rng)
+size_t test_ecdsa()
    {
-   std::cout << "Testing ECDSA (InSiTo unit tests): ";
+   size_t fails = 0;
 
-   test_hash_larger_than_n(rng);
-   test_decode_ecdsa_X509();
-   test_decode_ver_link_SHA256();
-   test_decode_ver_link_SHA1();
-   test_sign_then_ver(rng);
-   test_ec_sign(rng);
-   test_create_pkcs8(rng);
-   test_create_and_verify(rng);
-   test_curve_registry(rng);
-   test_read_pkcs8(rng);
+   AutoSeeded_RNG rng;
 
-   test_ecc_key_with_rfc5915_extensions(rng);
+   fails += test_hash_larger_than_n(rng);
+   fails += test_decode_ecdsa_X509();
+   fails += test_decode_ver_link_SHA256();
+   fails += test_decode_ver_link_SHA1();
+   fails += test_sign_then_ver(rng);
+   fails += test_ec_sign(rng);
+   fails += test_create_pkcs8(rng);
+   fails += test_create_and_verify(rng);
+   fails += test_curve_registry(rng);
+   fails += test_read_pkcs8(rng);
+   fails += test_ecc_key_with_rfc5915_extensions(rng);
 
-   std::cout << std::endl;
+   test_report("ECDSA", 11, fails);
 
-   return 0;
+   return fails;
    }
-#else
-u32bit do_ecdsa_tests(Botan::RandomNumberGenerator&) { return 0; }
-#endif
diff --git a/checks/pk.cpp b/checks/pk.cpp
index a9093848a..e33caceef 100644
--- a/checks/pk.cpp
+++ b/checks/pk.cpp
@@ -4,6 +4,9 @@
 * Distributed under the terms of the Botan license
 */
 
+#include "tests.h"
+#include "common.h"
+
 #include <iostream>
 #include <fstream>
 #include <string>
@@ -65,9 +68,6 @@
 #include <botan/numthry.h>
 using namespace Botan;
 
-#include "common.h"
-#include "validate.h"
-
 namespace {
 
 BigInt to_bigint(std::string input)
@@ -96,11 +96,12 @@ void dump_data(const std::vector<byte>& out,
    std::cout << "Exp: " << pipe.read_all_as_string(1) << std::endl;
    }
 
-void validate_save_and_load(const Private_Key* priv_key,
-                            RandomNumberGenerator& rng)
+size_t validate_save_and_load(const Private_Key* priv_key,
+                              RandomNumberGenerator& rng)
    {
    std::string name = priv_key->algo_name();
 
+   size_t fails = 0;
    std::string pub_pem = X509::PEM_encode(*priv_key);
 
    try
@@ -109,15 +110,22 @@ void validate_save_and_load(const Private_Key* priv_key,
       std::auto_ptr<Public_Key> restored_pub(X509::load_key(input_pub));
 
       if(!restored_pub.get())
+         {
          std::cout << "Could not recover " << name << " public key\n";
+         ++fails;
+         }
       else if(restored_pub->check_key(rng, true) == false)
+         {
          std::cout << "Restored pubkey failed self tests " << name << "\n";
+         ++fails;
+         }
       }
    catch(std::exception& e)
       {
       std::cout << "Exception during load of " << name
                 << " key: " << e.what() << "\n";
       std::cout << "PEM for pubkey was:\n" << pub_pem << "\n";
+      ++fails;
       }
 
    std::string priv_pem = PKCS8::PEM_encode(*priv_key);
@@ -129,16 +137,25 @@ void validate_save_and_load(const Private_Key* priv_key,
          PKCS8::load_key(input_priv, rng));
 
       if(!restored_priv.get())
+         {
          std::cout << "Could not recover " << name << " privlic key\n";
+         ++fails;
+         }
       else if(restored_priv->check_key(rng, true) == false)
+         {
          std::cout << "Restored privkey failed self tests " << name << "\n";
+         ++fails;
+         }
       }
    catch(std::exception& e)
       {
       std::cout << "Exception during load of " << name
                 << " key: " << e.what() << "\n";
       std::cout << "PEM for privkey was:\n" << priv_pem << "\n";
+      ++fails;
       }
+
+   return fails;
    }
 
 void validate_decryption(PK_Decryptor& d, const std::string& algo,
@@ -689,32 +706,37 @@ u32bit validate_dlies(const std::string& algo,
 #endif
    }
 
-void do_pk_keygen_tests(RandomNumberGenerator& rng)
+}
+
+size_t test_pk_keygen()
    {
-   std::cout << "Testing PK key generation: " << std::flush;
+   AutoSeeded_RNG rng;
+
+   size_t fails = 0;
 
 #define DL_KEY(TYPE, GROUP)                 \
    {                                        \
    TYPE key(rng, DL_Group(GROUP));          \
    key.check_key(rng, true);                \
-   validate_save_and_load(&key, rng);       \
-   std::cout << '.' << std::flush;          \
+   fails += validate_save_and_load(&key, rng);       \
    }
 
 #define EC_KEY(TYPE, GROUP)                 \
    {                                        \
    TYPE key(rng, EC_Group(OIDS::lookup(GROUP)));        \
    key.check_key(rng, true);                \
-   validate_save_and_load(&key, rng);       \
-   std::cout << '.' << std::flush;          \
+   fails += validate_save_and_load(&key, rng);       \
    }
 
 #if defined(BOTAN_HAS_RSA)
       {
       RSA_PrivateKey rsa1024(rng, 1024);
       rsa1024.check_key(rng, true);
-      validate_save_and_load(&rsa1024, rng);
-      std::cout << '.' << std::flush;
+      fails += validate_save_and_load(&rsa1024, rng);
+
+      RSA_PrivateKey rsa2048(rng, 2048);
+      rsa2048.check_key(rng, true);
+      fails += validate_save_and_load(&rsa2048, rng);
       }
 #endif
 
@@ -722,8 +744,7 @@ void do_pk_keygen_tests(RandomNumberGenerator& rng)
       {
       RW_PrivateKey rw1024(rng, 1024);
       rw1024.check_key(rng, true);
-      validate_save_and_load(&rw1024, rng);
-      std::cout << '.' << std::flush;
+      fails += validate_save_and_load(&rw1024, rng);
       }
 #endif
 
@@ -776,27 +797,26 @@ void do_pk_keygen_tests(RandomNumberGenerator& rng)
    EC_KEY(GOST_3410_PrivateKey, "secp521r1");
 #endif
 
-   std::cout << std::endl;
+   return fails;
    }
 
-}
-
-u32bit do_pk_validation_tests(const std::string& filename,
-                              RandomNumberGenerator& rng)
+size_t test_pubkey()
    {
+   AutoSeeded_RNG rng;
+   const std::string filename = "checks/pk_valid.dat";
    std::ifstream test_data(filename.c_str());
 
    if(!test_data)
       throw Botan::Stream_IO_Error("Couldn't open test file " + filename);
 
-   u32bit errors = 0, alg_count = 0;
+   size_t total_errors = 0;
+   size_t errors = 0, alg_count = 0, total_tests = 0;
    std::string algorithm, print_algorithm;
 
    while(!test_data.eof())
       {
       if(test_data.bad() || test_data.fail())
-         throw Botan::Stream_IO_Error("File I/O error reading from " +
-                                      filename);
+         throw std::runtime_error("File I/O error reading from " + filename);
 
       std::string line;
       std::getline(test_data, line);
@@ -818,7 +838,7 @@ u32bit do_pk_validation_tests(const std::string& filename,
 
       if(line[0] == '[' && line[line.size() - 1] == ']')
          {
-         std::string old_algo = print_algorithm;
+         const std::string old_algo = print_algorithm;
          algorithm = line.substr(1, line.size() - 2);
          print_algorithm = algorithm;
          if(print_algorithm.find("_PKCS8") != std::string::npos)
@@ -830,22 +850,18 @@ u32bit do_pk_validation_tests(const std::string& filename,
 
          if(old_algo != print_algorithm && old_algo != "")
             {
-            std::cout << std::endl;
+            test_report(old_algo, alg_count, errors);
             alg_count = 0;
+            total_errors += errors;
+            errors = 0;
             }
 
-         if(old_algo != print_algorithm)
-            std::cout << "Testing " << print_algorithm << ": ";
          continue;
          }
 
       std::vector<std::string> substr = parse(line);
 
-#if 0
-      std::cout << "Testing: " << print_algorithm << std::endl;
-#endif
-
-      u32bit new_errors = 0;
+      size_t new_errors = 0;
 
       try
          {
@@ -885,38 +901,28 @@ u32bit do_pk_validation_tests(const std::string& filename,
          else if(algorithm.find("DLIES/") == 0)
             new_errors = validate_dlies(algorithm, substr, rng);
          else
+            {
             std::cout << "WARNING: Unknown PK algorithm "
                       << algorithm << std::endl;
-
-         if(new_errors == 0) // OK
-            std::cout << '.';
-         else if(new_errors == 1) // test failed
-            std::cout << 'X';
-         else if(new_errors == 2) // unknown algo
-            std::cout << '?';
-
-         std::cout.flush();
+            ++new_errors;
+            }
 
          alg_count++;
-         if(new_errors == 1)
-            errors += new_errors;
+         total_tests++;
+         errors += new_errors;
          }
       catch(std::exception& e)
          {
          std::cout << "Exception: " << e.what() << "\n";
+         new_errors++;
          }
 
-      if(new_errors == 1)
+      if(new_errors)
          std::cout << "ERROR: \"" << algorithm << "\" failed test #"
                    << std::dec << alg_count << std::endl;
       }
 
-   std::cout << std::endl;
-
-   do_ec_tests(rng);
-   errors += do_ecdsa_tests(rng);
-   errors += do_ecdh_tests(rng);
-   do_pk_keygen_tests(rng);
+   test_report("Pubkey", total_tests, errors);
 
    return errors;
    }
diff --git a/checks/tests.cpp b/checks/tests.cpp
index f9b5545ce..2692d9e57 100644
--- a/checks/tests.cpp
+++ b/checks/tests.cpp
@@ -5,13 +5,31 @@ size_t run_tests(const std::vector<test_fn>& tests)
    {
    size_t fails = 0;
    for(auto& test : tests)
-      fails += test();
+      {
+      try
+         {
+         fails += test();
+         }
+      catch(std::exception& e)
+         {
+         std::cout << "Exception escaped callback: " << e.what() << "\n";
+         ++fails;
+         }
+      }
    return fails;
    }
 
 void test_report(const std::string& name, size_t ran, size_t failed)
    {
-   std::cout << name << " tests: " << ran << " completed " << failed << " failed\n";
+   std::cout << name << " tests:";
+
+   if(ran > 0)
+      std::cout << " " << ran << " total";
+
+   if(failed)
+      std::cout << " " << failed << " FAILs\n";
+   else
+      std::cout << " all ok\n";
    }
 
 size_t run_tests_bb(std::istream& src,
@@ -27,8 +45,8 @@ size_t run_tests_bb(std::istream& src,
       }
 
    std::map<std::string, std::string> vars;
-   size_t test_cnt = 0;
-   size_t test_fail = 0;
+   size_t test_fails = 0, algo_fail = 0;
+   size_t test_count = 0, algo_count = 0;
 
    std::string fixed_name;
 
@@ -45,6 +63,13 @@ size_t run_tests_bb(std::istream& src,
 
       if(line[0] == '[' && line[line.size()-1] == ']')
          {
+         if(fixed_name != "")
+            test_report(fixed_name, algo_count, algo_fail);
+
+         test_count += algo_count;
+         test_fails += algo_fail;
+         algo_count = 0;
+         algo_fail = 0;
          fixed_name = line.substr(1, line.size() - 2);
          vars[name_key] = fixed_name;
          continue;
@@ -61,19 +86,19 @@ size_t run_tests_bb(std::istream& src,
       if(key == output_key)
          {
          //std::cout << vars[name_key] << " " << test_cnt << "\n";
-         ++test_cnt;
+         ++algo_count;
          try
             {
             if(!cb(vars))
                {
-               std::cout << vars[name_key] << " test " << test_cnt << " failed\n";
-               ++test_fail;
+               std::cout << vars[name_key] << " test " << algo_count << " failed\n";
+               ++algo_fail;
                }
             }
          catch(std::exception& e)
             {
-            std::cout << vars[name_key] << " test " << test_cnt << " failed: " << e.what() << "\n";
-            ++test_fail;
+            std::cout << vars[name_key] << " test " << algo_count << " failed: " << e.what() << "\n";
+            ++algo_fail;
             }
 
          if(clear_between_cb)
@@ -84,9 +109,9 @@ size_t run_tests_bb(std::istream& src,
          }
       }
 
-   test_report(name_key, test_cnt, test_fail);
+   test_report(name_key, test_count, test_fails);
 
-   return test_fail;
+   return test_fails;
    }
 
 size_t run_tests(std::istream& src,
@@ -135,5 +160,16 @@ size_t run_all_tests()
    all_tests.push_back(test_bcrypt);
    all_tests.push_back(test_cryptobox);
 
+   all_tests.push_back(test_bigint);
+   all_tests.push_back(test_pubkey);
+
+   all_tests.push_back(test_ecc);
+   all_tests.push_back(test_ecdsa);
+   all_tests.push_back(test_ecdh);
+   all_tests.push_back(test_pk_keygen);
+   all_tests.push_back(test_cvc);
+   all_tests.push_back(test_x509);
+   all_tests.push_back(test_tls);
+
    return run_tests(all_tests);
    }
diff --git a/checks/tests.h b/checks/tests.h
index d0c81b62e..ae842f8a0 100644
--- a/checks/tests.h
+++ b/checks/tests.h
@@ -54,4 +54,19 @@ size_t test_bcrypt();
 size_t test_passhash9();
 size_t test_cryptobox();
 
+size_t test_bigint();
+
+size_t test_pubkey();
+size_t test_pk_keygen();
+
+size_t test_ecc();
+
+size_t test_ecdsa();
+size_t test_ecdh();
+
+size_t test_x509();
+size_t test_cvc();
+
+size_t test_tls();
+
 #endif
diff --git a/checks/tls.cpp b/checks/tls.cpp
index c0b85cc7e..9958ed3b7 100644
--- a/checks/tls.cpp
+++ b/checks/tls.cpp
@@ -1,4 +1,5 @@
-#include "validate.h"
+
+#include "tests.h"
 
 #if defined(BOTAN_HAS_TLS)
 
@@ -135,8 +136,9 @@ Credentials_Manager* create_creds(RandomNumberGenerator& rng)
    }
 
 
-void test_handshake(RandomNumberGenerator& rng)
+size_t test_handshake()
    {
+   AutoSeeded_RNG rng;
    TLS::Policy default_policy;
 
    std::auto_ptr<Credentials_Manager> creds(create_creds(rng));
@@ -148,14 +150,17 @@ void test_handshake(RandomNumberGenerator& rng)
 
    auto handshake_complete = [](const TLS::Session& session) -> bool
    {
-      std::cout << "Handshake complete, " << session.version().to_string()
-      << " using " << session.ciphersuite().to_string() << "\n";
+      if(false)
+         {
+         std::cout << "Handshake complete, " << session.version().to_string()
+                   << " using " << session.ciphersuite().to_string() << "\n";
 
-      if(!session.session_id().empty())
-         std::cout << "Session ID " << hex_encode(session.session_id()) << "\n";
+         if(!session.session_id().empty())
+            std::cout << "Session ID " << hex_encode(session.session_id()) << "\n";
 
-      if(!session.session_ticket().empty())
-         std::cout << "Session ticket " << hex_encode(session.session_ticket()) << "\n";
+         if(!session.session_ticket().empty())
+            std::cout << "Session ticket " << hex_encode(session.session_ticket()) << "\n";
+         }
 
       return true;
    };
@@ -239,7 +244,7 @@ void test_handshake(RandomNumberGenerator& rng)
          if(c2s_data[0] != '1')
             {
             std::cout << "Error\n";
-            break;
+            return 1;
             }
          }
 
@@ -248,7 +253,7 @@ void test_handshake(RandomNumberGenerator& rng)
          if(s2c_data[0] != '2')
             {
             std::cout << "Error\n";
-            break;
+            return 1;
             }
          }
 
@@ -259,19 +264,15 @@ void test_handshake(RandomNumberGenerator& rng)
 
 }
 
-size_t do_tls_tests(RandomNumberGenerator& rng)
+size_t test_tls()
    {
    size_t errors = 0;
 
-   std::cout << "TLS tests: ";
-
-   test_handshake(rng);
-
-   std::cout << std::endl;
+   errors += test_handshake(rng);
 
    return errors;
    }
 
 #else
-size_t do_tls_tests(RandomNumberGenerator&) { return 0; }
+size_t test_tls() { return 0; }
 #endif
diff --git a/checks/validate.h b/checks/validate.h
deleted file mode 100644
index f32e7a098..000000000
--- a/checks/validate.h
+++ /dev/null
@@ -1,32 +0,0 @@
-
-#ifndef BOTAN_TEST_VALIDATE_H__
-#define BOTAN_TEST_VALIDATE_H__
-
-#include <botan/types.h>
-#include <botan/rng.h>
-#include <string>
-#include <functional>
-#include <istream>
-#include <map>
-
-using Botan::RandomNumberGenerator;
-
-using Botan::u32bit;
-
-u32bit do_bigint_tests(const std::string&,
-                       RandomNumberGenerator& rng);
-
-u32bit do_pk_validation_tests(const std::string&,
-                              RandomNumberGenerator&);
-
-void do_ec_tests(RandomNumberGenerator& rng);
-
-u32bit do_ecdsa_tests(RandomNumberGenerator& rng);
-u32bit do_ecdh_tests(RandomNumberGenerator& rng);
-u32bit do_cvc_tests(RandomNumberGenerator& rng);
-
-void do_x509_tests(RandomNumberGenerator&);
-
-size_t do_tls_tests(RandomNumberGenerator& rng);
-
-#endif
diff --git a/checks/x509.cpp b/checks/x509.cpp
index 77d4e803f..d5f8dfc7b 100644
--- a/checks/x509.cpp
+++ b/checks/x509.cpp
@@ -4,8 +4,11 @@
 * Distributed under the terms of the Botan license
 */
 
+#include "tests.h"
+#include "common.h"
 
 #include <botan/filters.h>
+#include <botan/auto_rng.h>
 
 #if defined(BOTAN_HAS_RSA)
   #include <botan/rsa.h>
@@ -31,8 +34,6 @@ using namespace Botan;
 #include <iostream>
 #include <memory>
 
-#include "validate.h"
-#include "common.h"
 
 #if defined(BOTAN_HAS_X509_CERTIFICATES) && \
     defined(BOTAN_HAS_RSA) && \
@@ -129,34 +130,29 @@ u32bit check_against_copy(const Private_Key& orig,
 
 }
 
-void do_x509_tests(RandomNumberGenerator& rng)
+size_t test_x509()
    {
-   std::cout << "Testing X.509 CA/CRL/cert/cert request: " << std::flush;
+   AutoSeeded_RNG rng;
+   const std::string hash_fn = "SHA-256";
 
-   std::string hash_fn = "SHA-256";
+   size_t fails = 0;
 
    /* Create the CA's key and self-signed cert */
-   std::cout << '.' << std::flush;
    RSA_PrivateKey ca_key(rng, 2048);
 
-   std::cout << '.' << std::flush;
    X509_Certificate ca_cert = X509::create_self_signed_cert(ca_opts(),
                                                             ca_key,
                                                             hash_fn,
                                                             rng);
-   std::cout << '.' << std::flush;
-
    /* Create user #1's key and cert request */
    DSA_PrivateKey user1_key(rng, DL_Group("dsa/botan/2048"));
 
-   std::cout << '.' << std::flush;
    PKCS10_Request user1_req = X509::create_cert_req(req_opts1(),
                                                     user1_key,
                                                     "SHA-1",
                                                     rng);
 
    /* Create user #2's key and cert request */
-   std::cout << '.' << std::flush;
 #if defined(BOTAN_HAS_ECDSA)
    EC_Group ecc_domain(OID("1.2.840.10045.3.1.7"));
    ECDSA_PrivateKey user2_key(rng, ecc_domain);
@@ -164,29 +160,22 @@ void do_x509_tests(RandomNumberGenerator& rng)
    RSA_PrivateKey user2_key(rng, 1536);
 #endif
 
-   std::cout << '.' << std::flush;
    PKCS10_Request user2_req = X509::create_cert_req(req_opts2(),
                                                     user2_key,
                                                     hash_fn,
                                                     rng);
 
    /* Create the CA object */
-   std::cout << '.' << std::flush;
    X509_CA ca(ca_cert, ca_key, hash_fn);
-   std::cout << '.' << std::flush;
 
    /* Sign the requests to create the certs */
-   std::cout << '.' << std::flush;
    X509_Certificate user1_cert =
       ca.sign_request(user1_req, rng,
                       X509_Time("2008-01-01"), X509_Time("2100-01-01"));
 
-   std::cout << '.' << std::flush;
    X509_Certificate user2_cert = ca.sign_request(user2_req, rng,
                                                  X509_Time("2008-01-01"),
                                                  X509_Time("2100-01-01"));
-   std::cout << '.' << std::flush;
-
    X509_CRL crl1 = ca.new_crl(rng);
 
    /* Verify the certs */
@@ -194,19 +183,23 @@ void do_x509_tests(RandomNumberGenerator& rng)
 
    store.add_certificate(ca_cert);
 
-   std::cout << '.' << std::flush;
-
    Path_Validation_Restrictions restrictions;
 
    Path_Validation_Result result_u1 = x509_path_validate(user1_cert, restrictions, store);
    if(!result_u1.successful_validation())
+      {
       std::cout << "FAILED: User cert #1 did not validate - "
                 << result_u1.result_string() << std::endl;
+      ++fails;
+      }
 
    Path_Validation_Result result_u2 = x509_path_validate(user2_cert, restrictions, store);
    if(!result_u2.successful_validation())
+      {
       std::cout << "FAILED: User cert #2 did not validate - "
                 << result_u2.result_string() << std::endl;
+      ++fails;
+      }
 
    store.add_crl(crl1);
 
@@ -220,13 +213,19 @@ void do_x509_tests(RandomNumberGenerator& rng)
 
    result_u1 = x509_path_validate(user1_cert, restrictions, store);
    if(result_u1.result() != Certificate_Status_Code::CERT_IS_REVOKED)
+      {
       std::cout << "FAILED: User cert #1 was not revoked - "
                 << result_u1.result_string() << std::endl;
+      ++fails;
+      }
 
    result_u2 = x509_path_validate(user2_cert, restrictions, store);
    if(result_u2.result() != Certificate_Status_Code::CERT_IS_REVOKED)
+      {
       std::cout << "FAILED: User cert #2 was not revoked - "
                 << result_u2.result_string() << std::endl;
+      ++fails;
+      }
 
    revoked.clear();
    revoked.push_back(CRL_Entry(user1_cert, REMOVE_FROM_CRL));
@@ -236,22 +235,22 @@ void do_x509_tests(RandomNumberGenerator& rng)
 
    result_u1 = x509_path_validate(user1_cert, restrictions, store);
    if(!result_u1.successful_validation())
+      {
       std::cout << "FAILED: User cert #1 was not un-revoked - "
                 << result_u1.result_string() << std::endl;
+      ++fails;
+      }
 
    check_against_copy(ca_key, rng);
    check_against_copy(user1_key, rng);
    check_against_copy(user2_key, rng);
 
-   std::cout << std::endl;
+   return fails;
    }
 
 #else
 
-void do_x509_tests(RandomNumberGenerator&)
-   {
-   std::cout << "Skipping Botan X.509 tests (disabled in build)\n";
-   }
+size_t test_x590() { return 0; }
 
 #endif