Add a simple OpenSSL vs Botan benchmark script

2 files changed, 217 insertions, 0 deletions

diff --git a/src/scripts/bench.py b/src/scripts/bench.py
new file mode 100755
index 000000000..13de0c077
--- /dev/null
+++ b/src/scripts/bench.py
@@ -0,0 +1,216 @@
+#!/usr/bin/python
+
+"""
+Compare Botan with OpenSSL using their respective benchmark utils
+
+(C) 2017 Jack Lloyd
+
+Botan is released under the Simplified BSD License (see license.txt)
+
+TODO
+ - Also compare RSA, ECDSA, ECDH
+ - Output pretty graphs with matplotlib
+"""
+
+import logging
+import os
+import sys
+import optparse # pylint: disable=deprecated-module
+import subprocess
+import re
+import json
+
+def setup_logging(options):
+    if options.verbose:
+        log_level = logging.DEBUG
+    elif options.quiet:
+        log_level = logging.WARNING
+    else:
+        log_level = logging.INFO
+
+    class LogOnErrorHandler(logging.StreamHandler, object):
+        def emit(self, record):
+            super(LogOnErrorHandler, self).emit(record)
+            if record.levelno >= logging.ERROR:
+                sys.exit(1)
+
+    lh = LogOnErrorHandler(sys.stdout)
+    lh.setFormatter(logging.Formatter('%(levelname) 7s: %(message)s'))
+    logging.getLogger().addHandler(lh)
+    logging.getLogger().setLevel(log_level)
+
+def run_command(cmd):
+    logging.debug("Running '%s'", ' '.join(cmd))
+
+    proc = subprocess.Popen(cmd,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE,
+                            universal_newlines=True)
+    stdout, stderr = proc.communicate()
+
+    if proc.returncode != 0:
+        logging.error("Running command %s failed ret %d", ' '.join(cmd), proc.returncode)
+
+    return stdout + stderr
+
+def get_openssl_version(openssl):
+    output = run_command([openssl, 'version'])
+
+    openssl_version_re = re.compile(r'OpenSSL ([0-9a-z\.]+) .*')
+
+    match = openssl_version_re.match(output)
+
+    if match:
+        return match.group(1)
+    else:
+        logging.warning("Unable to parse OpenSSL version output %s", output)
+        return output
+
+def get_botan_version(botan):
+    return run_command([botan, 'version']).strip()
+
+EVP_MAP = {
+    'Blowfish': 'bf-ecb',
+    'AES-128/GCM': 'aes-128-gcm',
+    'AES-256/GCM': 'aes-256-gcm',
+    'ChaCha20': 'chacha20',
+    'MD5': 'md5',
+    'SHA-1': 'sha1',
+    'RIPEMD-160': 'ripemd160',
+    'SHA-256': 'sha256',
+    'SHA-384': 'sha384',
+    'SHA-512': 'sha512'
+    }
+
+def run_openssl_bench(openssl, algo):
+
+    logging.info('Running OpenSSL benchmark for %s', algo)
+
+    cmd = [openssl, 'speed', '-mr']
+
+    if algo in EVP_MAP:
+        cmd += ['-evp', EVP_MAP[algo]]
+    else:
+        cmd += [algo]
+
+    output = run_command(cmd)
+
+    buf_header = re.compile(r'\+DT:([a-z0-9-]+):([0-9]+):([0-9]+)$')
+    res_header = re.compile(r'\+R:([0-9]+):[a-z0-9-]+:([0-9]+\.[0-9]+)$')
+    ignored = re.compile(r'\+(H|F):.*')
+
+    results = []
+
+    result = None
+
+    for l in output.splitlines():
+        if ignored.match(l):
+            continue
+
+        if result is None:
+            match = buf_header.match(l)
+            if match is None:
+                logging.error("Unexpected output from OpenSSL %s", l)
+
+            result = {'algo': algo, 'buf_size': int(match.group(3))}
+        else:
+            match = res_header.match(l)
+
+            result['bytes'] = int(match.group(1)) * result['buf_size']
+            result['runtime'] = float(match.group(2))
+            result['bps'] = int(result['bytes'] / result['runtime'])
+            results.append(result)
+            result = None
+
+    return results
+
+def run_botan_bench(botan, runtime, buf_sizes, algo):
+
+    runtime = .05
+
+    cmd = [botan, 'speed', '--format=json', '--msec=%d' % int(runtime * 1000),
+           '--buf-size=%s' % (','.join(map(str, buf_sizes))), algo]
+    output = run_command(cmd)
+    output = json.loads(output)
+
+    return output
+
+class BenchmarkResult(object):
+    def __init__(self, algo, buf_sizes, openssl_results, botan_results):
+        self.algo = algo
+        self.results = {}
+
+        def find_result(results, sz):
+            for r in results:
+                if 'buf_size' in r and r['buf_size'] == sz:
+                    return r['bps']
+            raise Exception("Could not find expected result in data")
+
+        for buf_size in buf_sizes:
+            self.results[buf_size] = {
+                'openssl': find_result(openssl_results, buf_size),
+                'botan': find_result(botan_results, buf_size)
+            }
+
+    def result_string(self):
+
+        out = ""
+        for (k, v) in self.results.items():
+            out += "algo %s buf_size % 6d botan % 12d bps openssl % 12d bps adv %.02f\n" % (
+                self.algo, k, v['botan'], v['openssl'], float(v['botan']) / v['openssl'])
+        return out
+
+def bench_algo(openssl, botan, algo):
+    openssl_results = run_openssl_bench(openssl, algo)
+
+    buf_sizes = sorted([x['buf_size'] for x in openssl_results])
+    runtime = sum(x['runtime'] for x in openssl_results) / len(openssl_results)
+
+    botan_results = run_botan_bench(botan, runtime, buf_sizes, algo)
+
+    return BenchmarkResult(algo, buf_sizes, openssl_results, botan_results)
+
+def main(args=None):
+    if args is None:
+        args = sys.argv
+
+    parser = optparse.OptionParser()
+
+    parser.add_option('--verbose', action='store_true', default=False, help="be noisy")
+    parser.add_option('--quiet', action='store_true', default=False, help="be very quiet")
+
+    parser.add_option('--openssl-cli', metavar='PATH',
+                      default='/usr/bin/openssl',
+                      help='Path to openssl binary (default %default)')
+
+    parser.add_option('--botan-cli', metavar='PATH',
+                      default='/usr/bin/botan',
+                      help='Path to botan binary (default %default)')
+
+    (options, args) = parser.parse_args(args)
+
+    setup_logging(options)
+
+    openssl = options.openssl_cli
+    botan = options.botan_cli
+
+    if os.access(openssl, os.X_OK) is False:
+        logging.error("Unable to access openssl binary at %s", openssl)
+
+    if os.access(botan, os.X_OK) is False:
+        logging.error("Unable to access botan binary at %s", botan)
+
+    openssl_version = get_openssl_version(openssl)
+    botan_version = get_botan_version(botan)
+
+    logging.info("Comparing Botan %s with OpenSSL %s", botan_version, openssl_version)
+
+    for algo in sorted(EVP_MAP.keys()):
+        result = bench_algo(openssl, botan, algo)
+        print(result.result_string())
+
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
index 69617dee0..72e90eded 100755
--- a/src/scripts/ci_build.py
+++ b/src/scripts/ci_build.py
@@ -369,6 +369,7 @@ def main(args=None):
             'src/scripts/cleanup.py',
             'src/scripts/build_docs.py',
             'src/scripts/website.py',
+            'src/scripts/bench.py',
             'src/scripts/python_unittests.py',
             'src/scripts/python_unittests_unix.py']