Merge GH #1162 Use Python script to orchestrate CI build

13 files changed, 489 insertions, 399 deletions

diff --git a/.travis.yml b/.travis.yml
index 8431729b4..4e646d9c7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,13 +14,11 @@ compiler:
 
 env:
   global:
-    - CCACHE_SIZE=50M
+    - CCACHE_MAXSIZE=100M
+    - CCACHE_MAXFILES=1000
     - CCACHE_COMPRESS=1
-    - BUILD_JOBS=2
     # SONAR_TOKEN for accessing the SonarQube server
     - secure: "VezRbHFg6kllV5WG06M3tG3aHJaC3xrMylJ6RCVbL+uz2JeralVCqV7eIk4fVb9cu83Li+weEa0AJj0wkxpIUJ+vUh5F65L6gSWSbgHP7muOSVsmnEc6KvX4n3av/ZGe4geSmsxqh2pd/2xI1h7KioGRhKeqaZIdjVgWgGJW2iQ="
-    # GITHUB_TOKEN for posting found issues as comments in the PR that's beeing analyzed by SonarQube
-    - secure: "Th0mBSkUCDqu+EA6F7zA6DCSDZBNunfndANyq06BwaFlj71daWWjthwYFsfg3T5N2ZmI+PsULQQpOirCnJt1lbNHhMVJwZPkW0JnjoxbSNpSI2+nHv7+GO9X9WjK0LRFawiQu8WxmLMQDA+0oR0BERSFKc3gmbuav9fDfla0dXg="
 
   matrix:
     - BUILD_MODE="docs"
@@ -97,13 +95,10 @@ matrix:
       env: BUILD_MODE="cross-arm64"
 
 install:
-  - ./src/scripts/ci/travis/install.sh
+  - ./src/scripts/ci/setup_travis.sh
 
 script:
-  - if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then ./src/scripts/ci/travis/main.sh ; fi
-
-after_success:
-  - ./src/scripts/ci/travis/after_success.sh
+  - if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then ./src/scripts/ci_build.py --os=$TRAVIS_OS_NAME --cc=$CC --cc-bin=$CXX $BUILD_MODE; fi
 
 # whitelist branches to avoid testing feature branches twice (as branch and as pull request)
 branches:
@@ -115,11 +110,6 @@ branches:
 notifications:
   email: [email protected]
 
-  webhooks:
-    urls:
-      - https://webhooks.gitter.im/e/9caf8eaa06913b69443f
-    on_start: never
-
 git:
   depth: 5
 
diff --git a/configure.py b/configure.py
index f9bf90fc9..5d79aa012 100755
--- a/configure.py
+++ b/configure.py
@@ -1785,7 +1785,7 @@ class MakefileListsGenerator(object):
         if is_fuzzer:
             for (obj_file, src) in zip(self._objectfile_list(sources, obj_dir), sources):
 
-                yield '%s: %s\n\t$(CXX) %s $(%s_FLAGS) %s -L. -lbotan-2 $(FUZZER_LINKS_TO) %s$@\n' % (
+                yield '%s: %s $(LIBRARIES)\n\t$(CXX) %s $(%s_FLAGS) %s -L. -lbotan-2 $(FUZZER_LINKS_TO) %s$@\n' % (
                     obj_file, src, includes, flags, src, self._cc.output_to_option)
         else:
             for (obj_file, src) in zip(self._objectfile_list(sources, obj_dir), sources):
diff --git a/src/build-data/makefile/gmake.in b/src/build-data/makefile/gmake.in
index c77c1908c..4b4426e5d 100644
--- a/src/build-data/makefile/gmake.in
+++ b/src/build-data/makefile/gmake.in
@@ -21,7 +21,7 @@ CLIOBJS       = %{cli_objs}
 TESTOBJS      = %{test_objs}
 
 # First make target. Will be used by default
-all: $(CLI) $(TEST)
+all: libs cli tests
 
 # Build Commands
 %{lib_build_cmds}
@@ -34,6 +34,8 @@ all: $(CLI) $(TEST)
 %{gmake_dso_in}
 
 libs: $(LIBRARIES)
+cli: $(CLI)
+tests: $(TEST)
 
 $(CLI): $(LIBRARIES) $(CLIOBJS)
 	$(CLI_LINK_CMD) $(LDFLAGS) $(CLIOBJS) -L%{out_dir} -l%{libname} $(CLI_LINKS_TO) -o $(CLI)
diff --git a/src/lib/prov/openssl/info.txt b/src/lib/prov/openssl/info.txt
index 75ff3b6df..cecfaca56 100644
--- a/src/lib/prov/openssl/info.txt
+++ b/src/lib/prov/openssl/info.txt
@@ -12,3 +12,7 @@ openssl.h
 all!windows -> crypto
 windows -> libeay32.lib
 </libs>
+
+<requires>
+pubkey
+</requires>
diff --git a/src/scripts/ci/codecov/codecov.yml b/src/scripts/ci/codecov.yml
index db26a6bd5..db26a6bd5 100644
--- a/src/scripts/ci/codecov/codecov.yml
+++ b/src/scripts/ci/codecov.yml
diff --git a/src/scripts/ci/setup_travis.sh b/src/scripts/ci/setup_travis.sh
new file mode 100755
index 000000000..425602e71
--- /dev/null
+++ b/src/scripts/ci/setup_travis.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+set -ev
+which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
+
+if [ "$TRAVIS_OS_NAME" = "linux" ]; then
+
+    if [ "$BUILD_MODE" = "valgrind" ]; then
+        sudo apt-get -qq update
+        sudo apt-get install valgrind
+
+    elif [ "$BUILD_MODE" = "cross-win32" ]; then
+        # See https://github.com/travis-ci/travis-ci/issues/6460
+        sudo dpkg --add-architecture i386
+        sudo apt-get -qq update # have to update again due to adding i386 above
+        sudo apt-get install wine
+        sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev
+
+    elif [ "${BUILD_MODE:0:5}" = "cross" ]; then
+         # Need updated qemu
+         sudo add-apt-repository -y ppa:ubuntu-cloud-archive/kilo-staging
+         sudo apt-get -qq update
+         sudo apt-get install qemu
+
+         if [ "$BUILD_MODE" = "cross-arm32" ]; then
+             sudo apt-get install g++-arm-linux-gnueabihf libc6-dev-armhf-cross
+         elif [ "$BUILD_MODE" = "cross-arm64" ]; then
+             sudo apt-get install g++-aarch64-linux-gnu libc6-dev-arm64-cross
+         elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
+             sudo apt-get install g++-powerpc-linux-gnu libc6-dev-powerpc-cross
+         elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
+             sudo apt-get install g++-powerpc64le-linux-gnu libc6-dev-ppc64el-cross
+         fi
+
+    elif [ "$BUILD_MODE" = "lint" ]; then
+        pip install --user pylint
+        pip3 install --user pylint
+
+    elif [ "$BUILD_MODE" = "coverage" ]; then
+        sudo apt-get -qq update
+        sudo apt-get install trousers libtspi-dev
+
+        # need updated lcov for gcc 4.8 coverage format
+        wget http://mirrors.kernel.org/ubuntu/pool/universe/l/lcov/lcov_1.12-2_all.deb
+        sudo dpkg -i lcov_1.12-2_all.deb
+
+        # ccache in Trusty doesn't understand --coverage
+        wget http://mirrors.kernel.org/ubuntu/pool/main/c/ccache/ccache_3.2.4-1_amd64.deb
+        sudo dpkg -i ccache_3.2.4-1_amd64.deb
+
+        (cd /home/travis/bin && ln -s gcov-4.8 gcov)
+
+        pip install --user coverage
+        pip install --user codecov
+
+        # SoftHSMv1 in 14.04 does not work
+        # Installs prebuilt SoftHSMv2 binaries into /tmp
+        wget https://www.randombit.net/softhsm2-trusty-bin.tar.bz2
+        tar -C / -xvjf softhsm2-trusty-bin.tar.bz2
+        /tmp/softhsm/bin/softhsm2-util --init-token --free --label test --pin 123456 --so-pin 12345678
+
+    elif [ "$BUILD_MODE" = "sonarqube" ]; then
+        wget https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip
+        unzip build-wrapper-linux-x86.zip
+
+    elif [ "$BUILD_MODE" = "docs" ]; then
+        sudo apt-get -qq update
+        sudo apt-get install doxygen
+
+        # The version of Sphinx in 14.04 is too old (1.2.2) and does not support
+        # all C++ features used in the manual. Install python-requests to avoid
+        # problem in Ubuntu packaged version, see
+        # http://stackoverflow.com/questions/32779919/no-module-named-for-requests
+        sudo apt-get remove python-requests python-openssl
+        sudo pip install requests sphinx pyopenssl
+    fi
+
+elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
+    # Workaround for https://github.com/Homebrew/homebrew/issues/42553
+    brew update || brew update
+
+    brew install ccache
+
+    if [ "$BUILD_MODE" != "cross-arm32" ] && [ "$BUILD_MODE" != "cross-arm64" ]; then
+        brew install xz
+        # Python2 is already installed
+        brew install python3
+
+        # Boost 1.58 is installed on Travis OS X images
+        # brew install boost
+    fi
+
+fi
diff --git a/src/scripts/ci/travis/after_success.sh b/src/scripts/ci/travis/after_success.sh
deleted file mode 100755
index 66612fa10..000000000
--- a/src/scripts/ci/travis/after_success.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-if [ "$BUILD_MODE" = "coverage" ]; then
-    GCOV="/usr/bin/gcov-4.8"
-    /tmp/bin/lcov --gcov-tool "$GCOV" --directory . --capture --output-file "$(pwd)/coverage.info.in"
-    /tmp/bin/lcov --gcov-tool "$GCOV" --remove "$(pwd)/coverage.info.in" 'tests/*' '/usr/*' --output-file "$(pwd)/coverage.info"
-    /tmp/bin/lcov --gcov-tool "$GCOV" --list "$(pwd)/coverage.info"
-
-    LD_LIBRARY_PATH=. coverage run --branch src/python/botan2.py
-
-    codecov
-fi
diff --git a/src/scripts/ci/travis/build.sh b/src/scripts/ci/travis/build.sh
deleted file mode 100755
index bbf4e986c..000000000
--- a/src/scripts/ci/travis/build.sh
+++ /dev/null
@@ -1,216 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-MAKE_PREFIX=()
-TEST_PREFIX=()
-TEST_EXE=./botan-test
-TEST_FLAGS=()
-CLI_EXE=./botan
-CFG_FLAGS=(--prefix=/tmp/botan-installation --cc=$CC --os=$TRAVIS_OS_NAME)
-
-CC_BIN=$CXX
-
-if [ "$BUILD_MODE" = "static" ] || [ "$BUILD_MODE" = "mini-static" ]; then
-    CFG_FLAGS+=(--disable-shared --amalgamation)
-elif [ "$BUILD_MODE" = "shared" ] || [ "$BUILD_MODE" = "mini-shared" ]; then
-    # No special flags required for shared lib build
-    CFG_FLAGS+=()
-elif [ "$BUILD_MODE" = "bsi" ]; then
-    CFG_FLAGS+=(--module-policy=bsi)
-elif [ "$BUILD_MODE" = "nist" ]; then
-    CFG_FLAGS+=(--module-policy=nist)
-elif [ "$BUILD_MODE" = "sonarqube" ]; then
-    # No special flags required
-    CFG_FLAGS+=()
-elif [ "$BUILD_MODE" = "fuzzers" ]; then
-    CFG_FLAGS+=(--build-fuzzers=test --with-sanitizers --with-debug-info --disable-shared)
-elif [ "$BUILD_MODE" = "parallel" ]; then
-
-    if [ "$CC" = "gcc" ]; then
-        CFG_FLAGS+=(--with-cilkplus)
-    else
-        CFG_FLAGS+=(--with-openmp)
-    fi
-
-elif [ "$BUILD_MODE" = "coverage" ]; then
-    CFG_FLAGS+=(--with-coverage --build-fuzzers=test --no-optimizations)
-elif [ "$BUILD_MODE" = "sanitizer" ]; then
-    export ASAN_OPTIONS=detect_leaks=0
-    CFG_FLAGS+=(--with-sanitizers --disable-modules=locking_allocator)
-elif [ "$BUILD_MODE" = "valgrind" ]; then
-    CFG_FLAGS+=(--with-valgrind --with-debug-info --disable-modules=locking_allocator)
-    TEST_PREFIX=(valgrind --error-exitcode=9 -v)
-fi
-
-if [ "$BUILD_MODE" = "mini-static" ] || [ "$BUILD_MODE" = "mini-shared" ]; then
-    CFG_FLAGS+=(--minimized-build --enable-modules="base,dev_random,system_rng,sha2_32,sha2_64,aes")
-elif [ "$BUILD_MODE" = "valgrind" ]; then
-    # Valgrind on Travis on full build takes too long and the job is killed
-    # Prune to the most important stuff
-    CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-
-elif [ "${BUILD_MODE:0:5}" != "cross" ]; then
-    # Only use external libraries when compiling natively
-    CFG_FLAGS+=(--with-bzip2 --with-lzma --with-sqlite --with-zlib)
-
-    if [ "$BUILD_MODE" = "coverage" ]; then
-        CFG_FLAGS+=(--with-tpm)
-        TEST_FLAGS=(--run-long-tests --run-online-tests --pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so)
-    fi
-
-    # Avoid OpenSSL when using dynamic checkers, or on OS X where it sporadically
-    # is not installed on the CI image
-    if [ "$TRAVIS_OS_NAME" != "osx" ] && [ "$BUILD_MODE" != "sanitizer" ] && [ "$BUILD_MODE" != "valgrind" ]; then
-        CFG_FLAGS+=(--with-openssl)
-    fi
-fi
-
-if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "${BUILD_MODE:0:5}" != "cross" ]; then
-    # The Boost-specific codepaths are tested via the OS X CI
-    CFG_FLAGS+=(--with-boost)
-fi
-
-if [ "${BUILD_MODE:0:6}" = "cross-" ]; then
-
-    if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-        CFG_FLAGS+=(--disable-shared)
-        MAKE_PREFIX=(xcrun --sdk iphoneos)
-        if [ "$BUILD_MODE" = "cross-arm32" ]; then
-            CFG_FLAGS+=(--os=ios)
-            CFG_FLAGS+=(--cpu=armv7 --cc-abi-flags="-arch armv7 -arch armv7s -stdlib=libc++")
-        elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-            CFG_FLAGS+=(--os=ios)
-            CFG_FLAGS+=(--cpu=armv8-a --cc-abi-flags="-arch arm64 -stdlib=libc++")
-        fi
-    elif [ "$TRAVIS_OS_NAME" = "linux" ]; then
-        CFG_FLAGS+=(--disable-modules=ffi)
-
-        if [ "$BUILD_MODE" = "cross-arm32" ]; then
-            CC_BIN=arm-linux-gnueabihf-g++-4.8
-            TEST_PREFIX=(qemu-arm -L /usr/arm-linux-gnueabihf/)
-            CFG_FLAGS+=(--cpu=armv7)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-            CC_BIN=aarch64-linux-gnu-g++-4.8
-            TEST_PREFIX=(qemu-aarch64 -L /usr/aarch64-linux-gnu/)
-            CFG_FLAGS+=(--cpu=armv8-a)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
-            CC_BIN=powerpc-linux-gnu-g++-4.8
-            TEST_PREFIX=(qemu-ppc -L /usr/powerpc-linux-gnu/)
-            CFG_FLAGS+=(--cpu=ppc32)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
-            CC_BIN=powerpc64le-linux-gnu-g++-4.8
-            TEST_PREFIX=(qemu-ppc64le -L /usr/powerpc64le-linux-gnu/)
-            CFG_FLAGS+=(--cpu=ppc64 --with-endian=little)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-win32" ]; then
-            CC_BIN=i686-w64-mingw32-g++
-            # No test prefix needed, PE executes as usual with Wine installed
-            CFG_FLAGS+=(--os=mingw)
-            CFG_FLAGS+=(--cpu=x86_32 --cc-abi-flags="-static" --disable-shared)
-            TEST_EXE=./botan-test.exe
-        fi
-    fi
-fi
-
-CFG_FLAGS+=(--cc-bin="ccache $CC_BIN")
-
-if [ "$BUILD_MODE" = "sonarqube" ]; then
-   MAKE_PREFIX=(./build-wrapper-linux-x86/build-wrapper-linux-x86-64 --out-dir bw-outputs)
-fi
-
-# configure
-./configure.py "${CFG_FLAGS[@]}"
-
-# pre-build ccache stats
-ccache --show-stats
-
-# build!
-
-if [ "$BUILD_MODE" = "docs" ]; then
-    doxygen build/botan.doxy
-    sphinx-build -a -W -c src/build-data/sphinx doc/manual manual-out
-else
-    MAKE_CMD=("${MAKE_PREFIX[@]}" make -j "$BUILD_JOBS")
-    echo "Running" "${MAKE_CMD[@]}"
-    time "${MAKE_CMD[@]}"
-fi
-
-if [ "$BUILD_MODE" = "fuzzers" ] || [ "$BUILD_MODE" = "coverage" ]; then
-    make fuzzers
-    make fuzzer_corpus_zip
-fi
-
-# post-build ccache stats
-ccache --show-stats
-
-# Run SonarQube analysis
-
-if [ "$BUILD_MODE" = "sonarqube" ]; then
-
-    cp src/build-data/sonar-project.properties .
-
-    if [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
-       # => This will run a full analysis of the project and push results to the SonarQube server.
-       #
-       # Analysis is done only on master so that build of branches don't push analyses to the same project and therefore "pollute" the results
-       echo "Starting analysis by SonarQube..."
-       sonar-scanner "-Dsonar.login=$SONAR_TOKEN"
-
-    # PR analysis deactivated at least until custom quality profiles can be created
-    elif false && [ "$TRAVIS_PULL_REQUEST" != "false" ] && [ -n "${GITHUB_TOKEN-}" ]; then
-        # => This will analyse the PR and display found issues as comments in the PR, but it won't push results to the SonarQube server
-        #
-        # For security reasons environment variables are not available on the pull requests
-        # coming from outside repositories
-        # http://docs.travis-ci.com/user/pull-requests/#Security-Restrictions-when-testing-Pull-Requests
-        # That's why the analysis does not need to be executed if the variable GITHUB_TOKEN is not defined.
-        echo "Starting Pull Request analysis by SonarQube..."
-        sonar-scanner -Dsonar.login="$SONAR_TOKEN" \
-                      -Dsonar.analysis.mode=preview \
-                      -Dsonar.github.oauth="$GITHUB_TOKEN" \
-                      -Dsonar.github.repository="$TRAVIS_REPO_SLUG" \
-                      -Dsonar.github.pullRequest="$TRAVIS_PULL_REQUEST"
-    fi
-       # When neither on master branch nor on a non-external pull request => nothing to do
-    fi
-
-if [ "$BUILD_MODE" = "sonarqube" ] || [ "$BUILD_MODE" = "docs" ] || 
-       ( [ "${BUILD_MODE:0:5}" = "cross" ] && [ "$TRAVIS_OS_NAME" = "osx" ] ); then
-    echo "Running tests disabled on this build type"
-
-elif [ "$BUILD_MODE" != "fuzzers" ]; then
-    TEST_CMD=("${TEST_PREFIX[@]}" $TEST_EXE "${TEST_FLAGS[@]}")
-    echo "Running" "${TEST_CMD[@]}"
-    time "${TEST_CMD[@]}"
-fi
-
-if [ "$BUILD_MODE" = "fuzzers" ] || [ "$BUILD_MODE" = "coverage" ]; then
-    # Test each fuzzer against its corpus
-    LD_LIBRARY_PATH=. ./src/scripts/test_fuzzers.py fuzzer_corpus build/fuzzer
-fi
-
-if [ "$BUILD_MODE" = "static" ] || [ "$BUILD_MODE" = "shared" ]
-then
-    echo "Running cli tests ..."
-    ./src/scripts/cli_tests.py "$CLI_EXE"
-fi
-
-# Run Python tests (need shared libs)
-if [ "$BUILD_MODE" = "shared" ] || [ "$BUILD_MODE" = "coverage" ];
-then
-    # TODO: find all things in PATH that begin with python- and execute them :)
-    for py in python2 python3
-    do
-        $py --version
-        LD_LIBRARY_PATH=. $py src/python/botan2.py
-    done
-fi
-
-if [ "$BUILD_MODE" != "docs" ]; then
-    # Test make install
-    make install
-fi
diff --git a/src/scripts/ci/travis/install.sh b/src/scripts/ci/travis/install.sh
deleted file mode 100755
index bea57cc82..000000000
--- a/src/scripts/ci/travis/install.sh
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-if [ "$BUILD_MODE" = "coverage" ]; then
-    wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.13.orig.tar.gz
-    tar -xvf lcov_1.13.orig.tar.gz
-    make PREFIX="/tmp" -C lcov-1.13/ install
-
-    pip install --user coverage
-
-    pip install --user codecov
-fi
-
-if [ "$BUILD_MODE" = "sonarqube" ]; then
-    curl -LsS https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip > build-wrapper-linux-x86.zip
-    unzip build-wrapper-linux-x86.zip
-fi
-
-if [ "$TRAVIS_OS_NAME" = "linux" ]; then
-    sudo apt-get -qq update
-
-    if [ "$BUILD_MODE" = "lint" ]; then
-        pip install --user pylint
-        pip3 install --user pylint
-    fi
-
-    if [ "$BUILD_MODE" = "docs" ]; then
-        sudo apt-get install doxygen
-
-        # The version of Sphinx in 14.04 is too old (1.2.2) and does not support
-        # all C++ features used in the manual. Install python-requests to avoid
-        # problem in Ubuntu packaged version, see
-        # http://stackoverflow.com/questions/32779919/no-module-named-for-requests
-        sudo apt-get remove python-requests python-openssl
-        sudo pip install requests sphinx pyopenssl
-    fi
-
-    if [ "$BUILD_MODE" = "coverage" ]; then
-        sudo apt-get install trousers libtspi-dev
-
-        # SoftHSMv1 in 14.04 does not work
-        # Installs prebuilt SoftHSMv2 binaries into /tmp
-        wget https://www.randombit.net/softhsm2-trusty-bin.tar.bz2
-        tar -C / -xvjf softhsm2-trusty-bin.tar.bz2
-        /tmp/softhsm/bin/softhsm2-util --init-token --free --label test --pin 123456 --so-pin 12345678
-    fi
-
-    if [ "$BUILD_MODE" = "valgrind" ] || [ "${BUILD_MODE:0:5}" = "cross" ]; then
-        if [ "$BUILD_MODE" = "valgrind" ]; then
-            sudo apt-get install valgrind
-        elif [ "$BUILD_MODE" = "cross-win32" ]; then
-            sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev
-
-            # See https://github.com/travis-ci/travis-ci/issues/6460
-            sudo dpkg --add-architecture i386
-            sudo apt-get -qq update # have to update again due to adding i386 above
-            sudo apt-get install wine
-        else
-
-            # Need updated qemu
-            sudo add-apt-repository -y ppa:ubuntu-cloud-archive/kilo-staging
-            sudo apt-get -qq update
-            sudo apt-get install qemu
-
-            if [ "$BUILD_MODE" = "cross-arm32" ]; then
-                sudo apt-get install g++-4.8-arm-linux-gnueabihf libc6-dev-armhf-cross
-            elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-                sudo apt-get install g++-4.8-aarch64-linux-gnu libc6-dev-arm64-cross
-            elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
-                sudo apt-get install g++-4.8-powerpc-linux-gnu libc6-dev-powerpc-cross
-            elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
-                sudo apt-get install g++-4.8-powerpc64le-linux-gnu libc6-dev-ppc64el-cross
-            fi
-        fi
-    fi
-fi
-
-if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-    # Workaround for https://github.com/Homebrew/homebrew/issues/42553
-    brew update || brew update
-
-    brew install ccache
-
-    if [ "$BUILD_MODE" != "cross-arm32" ] && [ "$BUILD_MODE" != "cross-arm64" ]; then
-        brew install xz
-        # Python2 is already installed
-        brew install python3
-
-        # Boost 1.58 is installed on Travis OS X images
-        # brew install boost
-    fi
-
-fi
diff --git a/src/scripts/ci/travis/lint.sh b/src/scripts/ci/travis/lint.sh
deleted file mode 100755
index 65f885365..000000000
--- a/src/scripts/ci/travis/lint.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-shopt -s expand_aliases
-
-# Disabled rules in Python 2 only
-# superfluous-parens: needed for print statements
-# too-many-locals: variable counting differs from pylint3
-alias python2_pylint='python2 -m pylint --disable=superfluous-parens,too-many-locals'
-alias python3_pylint='python3 -m pylint'
-
-echo "travis_fold:start:pylint_configure"
-python2_pylint configure.py
-python3_pylint configure.py
-echo "travis_fold:end:pylint_configure"
-
-echo "travis_fold:start:pylint_install"
-python2_pylint src/scripts/install.py
-python3_pylint src/scripts/install.py
-echo "travis_fold:end:pylint_configure"
-
-echo "travis_fold:start:pylint_python_unittests"
-python3_pylint src/scripts/python_unittests.py
-echo "travis_fold:end:pylint_python_unittests"
-
-echo "travis_fold:start:pylint_python_unittests_unix"
-python3_pylint src/scripts/python_unittests_unix.py
-echo "travis_fold:end:pylint_python_unittests_unix"
-
-echo "travis_fold:start:pylint_botanpy"
-python2_pylint src/python/botan2.py
-python3_pylint src/python/botan2.py
-echo "travis_fold:end:pylint_botanpy"
diff --git a/src/scripts/ci/travis/main.sh b/src/scripts/ci/travis/main.sh
deleted file mode 100755
index ed6f56063..000000000
--- a/src/scripts/ci/travis/main.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-PARENT_DIR=$(dirname "$0")
-
-if [ "$BUILD_MODE" = "lint" ]; then
-    "$PARENT_DIR"/lint.sh
-else
-    "$PARENT_DIR"/build.sh
-fi
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
new file mode 100755
index 000000000..e35315bd3
--- /dev/null
+++ b/src/scripts/ci_build.py
@@ -0,0 +1,385 @@
+#!/usr/bin/env python
+
+"""
+CI build script
+(C) 2017 Jack Lloyd
+Botan is released under the Simplified BSD License (see license.txt)
+"""
+
+import os
+import platform
+import subprocess
+import sys
+import time
+import optparse # pylint: disable=deprecated-module
+
+def get_concurrency():
+    """
+    Get default concurrency level of build
+    """
+    def_concurrency = 2
+
+    try:
+        import multiprocessing
+        return max(def_concurrency, multiprocessing.cpu_count())
+    except ImportError:
+        return def_concurrency
+
+def getenv_or_die(var):
+    """
+    Like it says...
+    """
+    val = os.getenv(var)
+    if val is None:
+        raise Exception('Required variable %s not set in environment' % (var))
+    return val
+
+def determine_flags(target, target_os, target_cc, cc_bin, use_ccache, root_dir):
+    # pylint: disable=too-many-branches,too-many-statements,too-many-arguments
+
+    """
+    Return the configure.py flags as well as make/test running prefixes
+    """
+    is_cross_target = target.startswith('cross-')
+
+    if target_os not in ['linux', 'osx']:
+        print('Error unknown OS %s' % (target_os))
+        return 1
+
+    if is_cross_target:
+        if target_os == 'osx':
+            target_os = 'ios'
+        elif target == 'cross-win32':
+            target_os = 'mingw'
+
+    make_prefix = []
+    test_prefix = []
+    test_cmd = [os.path.join(root_dir, 'botan-test')]
+
+    flags = ['--prefix=/tmp/botan-install', '--cc=%s' % (target_cc), '--os=%s' % (target_os)]
+
+    if target in ['static', 'mini-static', 'fuzzers'] or target_os in ['ios', 'mingw']:
+        flags += ['--disable-shared']
+
+    if target in ['mini-static', 'mini-shared']:
+        flags += ['--minimized-build', '--enable-modules=dev_random,system_rng,sha2_32,sha2_64,aes']
+
+    if target == 'shared':
+        # Arbitrarily test amalgamation on shared obj builds
+        flags += ['--amalgamation']
+
+    if target in ['bsi', 'nist']:
+        flags += ['--module-policy=%s' % (target)]
+
+    if target == 'docs':
+        flags += ['--with-doxygen', '--with-sphinx']
+        test_cmd = None
+
+    if target == 'coverage':
+        flags += ['--with-coverage-info']
+    if target == 'valgrind':
+        flags += ['--with-valgrind']
+        test_prefix = ['valgrind', '--error-exitcode=9', '-v']
+
+    if target in ['fuzzers', 'coverage', 'valgrind']:
+        flags += ['--with-debug-info']
+    if target in ['fuzzers', 'coverage']:
+        flags += ['--build-fuzzers=test']
+    if target in ['fuzzers', 'sanitizer']:
+        flags += ['--with-sanitizers']
+    if target in ['valgrind', 'sanitizer', 'fuzzers']:
+        flags += ['--disable-modules=locking_allocator']
+
+    if target == 'parallel':
+        if 'cc' == 'gcc':
+            flags += ['--with-cilkplus']
+        else:
+            flags += ['--with-openmp']
+
+    if target == 'sonarqube':
+        make_prefix = [os.path.join(root_dir, 'build-wrapper-linux-x86/build-wrapper-linux-x86-64'),
+                       '--out-dir', 'bw-outputs']
+        test_cmd = ['sonar-scanner',
+                    '-Dproject.settings=%s' % (os.path.join(root_dir, 'src', 'build-data', 'sonar-project.properties')),
+                    '-Dsonar.login=%s' % (getenv_or_die('SONAR_TOKEN'))]
+
+    if target_os == 'linux' and (target == 'valgrind' or is_cross_target):
+        # Minimize the build when doing something that is slow
+        # Note this skips os == 'mingw' since the tests are fast under Wine
+        flags += ['--module-policy=modern', '--enable-modules=tls']
+
+    if is_cross_target:
+        if target_os == 'ios':
+            make_prefix = ['xcrun', '--sdk', 'iphoneos']
+            test_cmd = None
+            if target == 'cross-arm32':
+                flags += ['--cpu=armv7', '--cc-abi-flags=-arch armv7 -arch armv7s -stdlib=libc++']
+            elif target == 'cross-arm64':
+                flags += ['--cpu=armv8-a', '--cc-abi-flags=-arch arm64 -stdlib=libc++']
+            else:
+                raise Exception("Unknown cross target '%s' for iOS" % (target))
+        elif target == 'cross-win32':
+            flags += ['--cpu=x86_32', '--cc-abi-flags=-static']
+            cc_bin = 'i686-w64-mingw32-g++'
+            test_cmd = [os.path.join(root_dir, 'botan-test.exe')]
+            # No runtime prefix required for Wine
+        else:
+            if target == 'cross-arm32':
+                flags += ['--cpu=armv7']
+                cc_bin = 'arm-linux-gnueabihf-g++'
+                test_prefix = ['qemu-arm', '-L', '/usr/arm-linux-gnueabihf/']
+            elif target == 'cross-arm64':
+                flags += ['--cpu=armv8-a']
+                cc_bin = 'aarch64-linux-gnu-g++'
+                test_prefix = ['qemu-aarch64', '-L', '/usr/aarch64-linux-gnu/']
+            elif target == 'cross-ppc32':
+                flags += ['--cpu=ppc32']
+                cc_bin = 'powerpc-linux-gnu-g++'
+                test_prefix = ['qemu-ppc', '-L', '/usr/powerpc-linux-gnu/']
+            elif target == 'cross-ppc64':
+                flags += ['--cpu=ppc64', '--with-endian=little']
+                cc_bin = 'powerpc64le-linux-gnu-g++'
+                test_prefix = ['qemu-ppc64le', '-L', '/usr/powerpc64le-linux-gnu/']
+            else:
+                raise Exception("Unknown cross target '%s' for Linux" % (target))
+    else:
+        # Flags specific to native targets
+        flags += ['--with-bzip2', '--with-lzma', '--with-sqlite', '--with-zlib']
+
+        if target_os == 'osx':
+            # Test Boost on OS X
+            flags += ['--with-boost']
+        elif target not in ['sanitizer', 'valgrind', 'mini-shared', 'mini-static']:
+            # Avoid OpenSSL when using dynamic checkers, or on OS X where it sporadically
+            # is not installed on the CI image
+            flags += ['--with-openssl']
+
+        if target == 'coverage':
+            flags += ['--with-tpm']
+            test_cmd += ['--run-long-tests', '--run-online-tests']
+
+            softhsm_lib = '/tmp/softhsm/lib/softhsm/libsofthsm2.so'
+            if os.access(softhsm_lib, os.R_OK):
+                test_cmd += ['--pkcs11-lib=%s' % (softhsm_lib)]
+
+    flags += ['--cc-bin=%s%s' % ('ccache ' if use_ccache else '', cc_bin)]
+
+    if test_cmd is None:
+        run_test_command = None
+    else:
+        run_test_command = test_prefix + test_cmd
+
+    return flags, run_test_command, make_prefix
+
+def run_cmd(cmd, root_dir):
+    """
+    Execute a command, die if it failed
+    """
+    print("Running '%s':\n" % (' '.join(cmd)))
+    sys.stdout.flush()
+
+    start = time.time()
+
+    sub_env = os.environ.copy()
+    sub_env['LD_LIBRARY_PATH'] = root_dir
+    proc = subprocess.Popen(cmd, close_fds=True, env=sub_env)
+    proc.communicate()
+
+    time_taken = int(time.time() - start)
+
+    if time_taken > 2:
+        print("Ran for %d seconds" % (time_taken))
+
+    if proc.returncode != 0:
+        print("Command failed with error code %d" % (proc.returncode))
+        sys.exit(proc.returncode)
+
+def parse_args(args):
+    """
+    Parse arguments
+    """
+    parser = optparse.OptionParser()
+
+    parser.add_option('--os', default=platform.system().lower(),
+                      help='Set the target os (default %default)')
+    parser.add_option('--cc', default='gcc',
+                      help='Set the target compiler type (default %default)')
+    parser.add_option('--cc-bin', default=None,
+                      help='Set path to compiler')
+    parser.add_option('--root-dir', metavar='D', default='.',
+                      help='Set directory to execute from (default %default)')
+
+    parser.add_option('--branch', metavar='B', default=None,
+                      help='Specify branch being built')
+
+    parser.add_option('--add-travis-folds', action='store_true', default=False,
+                      help='Add fold markers for Travis UI')
+
+    parser.add_option('--dry-run', action='store_true', default=False,
+                      help='Just show commands to be executed')
+    parser.add_option('--build-jobs', metavar='J', default=get_concurrency(),
+                      help='Set number of jobs to run in parallel (default %default)')
+
+    parser.add_option('--with-ccache', dest='use_ccache', action='store_true', default=None,
+                      help='Enable using ccache')
+    parser.add_option('--without-ccache', dest='use_ccache', action='store_false',
+                      help='Disable using ccache')
+
+    parser.add_option('--with-python3', dest='use_python3', action='store_true', default=None,
+                      help='Enable using python3')
+    parser.add_option('--without-python3', dest='use_python3', action='store_false',
+                      help='Disable using python3')
+
+    return parser.parse_args(args)
+
+def have_prog(prog):
+    """
+    Check if some named program exists in the path
+    """
+    for path in os.environ['PATH'].split(os.pathsep):
+        exe_file = os.path.join(path, prog)
+        if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
+            return True
+
+def main(args=None):
+    # pylint: disable=too-many-branches,too-many-statements
+    """
+    Parse options, do the things
+    """
+    (options, args) = parse_args(args or sys.argv)
+
+    if len(args) != 2:
+        print('Usage: %s [options] target' % (args[0]))
+        return 1
+
+    if options.use_ccache is None:
+        options.use_ccache = have_prog('ccache')
+
+    use_python2 = have_prog('python2')
+
+    if options.use_python3 is None:
+        use_python3 = have_prog('python3')
+    else:
+        use_python3 = options.use_python3
+
+    if options.cc_bin is None:
+        if options.cc == 'gcc':
+            options.cc_bin = 'g++'
+        elif options.cc == 'clang':
+            options.cc_bin = 'clang++'
+        else:
+            print('Error unknown compiler %s' % (options.cc))
+            return 1
+
+    target = args[1]
+
+    root_dir = options.root_dir
+
+    if os.access(root_dir, os.R_OK) != True:
+        raise Exception('Bad root dir setting, dir %s not readable', root_dir)
+
+    cmds = []
+
+    if target == 'lint':
+
+        if not use_python2 and not use_python3:
+            raise Exception('No python interpreters found cannot lint')
+
+        py_scripts = [
+            'configure.py',
+            'src/python/botan2.py',
+            'src/scripts/ci_build.py',
+            'src/scripts/install.py',
+            'src/scripts/python_unittests.py',
+            'src/scripts/python_unittests_unix.py']
+
+        for target in py_scripts:
+            target_path = os.path.join(root_dir, target)
+
+            if use_python2:
+                # Some disabled rules specific to Python2
+                # superfluous-parens: needed for Python3 compatible print statements
+                # too-many-locals: variable counting differs from pylint3
+                py2_flags = '--disable=superfluous-parens,too-many-locals'
+                cmds.append(['python2', '-m', 'pylint', py2_flags, target_path])
+
+            if use_python3:
+                cmds.append(['python3', '-m', 'pylint', target_path])
+
+    else:
+        config_flags, run_test_command, make_prefix = determine_flags(
+            target, options.os, options.cc, options.cc_bin, options.use_ccache, root_dir)
+
+        cmds.append([os.path.join(root_dir, 'configure.py')] + config_flags)
+
+        if target == 'docs':
+            cmds.append(['make', '-C', root_dir, 'docs'])
+        else:
+            if options.use_ccache:
+                cmds.append(['ccache', '--show-stats'])
+
+            make_targets = ['libs', 'cli', 'tests']
+            if target in ['coverage', 'fuzzers']:
+                make_targets += ['fuzzers', 'fuzzer_corpus_zip']
+
+            cmds.append(make_prefix + ['make', '-C', root_dir] + make_targets)
+
+            if options.use_ccache:
+                cmds.append(['ccache', '--show-stats'])
+
+        if run_test_command != None:
+            cmds.append(run_test_command)
+
+        if target in ['coverage', 'fuzzers']:
+            cmds.append([os.path.join(root_dir, 'src/scripts/test_fuzzers.py'),
+                         os.path.join(root_dir, 'fuzzer_corpus'),
+                         os.path.join(root_dir, 'build/fuzzer')])
+
+        if target in ['static', 'shared']:
+            cmds.append([os.path.join(root_dir, 'src/scripts/cli_tests.py'),
+                         os.path.join(root_dir, 'botan')])
+
+        if target in ['shared', 'coverage']:
+
+            if use_python2:
+                cmds.append(['python2', os.path.join(root_dir, 'src/python/botan2.py')])
+            if use_python3:
+                cmds.append(['python3', os.path.join(root_dir, 'src/python/botan2.py')])
+
+        if target != 'docs':
+            cmds.append(['make', '-C', root_dir, 'install'])
+
+        if target in ['coverage']:
+
+            if not have_prog('lcov'):
+                print('Error: lcov not found in PATH (%s)' % (os.getenv('PATH')))
+                return 1
+
+            if not have_prog('gcov'):
+                print('Error: gcov not found in PATH (%s)' % (os.getenv('PATH')))
+                return 1
+
+            cmds.append(['lcov', '--capture', '--directory', options.root_dir, '--output-file', 'coverage.info.raw'])
+            cmds.append(['lcov', '--remove', 'coverage.info.raw', '/usr/*', '--output-file', 'coverage.info'])
+            cmds.append(['lcov', '--list', 'coverage.info'])
+
+            if have_prog('coverage'):
+                cmds.append(['coverage', 'run', '--branch', os.path.join(root_dir, 'src/python/botan2.py')])
+
+            if have_prog('codecov'):
+                # If codecov exists assume we are on Travis and report to codecov.io
+                cmds.append(['codecov'])
+            else:
+                # Otherwise generate a local HTML report
+                cmds.append(['genhtml', 'coverage.info', '--output-directory', 'lcov-out'])
+
+    for cmd in cmds:
+        if options.dry_run:
+            print('$ ' + ' '.join(cmd))
+        else:
+            run_cmd(cmd, root_dir)
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/src/scripts/lcov.sh b/src/scripts/lcov.sh
deleted file mode 100755
index 9de253069..000000000
--- a/src/scripts/lcov.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-./configure.py --with-debug-info --with-coverage-info --with-bzip2 --with-lzma --with-sqlite --with-zlib --with-sqlite3
-
-make -l4 -j$(nproc) -k
-./botan-test --pkcs11-lib=/usr/lib/libsofthsm2.so --run-online-tests
-
-#LCOV_OPTIONS="--rc lcov_branch_coverage=1"
-LCOV_OPTIONS=""
-
-rm -f coverage.info coverage.info.raw
-lcov $LCOV_OPTIONS --capture --directory . --output-file coverage.info.raw
-lcov $LCOV_OPTIONS  --remove coverage.info.raw '/usr/*' --output-file coverage.info
-genhtml $LCOV_OPTIONS coverage.info --output-directory lcov-out