We've always been at war with shell scripts

8 files changed, 208 insertions, 415 deletions

diff --git a/src/scripts/ci/codecov/codecov.yml b/src/scripts/ci/codecov.yml
index db26a6bd5..db26a6bd5 100644
--- a/src/scripts/ci/codecov/codecov.yml
+++ b/src/scripts/ci/codecov.yml
diff --git a/src/scripts/ci/travis/after_success.sh b/src/scripts/ci/travis/after_success.sh
deleted file mode 100755
index 66612fa10..000000000
--- a/src/scripts/ci/travis/after_success.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-if [ "$BUILD_MODE" = "coverage" ]; then
-    GCOV="/usr/bin/gcov-4.8"
-    /tmp/bin/lcov --gcov-tool "$GCOV" --directory . --capture --output-file "$(pwd)/coverage.info.in"
-    /tmp/bin/lcov --gcov-tool "$GCOV" --remove "$(pwd)/coverage.info.in" 'tests/*' '/usr/*' --output-file "$(pwd)/coverage.info"
-    /tmp/bin/lcov --gcov-tool "$GCOV" --list "$(pwd)/coverage.info"
-
-    LD_LIBRARY_PATH=. coverage run --branch src/python/botan2.py
-
-    codecov
-fi
diff --git a/src/scripts/ci/travis/build.sh b/src/scripts/ci/travis/build.sh
deleted file mode 100755
index bbf4e986c..000000000
--- a/src/scripts/ci/travis/build.sh
+++ /dev/null
@@ -1,216 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-MAKE_PREFIX=()
-TEST_PREFIX=()
-TEST_EXE=./botan-test
-TEST_FLAGS=()
-CLI_EXE=./botan
-CFG_FLAGS=(--prefix=/tmp/botan-installation --cc=$CC --os=$TRAVIS_OS_NAME)
-
-CC_BIN=$CXX
-
-if [ "$BUILD_MODE" = "static" ] || [ "$BUILD_MODE" = "mini-static" ]; then
-    CFG_FLAGS+=(--disable-shared --amalgamation)
-elif [ "$BUILD_MODE" = "shared" ] || [ "$BUILD_MODE" = "mini-shared" ]; then
-    # No special flags required for shared lib build
-    CFG_FLAGS+=()
-elif [ "$BUILD_MODE" = "bsi" ]; then
-    CFG_FLAGS+=(--module-policy=bsi)
-elif [ "$BUILD_MODE" = "nist" ]; then
-    CFG_FLAGS+=(--module-policy=nist)
-elif [ "$BUILD_MODE" = "sonarqube" ]; then
-    # No special flags required
-    CFG_FLAGS+=()
-elif [ "$BUILD_MODE" = "fuzzers" ]; then
-    CFG_FLAGS+=(--build-fuzzers=test --with-sanitizers --with-debug-info --disable-shared)
-elif [ "$BUILD_MODE" = "parallel" ]; then
-
-    if [ "$CC" = "gcc" ]; then
-        CFG_FLAGS+=(--with-cilkplus)
-    else
-        CFG_FLAGS+=(--with-openmp)
-    fi
-
-elif [ "$BUILD_MODE" = "coverage" ]; then
-    CFG_FLAGS+=(--with-coverage --build-fuzzers=test --no-optimizations)
-elif [ "$BUILD_MODE" = "sanitizer" ]; then
-    export ASAN_OPTIONS=detect_leaks=0
-    CFG_FLAGS+=(--with-sanitizers --disable-modules=locking_allocator)
-elif [ "$BUILD_MODE" = "valgrind" ]; then
-    CFG_FLAGS+=(--with-valgrind --with-debug-info --disable-modules=locking_allocator)
-    TEST_PREFIX=(valgrind --error-exitcode=9 -v)
-fi
-
-if [ "$BUILD_MODE" = "mini-static" ] || [ "$BUILD_MODE" = "mini-shared" ]; then
-    CFG_FLAGS+=(--minimized-build --enable-modules="base,dev_random,system_rng,sha2_32,sha2_64,aes")
-elif [ "$BUILD_MODE" = "valgrind" ]; then
-    # Valgrind on Travis on full build takes too long and the job is killed
-    # Prune to the most important stuff
-    CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-
-elif [ "${BUILD_MODE:0:5}" != "cross" ]; then
-    # Only use external libraries when compiling natively
-    CFG_FLAGS+=(--with-bzip2 --with-lzma --with-sqlite --with-zlib)
-
-    if [ "$BUILD_MODE" = "coverage" ]; then
-        CFG_FLAGS+=(--with-tpm)
-        TEST_FLAGS=(--run-long-tests --run-online-tests --pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so)
-    fi
-
-    # Avoid OpenSSL when using dynamic checkers, or on OS X where it sporadically
-    # is not installed on the CI image
-    if [ "$TRAVIS_OS_NAME" != "osx" ] && [ "$BUILD_MODE" != "sanitizer" ] && [ "$BUILD_MODE" != "valgrind" ]; then
-        CFG_FLAGS+=(--with-openssl)
-    fi
-fi
-
-if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "${BUILD_MODE:0:5}" != "cross" ]; then
-    # The Boost-specific codepaths are tested via the OS X CI
-    CFG_FLAGS+=(--with-boost)
-fi
-
-if [ "${BUILD_MODE:0:6}" = "cross-" ]; then
-
-    if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-        CFG_FLAGS+=(--disable-shared)
-        MAKE_PREFIX=(xcrun --sdk iphoneos)
-        if [ "$BUILD_MODE" = "cross-arm32" ]; then
-            CFG_FLAGS+=(--os=ios)
-            CFG_FLAGS+=(--cpu=armv7 --cc-abi-flags="-arch armv7 -arch armv7s -stdlib=libc++")
-        elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-            CFG_FLAGS+=(--os=ios)
-            CFG_FLAGS+=(--cpu=armv8-a --cc-abi-flags="-arch arm64 -stdlib=libc++")
-        fi
-    elif [ "$TRAVIS_OS_NAME" = "linux" ]; then
-        CFG_FLAGS+=(--disable-modules=ffi)
-
-        if [ "$BUILD_MODE" = "cross-arm32" ]; then
-            CC_BIN=arm-linux-gnueabihf-g++-4.8
-            TEST_PREFIX=(qemu-arm -L /usr/arm-linux-gnueabihf/)
-            CFG_FLAGS+=(--cpu=armv7)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-            CC_BIN=aarch64-linux-gnu-g++-4.8
-            TEST_PREFIX=(qemu-aarch64 -L /usr/aarch64-linux-gnu/)
-            CFG_FLAGS+=(--cpu=armv8-a)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
-            CC_BIN=powerpc-linux-gnu-g++-4.8
-            TEST_PREFIX=(qemu-ppc -L /usr/powerpc-linux-gnu/)
-            CFG_FLAGS+=(--cpu=ppc32)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
-            CC_BIN=powerpc64le-linux-gnu-g++-4.8
-            TEST_PREFIX=(qemu-ppc64le -L /usr/powerpc64le-linux-gnu/)
-            CFG_FLAGS+=(--cpu=ppc64 --with-endian=little)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-win32" ]; then
-            CC_BIN=i686-w64-mingw32-g++
-            # No test prefix needed, PE executes as usual with Wine installed
-            CFG_FLAGS+=(--os=mingw)
-            CFG_FLAGS+=(--cpu=x86_32 --cc-abi-flags="-static" --disable-shared)
-            TEST_EXE=./botan-test.exe
-        fi
-    fi
-fi
-
-CFG_FLAGS+=(--cc-bin="ccache $CC_BIN")
-
-if [ "$BUILD_MODE" = "sonarqube" ]; then
-   MAKE_PREFIX=(./build-wrapper-linux-x86/build-wrapper-linux-x86-64 --out-dir bw-outputs)
-fi
-
-# configure
-./configure.py "${CFG_FLAGS[@]}"
-
-# pre-build ccache stats
-ccache --show-stats
-
-# build!
-
-if [ "$BUILD_MODE" = "docs" ]; then
-    doxygen build/botan.doxy
-    sphinx-build -a -W -c src/build-data/sphinx doc/manual manual-out
-else
-    MAKE_CMD=("${MAKE_PREFIX[@]}" make -j "$BUILD_JOBS")
-    echo "Running" "${MAKE_CMD[@]}"
-    time "${MAKE_CMD[@]}"
-fi
-
-if [ "$BUILD_MODE" = "fuzzers" ] || [ "$BUILD_MODE" = "coverage" ]; then
-    make fuzzers
-    make fuzzer_corpus_zip
-fi
-
-# post-build ccache stats
-ccache --show-stats
-
-# Run SonarQube analysis
-
-if [ "$BUILD_MODE" = "sonarqube" ]; then
-
-    cp src/build-data/sonar-project.properties .
-
-    if [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
-       # => This will run a full analysis of the project and push results to the SonarQube server.
-       #
-       # Analysis is done only on master so that build of branches don't push analyses to the same project and therefore "pollute" the results
-       echo "Starting analysis by SonarQube..."
-       sonar-scanner "-Dsonar.login=$SONAR_TOKEN"
-
-    # PR analysis deactivated at least until custom quality profiles can be created
-    elif false && [ "$TRAVIS_PULL_REQUEST" != "false" ] && [ -n "${GITHUB_TOKEN-}" ]; then
-        # => This will analyse the PR and display found issues as comments in the PR, but it won't push results to the SonarQube server
-        #
-        # For security reasons environment variables are not available on the pull requests
-        # coming from outside repositories
-        # http://docs.travis-ci.com/user/pull-requests/#Security-Restrictions-when-testing-Pull-Requests
-        # That's why the analysis does not need to be executed if the variable GITHUB_TOKEN is not defined.
-        echo "Starting Pull Request analysis by SonarQube..."
-        sonar-scanner -Dsonar.login="$SONAR_TOKEN" \
-                      -Dsonar.analysis.mode=preview \
-                      -Dsonar.github.oauth="$GITHUB_TOKEN" \
-                      -Dsonar.github.repository="$TRAVIS_REPO_SLUG" \
-                      -Dsonar.github.pullRequest="$TRAVIS_PULL_REQUEST"
-    fi
-       # When neither on master branch nor on a non-external pull request => nothing to do
-    fi
-
-if [ "$BUILD_MODE" = "sonarqube" ] || [ "$BUILD_MODE" = "docs" ] || 
-       ( [ "${BUILD_MODE:0:5}" = "cross" ] && [ "$TRAVIS_OS_NAME" = "osx" ] ); then
-    echo "Running tests disabled on this build type"
-
-elif [ "$BUILD_MODE" != "fuzzers" ]; then
-    TEST_CMD=("${TEST_PREFIX[@]}" $TEST_EXE "${TEST_FLAGS[@]}")
-    echo "Running" "${TEST_CMD[@]}"
-    time "${TEST_CMD[@]}"
-fi
-
-if [ "$BUILD_MODE" = "fuzzers" ] || [ "$BUILD_MODE" = "coverage" ]; then
-    # Test each fuzzer against its corpus
-    LD_LIBRARY_PATH=. ./src/scripts/test_fuzzers.py fuzzer_corpus build/fuzzer
-fi
-
-if [ "$BUILD_MODE" = "static" ] || [ "$BUILD_MODE" = "shared" ]
-then
-    echo "Running cli tests ..."
-    ./src/scripts/cli_tests.py "$CLI_EXE"
-fi
-
-# Run Python tests (need shared libs)
-if [ "$BUILD_MODE" = "shared" ] || [ "$BUILD_MODE" = "coverage" ];
-then
-    # TODO: find all things in PATH that begin with python- and execute them :)
-    for py in python2 python3
-    do
-        $py --version
-        LD_LIBRARY_PATH=. $py src/python/botan2.py
-    done
-fi
-
-if [ "$BUILD_MODE" != "docs" ]; then
-    # Test make install
-    make install
-fi
diff --git a/src/scripts/ci/travis/install.sh b/src/scripts/ci/travis/install.sh
index bea57cc82..5e65e12d6 100755
--- a/src/scripts/ci/travis/install.sh
+++ b/src/scripts/ci/travis/install.sh
@@ -2,41 +2,52 @@
 set -ev
 which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
 
-if [ "$BUILD_MODE" = "coverage" ]; then
-    wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.13.orig.tar.gz
-    tar -xvf lcov_1.13.orig.tar.gz
-    make PREFIX="/tmp" -C lcov-1.13/ install
-
-    pip install --user coverage
-
-    pip install --user codecov
-fi
-
-if [ "$BUILD_MODE" = "sonarqube" ]; then
-    curl -LsS https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip > build-wrapper-linux-x86.zip
-    unzip build-wrapper-linux-x86.zip
-fi
-
 if [ "$TRAVIS_OS_NAME" = "linux" ]; then
     sudo apt-get -qq update
 
-    if [ "$BUILD_MODE" = "lint" ]; then
+    if [ "$BUILD_MODE" = "valgrind" ]; then
+        sudo apt-get install valgrind
+
+    elif [ "$BUILD_MODE" = "cross-win32" ]; then
+        sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev
+
+        # See https://github.com/travis-ci/travis-ci/issues/6460
+        sudo dpkg --add-architecture i386
+        sudo apt-get -qq update # have to update again due to adding i386 above
+        sudo apt-get install wine
+
+    elif [ "${BUILD_MODE:0:5}" = "cross" ]; then
+         # Need updated qemu
+         sudo add-apt-repository -y ppa:ubuntu-cloud-archive/kilo-staging
+         sudo apt-get -qq update
+         sudo apt-get install qemu
+
+         if [ "$BUILD_MODE" = "cross-arm32" ]; then
+             sudo apt-get install g++-arm-linux-gnueabihf libc6-dev-armhf-cross
+         elif [ "$BUILD_MODE" = "cross-arm64" ]; then
+             sudo apt-get install g++-aarch64-linux-gnu libc6-dev-arm64-cross
+         elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
+             sudo apt-get install g++-powerpc-linux-gnu libc6-dev-powerpc-cross
+         elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
+             sudo apt-get install g++-powerpc64le-linux-gnu libc6-dev-ppc64el-cross
+         fi
+
+    elif [ "$BUILD_MODE" = "lint" ]; then
         pip install --user pylint
         pip3 install --user pylint
-    fi
 
-    if [ "$BUILD_MODE" = "docs" ]; then
-        sudo apt-get install doxygen
+    elif [ "$BUILD_MODE" = "coverage" ]; then
+        # Need new lcov to handle GCC 4.8
+        wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.13.orig.tar.gz
+        tar -xvf lcov_1.13.orig.tar.gz
+        make PREFIX="/tmp" -C lcov-1.13/ install
+        export PATH=/tmp/bin:$PATH
 
-        # The version of Sphinx in 14.04 is too old (1.2.2) and does not support
-        # all C++ features used in the manual. Install python-requests to avoid
-        # problem in Ubuntu packaged version, see
-        # http://stackoverflow.com/questions/32779919/no-module-named-for-requests
-        sudo apt-get remove python-requests python-openssl
-        sudo pip install requests sphinx pyopenssl
-    fi
+        (cd /tmp/bin && ln -s gcov-4.8 gcov)
+
+        pip install --user coverage
+        pip install --user codecov
 
-    if [ "$BUILD_MODE" = "coverage" ]; then
         sudo apt-get install trousers libtspi-dev
 
         # SoftHSMv1 in 14.04 does not work
@@ -44,39 +55,23 @@ if [ "$TRAVIS_OS_NAME" = "linux" ]; then
         wget https://www.randombit.net/softhsm2-trusty-bin.tar.bz2
         tar -C / -xvjf softhsm2-trusty-bin.tar.bz2
         /tmp/softhsm/bin/softhsm2-util --init-token --free --label test --pin 123456 --so-pin 12345678
-    fi
 
-    if [ "$BUILD_MODE" = "valgrind" ] || [ "${BUILD_MODE:0:5}" = "cross" ]; then
-        if [ "$BUILD_MODE" = "valgrind" ]; then
-            sudo apt-get install valgrind
-        elif [ "$BUILD_MODE" = "cross-win32" ]; then
-            sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev
-
-            # See https://github.com/travis-ci/travis-ci/issues/6460
-            sudo dpkg --add-architecture i386
-            sudo apt-get -qq update # have to update again due to adding i386 above
-            sudo apt-get install wine
-        else
-
-            # Need updated qemu
-            sudo add-apt-repository -y ppa:ubuntu-cloud-archive/kilo-staging
-            sudo apt-get -qq update
-            sudo apt-get install qemu
-
-            if [ "$BUILD_MODE" = "cross-arm32" ]; then
-                sudo apt-get install g++-4.8-arm-linux-gnueabihf libc6-dev-armhf-cross
-            elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-                sudo apt-get install g++-4.8-aarch64-linux-gnu libc6-dev-arm64-cross
-            elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
-                sudo apt-get install g++-4.8-powerpc-linux-gnu libc6-dev-powerpc-cross
-            elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
-                sudo apt-get install g++-4.8-powerpc64le-linux-gnu libc6-dev-ppc64el-cross
-            fi
-        fi
+    elif [ "$BUILD_MODE" = "sonarqube" ]; then
+        curl -LsS https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip > build-wrapper-linux-x86.zip
+        unzip build-wrapper-linux-x86.zip
+
+    elif [ "$BUILD_MODE" = "docs" ]; then
+        sudo apt-get install doxygen
+
+        # The version of Sphinx in 14.04 is too old (1.2.2) and does not support
+        # all C++ features used in the manual. Install python-requests to avoid
+        # problem in Ubuntu packaged version, see
+        # http://stackoverflow.com/questions/32779919/no-module-named-for-requests
+        sudo apt-get remove python-requests python-openssl
+        sudo pip install requests sphinx pyopenssl
     fi
-fi
 
-if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
     # Workaround for https://github.com/Homebrew/homebrew/issues/42553
     brew update || brew update
 
diff --git a/src/scripts/ci/travis/lint.sh b/src/scripts/ci/travis/lint.sh
deleted file mode 100755
index 65f885365..000000000
--- a/src/scripts/ci/travis/lint.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-shopt -s expand_aliases
-
-# Disabled rules in Python 2 only
-# superfluous-parens: needed for print statements
-# too-many-locals: variable counting differs from pylint3
-alias python2_pylint='python2 -m pylint --disable=superfluous-parens,too-many-locals'
-alias python3_pylint='python3 -m pylint'
-
-echo "travis_fold:start:pylint_configure"
-python2_pylint configure.py
-python3_pylint configure.py
-echo "travis_fold:end:pylint_configure"
-
-echo "travis_fold:start:pylint_install"
-python2_pylint src/scripts/install.py
-python3_pylint src/scripts/install.py
-echo "travis_fold:end:pylint_configure"
-
-echo "travis_fold:start:pylint_python_unittests"
-python3_pylint src/scripts/python_unittests.py
-echo "travis_fold:end:pylint_python_unittests"
-
-echo "travis_fold:start:pylint_python_unittests_unix"
-python3_pylint src/scripts/python_unittests_unix.py
-echo "travis_fold:end:pylint_python_unittests_unix"
-
-echo "travis_fold:start:pylint_botanpy"
-python2_pylint src/python/botan2.py
-python3_pylint src/python/botan2.py
-echo "travis_fold:end:pylint_botanpy"
diff --git a/src/scripts/ci/travis/main.sh b/src/scripts/ci/travis/main.sh
deleted file mode 100755
index 2f883af06..000000000
--- a/src/scripts/ci/travis/main.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-PARENT_DIR=$(dirname "$0")
-
-if [ "$BUILD_MODE" = "lint" ]; then
-    "$PARENT_DIR"/lint.sh
-else
-
-    ./src/scripts/ci_build.py --build-jobs=2 --with-ccache --os=$TRAVIS_OS_NAME --cc=$CC --cc-bin=$CXX $BUILD_MODE
-fi
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
index 01602f41b..86f4d23a4 100755
--- a/src/scripts/ci_build.py
+++ b/src/scripts/ci_build.py
@@ -1,31 +1,53 @@
 #!/usr/bin/python
 
-# CI build script
-# (C) 2017 Jack Lloyd
-# Botan is released under the Simplified BSD License (see license.txt)
+"""
+CI build script
+(C) 2017 Jack Lloyd
+Botan is released under the Simplified BSD License (see license.txt)
+"""
 
-import time
-import subprocess
-import optparse
+import os
 import platform
+import subprocess
 import sys
-import os
+import time
+import optparse # pylint: disable=deprecated-module
+
+def get_concurrency():
+    """
+    Get default concurrency level of build
+    """
+    def_concurrency = 2
+
+    try:
+        import multiprocessing
+        return max(def_concurrency, multiprocessing.cpu_count())
+    except ImportError:
+        return def_concurrency
 
 def getenv_or_die(var):
+    """
+    Like it says...
+    """
     val = os.getenv(var)
     if val is None:
         raise Exception('Required variable %s not set in environment' % (var))
     return val
 
-def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir):
+def determine_flags(target, target_os, target_cc, cc_bin, use_ccache, root_dir):
+    # pylint: disable=too-many-branches,too-many-statements,too-many-arguments
+
+    """
+    Return the configure.py flags as well as make/test running prefixes
+    """
     is_cross_target = target.startswith('cross-')
 
     if target_os not in ['linux', 'osx']:
         print('Error unknown OS %s' % (target_os))
         return 1
 
-    if cc not in ['gcc', 'clang']:
-        print('Error unknown compiler %s' % (cc))
+    if target_cc not in ['gcc', 'clang']:
+        print('Error unknown compiler %s' % (target_cc))
         return 1
 
     if is_cross_target:
@@ -38,7 +60,7 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir):
     test_prefix = []
     test_cmd = [os.path.join(root_dir, 'botan-test')]
 
-    flags = ['--prefix=/tmp/botan-install', '--cc=%s' % (cc), '--os=%s' % (target_os)]
+    flags = ['--prefix=/tmp/botan-install', '--cc=%s' % (target_cc), '--os=%s' % (target_os)]
 
     if target in ['static', 'mini-static', 'fuzzers'] or target_os in ['ios', 'mingw']:
         flags += ['--disable-shared']
@@ -56,14 +78,8 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir):
     if target == 'docs':
         flags += ['--with-doxygen', '--with-sphinx']
 
-    if target == 'parallel':
-        if 'cc' == 'gcc':
-            flags += ['--with-cilkplus']
-        else:
-            flags += ['--with-openmp']
-
     if target == 'coverage':
-        flags += ['--with-coverage']
+        flags += ['--with-coverage-info']
     if target == 'valgrind':
         flags += ['--with-valgrind']
         test_prefix = ['valgrind', '--error-exitcode=9', '-v']
@@ -77,8 +93,15 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir):
     if target in ['valgrind', 'sanitizer', 'fuzzers']:
         flags += ['--disable-modules=locking_allocator']
 
+    if target == 'parallel':
+        if 'cc' == 'gcc':
+            flags += ['--with-cilkplus']
+        else:
+            flags += ['--with-openmp']
+
     if target == 'sonarqube':
-        make_prefix = [os.path.join(root_dir, 'build-wrapper-linux-x86/build-wrapper-linux-x86-64'), '--out-dir', 'bw-outputs']
+        make_prefix = [os.path.join(root_dir, 'build-wrapper-linux-x86/build-wrapper-linux-x86-64'),
+                       '--out-dir', 'bw-outputs']
         test_cmd = ['sonar-scanner', '-Dsonar.login=%s' % (getenv_or_die('SONAR_TOKEN'))]
 
     if target_os == 'linux' and (target == 'valgrind' or is_cross_target):
@@ -96,29 +119,28 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir):
                 flags += ['--cpu=armv8-a', '--cc-abi-flags=-arch arm64 -stdlib=libc++']
             else:
                 raise Exception("Unknown cross target '%s' for iOS" % (target))
+        elif target == 'cross-win32':
+            flags += ['--cpu=x86_32', '--cc-abi-flags=-static']
+            cc_bin = 'i686-w64-mingw32-g++'
+            test_cmd = os.path.join(root_dir, 'botan-test.exe')
+            # No runtime prefix required for Wine
         else:
-
             if target == 'cross-arm32':
                 flags += ['--cpu=armv7']
-                cc_bin = 'arm-linux-gnueabihf-g++-4.8'
+                cc_bin = 'arm-linux-gnueabihf-g++'
                 test_prefix = ['qemu-arm', '-L', '/usr/arm-linux-gnueabihf/']
             elif target == 'cross-arm64':
                 flags += ['--cpu=armv8-a']
-                cc_bin = 'aarch64-linux-gnu-g++-4.8'
+                cc_bin = 'aarch64-linux-gnu-g++'
                 test_prefix = ['qemu-aarch64', '-L', '/usr/aarch64-linux-gnu/']
             elif target == 'cross-ppc32':
                 flags += ['--cpu=ppc32']
-                cc_bin = 'powerpc-linux-gnu-g++-4.8'
+                cc_bin = 'powerpc-linux-gnu-g++'
                 test_prefix = ['qemu-ppc', '-L', '/usr/powerpc-linux-gnu/']
             elif target == 'cross-ppc64':
                 flags += ['--cpu=ppc64', '--with-endian=little']
-                cc_bin = 'powerpc64le-linux-gnu-g++-4.8'
+                cc_bin = 'powerpc64le-linux-gnu-g++'
                 test_prefix = ['qemu-ppc64le', '-L', '/usr/powerpc64le-linux-gnu/']
-            elif target == 'cross-win32':
-                flags += ['--cpu=x86_32', '--cc-abi-flags=-static']
-                cc_bin = 'i686-w64-mingw32-g++'
-                test_cmd = os.path.join(root_dir, 'botan-test.exe')
-                # No runtime prefix required for Wine
             else:
                 raise Exception("Unknown cross target '%s' for Linux" % (target))
     else:
@@ -127,7 +149,8 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir):
 
         if target == 'coverage':
             flags += ['--with-tpm']
-            test_cmd += ['--run-long-tests', '--run-online-tests', '--pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so']
+            test_cmd += ['--run-long-tests', '--run-online-tests',
+                         '--pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so']
 
         if target_os == 'osx':
             # Test Boost on OS X
@@ -146,15 +169,17 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir):
 
     return flags, run_test_command, make_prefix
 
-def run_cmd(cmd):
-
+def run_cmd(cmd, root_dir):
+    """
+    Execute a command, die if it failed
+    """
     print("Running '%s':\n" % (' '.join(cmd)))
     sys.stdout.flush()
 
     start = time.time()
 
-    # TODO pass LD_LIBRARY_PATH=.
-    proc = subprocess.Popen(cmd, close_fds=True)
+    env={'LD_LIBRARY_PATH': root_dir, 'PATH': os.getenv('PATH')}
+    proc = subprocess.Popen(cmd, close_fds=True, env=env)
     proc.communicate()
 
     time_taken = time.time() - start
@@ -165,10 +190,10 @@ def run_cmd(cmd):
     if proc.returncode != 0:
         raise Exception("Command failed with error code %d" % (proc.returncode))
 
-def main(args=None):
-    if args is None:
-        args = sys.argv
-
+def setup_option_parser():
+    """
+    Return an OptionParser instance
+    """
     parser = optparse.OptionParser()
 
     parser.add_option('--os', default=platform.system().lower(),
@@ -183,21 +208,46 @@ def main(args=None):
     parser.add_option('--branch', metavar='B', default=None,
                       help='Specify branch being built')
 
+    parser.add_option('--add-travis-folds', action='store_true', default=False,
+                      help='Add fold markers for Travis UI')
+
     parser.add_option('--dry-run', action='store_true', default=False,
                       help='Just show commands to be executed')
-    parser.add_option('--build-jobs', metavar='J', default='2',
+    parser.add_option('--build-jobs', metavar='J', default=get_concurrency(),
                       help='Set number of jobs to run in parallel (default %default)')
-    parser.add_option('--without-ccache', dest='use_ccache', action='store_false', default=True,
+
+    parser.add_option('--with-ccache', dest='use_ccache', action='store_true', default=None,
+                      help='Enable using ccache')
+    parser.add_option('--without-ccache', dest='use_ccache', action='store_false',
                       help='Disable using ccache')
+    return parser
+
+def have_prog(prog):
+    """
+    Check if some named program exists in the path
+    """
+    for path in os.environ['PATH'].split(os.pathsep):
+        exe_file = os.path.join(path, prog)
+        if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
+            return True
 
+def main(args=None):
+    # pylint: disable=too-many-branches
+    """
+    Parse options, do the things
+    """
+    if args is None:
+        args = sys.argv
+
+    parser = setup_option_parser()
     (options, args) = parser.parse_args(args)
 
     if len(args) != 2:
         print('Usage: %s [options] target' % (args[0]))
         return 1
 
-    if options.use_ccache == None:
-        options.use_ccache = True
+    if options.use_ccache is None:
+        options.ccache = have_prog('ccache')
 
     target = args[1]
 
@@ -206,50 +256,88 @@ def main(args=None):
     if os.access(root_dir, os.R_OK) != True:
         raise Exception('Bad root dir setting, dir %s not readable', root_dir)
 
-    config_flags, run_test_command, make_prefix = determine_flags(
-        target, options.os, options.cc, options.cc_bin, options.use_ccache, root_dir)
-
     cmds = []
 
-    cmds.append([os.path.join(root_dir, 'configure.py')] + config_flags)
+    if target == 'lint':
+
+        py_scripts = [
+            'configure.py',
+            'src/python/botan2.py',
+            'src/scripts/ci_build.py',
+            'src/scripts/install.py',
+            'src/scripts/python_unittests.py',
+            'src/scripts/python_unittests_unix.py']
+
+        for target in py_scripts:
+            target_path = os.path.join(root_dir, target)
+
+            # Some disabled rules specific to Python2
+            # superfluous-parens: needed for Python3 compatible print statements
+            # too-many-locals: variable counting differs from pylint3
+
+            py2_flags = '--disable=superfluous-parens,too-many-locals'
+            cmds.append(['python2', '-m', 'pylint', py2_flags, target_path])
+            cmds.append(['python3', '-m', 'pylint', target_path])
 
-    if target == 'docs':
-        cmds.append(['make', '-C', root_dir, 'docs'])
     else:
-        if options.use_ccache:
-            cmds.append(['ccache', '--show-stats'])
+        config_flags, run_test_command, make_prefix = determine_flags(
+            target, options.os, options.cc, options.cc_bin, options.use_ccache, root_dir)
+
+        cmds.append([os.path.join(root_dir, 'configure.py')] + config_flags)
 
-        cmds.append(make_prefix + ['make', '-C', root_dir, '-j', str(options.build_jobs)])
+        if target == 'docs':
+            cmds.append(['make', '-C', root_dir, 'docs'])
+        else:
+            if options.use_ccache:
+                cmds.append(['ccache', '--show-stats'])
+
+            cmds.append(make_prefix + ['make', '-C', root_dir, '-j', str(options.build_jobs)])
+
+            if target in ['coverage', 'fuzzers']:
+                cmds.append(make_prefix + ['make', '-C', root_dir, 'fuzzers', 'fuzzer_corpus_zip'])
+
+            if options.use_ccache:
+                cmds.append(['ccache', '--show-stats'])
+
+        if run_test_command != None:
+            cmds.append(run_test_command)
 
         if target in ['coverage', 'fuzzers']:
-            cmds.append(make_prefix + ['make', '-C', root_dir, 'fuzzers', 'fuzzer_corpus_zip'])
+            cmds.append([os.path.join(root_dir, 'src/scripts/test_fuzzers.py'),
+                         os.path.join(root_dir, 'fuzzer_corpus'),
+                         os.path.join(root_dir, 'build/fuzzer')])
 
-        if options.use_ccache:
-            cmds.append(['ccache', '--show-stats'])
+        if target in ['static', 'shared']:
+            cmds.append([os.path.join(root_dir, 'src/scripts/cli_tests.py'),
+                         os.path.join(root_dir, 'botan')])
 
-    if run_test_command != None:
-        cmds.append(run_test_command)
+        if target in ['shared', 'coverage']:
+            cmds.append(['python2', os.path.join(root_dir, 'src/python/botan2.py')])
+            cmds.append(['python3', os.path.join(root_dir, 'src/python/botan2.py')])
 
-    if target in ['coverage', 'fuzzers']:
-        cmds.append([os.path.join(root_dir, 'src/scripts/test_fuzzers.py'),
-                     os.path.join(root_dir, 'fuzzer_corpus'),
-                     os.path.join(root_dir, 'build/fuzzer')])
+        if target != 'docs':
+            cmds.append(['make', 'install'])
 
-    if target in ['static', 'shared']:
-        cmds.append([os.path.join(root_dir, 'src/scripts/cli_tests.py'), os.path.join(root_dir, 'botan')])
+        if target in ['coverage']:
+            cmds.append(['lcov', '--capture', '--directory', options.root_dir, '--output-file', 'coverage.info.raw'])
+            cmds.append(['lcov', '--remove', 'coverage.info.raw', '/usr/*', '--output-file', 'coverage.info'])
+            cmds.append(['lcov', '--list', 'coverage.info'])
 
-    if target in ['shared', 'coverage']:
-        cmds.append(['python2', os.path.join(root_dir, 'src/python/botan2.py')])
-        cmds.append(['python3', os.path.join(root_dir, 'src/python/botan2.py')])
+            if have_prog('coverage'):
+                cmds.append(['coverage', 'run', '--branch', os.path.join(root_dir,'src/python/botan2.py')])
 
-    if target != 'docs':
-        cmds.append(['make', 'install'])
+            if have_prog('codecov'):
+                # If codecov exists assume we are on Travis and report to codecov.io
+                cmds.append(['codecov'])
+            else:
+                # Otherwise generate a local HTML report
+                cmds.append(['genhtml', 'coverage.info', '--output-directory', 'lcov-out'])
 
     for cmd in cmds:
         if options.dry_run:
             print('$ ' + ' '.join(cmd))
         else:
-            run_cmd(cmd)
+            run_cmd(cmd, root_dir)
 
     return 0
 
diff --git a/src/scripts/lcov.sh b/src/scripts/lcov.sh
deleted file mode 100755
index 9de253069..000000000
--- a/src/scripts/lcov.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-./configure.py --with-debug-info --with-coverage-info --with-bzip2 --with-lzma --with-sqlite --with-zlib --with-sqlite3
-
-make -l4 -j$(nproc) -k
-./botan-test --pkcs11-lib=/usr/lib/libsofthsm2.so --run-online-tests
-
-#LCOV_OPTIONS="--rc lcov_branch_coverage=1"
-LCOV_OPTIONS=""
-
-rm -f coverage.info coverage.info.raw
-lcov $LCOV_OPTIONS --capture --directory . --output-file coverage.info.raw
-lcov $LCOV_OPTIONS  --remove coverage.info.raw '/usr/*' --output-file coverage.info
-genhtml $LCOV_OPTIONS coverage.info --output-directory lcov-out