We've always been at war with shell scripts

5 files changed, 52 insertions, 333 deletions

diff --git a/src/scripts/ci/travis/after_success.sh b/src/scripts/ci/travis/after_success.sh
deleted file mode 100755
index 66612fa10..000000000
--- a/src/scripts/ci/travis/after_success.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-if [ "$BUILD_MODE" = "coverage" ]; then
-    GCOV="/usr/bin/gcov-4.8"
-    /tmp/bin/lcov --gcov-tool "$GCOV" --directory . --capture --output-file "$(pwd)/coverage.info.in"
-    /tmp/bin/lcov --gcov-tool "$GCOV" --remove "$(pwd)/coverage.info.in" 'tests/*' '/usr/*' --output-file "$(pwd)/coverage.info"
-    /tmp/bin/lcov --gcov-tool "$GCOV" --list "$(pwd)/coverage.info"
-
-    LD_LIBRARY_PATH=. coverage run --branch src/python/botan2.py
-
-    codecov
-fi
diff --git a/src/scripts/ci/travis/build.sh b/src/scripts/ci/travis/build.sh
deleted file mode 100755
index bbf4e986c..000000000
--- a/src/scripts/ci/travis/build.sh
+++ /dev/null
@@ -1,216 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-MAKE_PREFIX=()
-TEST_PREFIX=()
-TEST_EXE=./botan-test
-TEST_FLAGS=()
-CLI_EXE=./botan
-CFG_FLAGS=(--prefix=/tmp/botan-installation --cc=$CC --os=$TRAVIS_OS_NAME)
-
-CC_BIN=$CXX
-
-if [ "$BUILD_MODE" = "static" ] || [ "$BUILD_MODE" = "mini-static" ]; then
-    CFG_FLAGS+=(--disable-shared --amalgamation)
-elif [ "$BUILD_MODE" = "shared" ] || [ "$BUILD_MODE" = "mini-shared" ]; then
-    # No special flags required for shared lib build
-    CFG_FLAGS+=()
-elif [ "$BUILD_MODE" = "bsi" ]; then
-    CFG_FLAGS+=(--module-policy=bsi)
-elif [ "$BUILD_MODE" = "nist" ]; then
-    CFG_FLAGS+=(--module-policy=nist)
-elif [ "$BUILD_MODE" = "sonarqube" ]; then
-    # No special flags required
-    CFG_FLAGS+=()
-elif [ "$BUILD_MODE" = "fuzzers" ]; then
-    CFG_FLAGS+=(--build-fuzzers=test --with-sanitizers --with-debug-info --disable-shared)
-elif [ "$BUILD_MODE" = "parallel" ]; then
-
-    if [ "$CC" = "gcc" ]; then
-        CFG_FLAGS+=(--with-cilkplus)
-    else
-        CFG_FLAGS+=(--with-openmp)
-    fi
-
-elif [ "$BUILD_MODE" = "coverage" ]; then
-    CFG_FLAGS+=(--with-coverage --build-fuzzers=test --no-optimizations)
-elif [ "$BUILD_MODE" = "sanitizer" ]; then
-    export ASAN_OPTIONS=detect_leaks=0
-    CFG_FLAGS+=(--with-sanitizers --disable-modules=locking_allocator)
-elif [ "$BUILD_MODE" = "valgrind" ]; then
-    CFG_FLAGS+=(--with-valgrind --with-debug-info --disable-modules=locking_allocator)
-    TEST_PREFIX=(valgrind --error-exitcode=9 -v)
-fi
-
-if [ "$BUILD_MODE" = "mini-static" ] || [ "$BUILD_MODE" = "mini-shared" ]; then
-    CFG_FLAGS+=(--minimized-build --enable-modules="base,dev_random,system_rng,sha2_32,sha2_64,aes")
-elif [ "$BUILD_MODE" = "valgrind" ]; then
-    # Valgrind on Travis on full build takes too long and the job is killed
-    # Prune to the most important stuff
-    CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-
-elif [ "${BUILD_MODE:0:5}" != "cross" ]; then
-    # Only use external libraries when compiling natively
-    CFG_FLAGS+=(--with-bzip2 --with-lzma --with-sqlite --with-zlib)
-
-    if [ "$BUILD_MODE" = "coverage" ]; then
-        CFG_FLAGS+=(--with-tpm)
-        TEST_FLAGS=(--run-long-tests --run-online-tests --pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so)
-    fi
-
-    # Avoid OpenSSL when using dynamic checkers, or on OS X where it sporadically
-    # is not installed on the CI image
-    if [ "$TRAVIS_OS_NAME" != "osx" ] && [ "$BUILD_MODE" != "sanitizer" ] && [ "$BUILD_MODE" != "valgrind" ]; then
-        CFG_FLAGS+=(--with-openssl)
-    fi
-fi
-
-if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "${BUILD_MODE:0:5}" != "cross" ]; then
-    # The Boost-specific codepaths are tested via the OS X CI
-    CFG_FLAGS+=(--with-boost)
-fi
-
-if [ "${BUILD_MODE:0:6}" = "cross-" ]; then
-
-    if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-        CFG_FLAGS+=(--disable-shared)
-        MAKE_PREFIX=(xcrun --sdk iphoneos)
-        if [ "$BUILD_MODE" = "cross-arm32" ]; then
-            CFG_FLAGS+=(--os=ios)
-            CFG_FLAGS+=(--cpu=armv7 --cc-abi-flags="-arch armv7 -arch armv7s -stdlib=libc++")
-        elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-            CFG_FLAGS+=(--os=ios)
-            CFG_FLAGS+=(--cpu=armv8-a --cc-abi-flags="-arch arm64 -stdlib=libc++")
-        fi
-    elif [ "$TRAVIS_OS_NAME" = "linux" ]; then
-        CFG_FLAGS+=(--disable-modules=ffi)
-
-        if [ "$BUILD_MODE" = "cross-arm32" ]; then
-            CC_BIN=arm-linux-gnueabihf-g++-4.8
-            TEST_PREFIX=(qemu-arm -L /usr/arm-linux-gnueabihf/)
-            CFG_FLAGS+=(--cpu=armv7)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-            CC_BIN=aarch64-linux-gnu-g++-4.8
-            TEST_PREFIX=(qemu-aarch64 -L /usr/aarch64-linux-gnu/)
-            CFG_FLAGS+=(--cpu=armv8-a)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
-            CC_BIN=powerpc-linux-gnu-g++-4.8
-            TEST_PREFIX=(qemu-ppc -L /usr/powerpc-linux-gnu/)
-            CFG_FLAGS+=(--cpu=ppc32)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
-            CC_BIN=powerpc64le-linux-gnu-g++-4.8
-            TEST_PREFIX=(qemu-ppc64le -L /usr/powerpc64le-linux-gnu/)
-            CFG_FLAGS+=(--cpu=ppc64 --with-endian=little)
-            CFG_FLAGS+=(--module-policy=modern --enable-modules=tls)
-        elif [ "$BUILD_MODE" = "cross-win32" ]; then
-            CC_BIN=i686-w64-mingw32-g++
-            # No test prefix needed, PE executes as usual with Wine installed
-            CFG_FLAGS+=(--os=mingw)
-            CFG_FLAGS+=(--cpu=x86_32 --cc-abi-flags="-static" --disable-shared)
-            TEST_EXE=./botan-test.exe
-        fi
-    fi
-fi
-
-CFG_FLAGS+=(--cc-bin="ccache $CC_BIN")
-
-if [ "$BUILD_MODE" = "sonarqube" ]; then
-   MAKE_PREFIX=(./build-wrapper-linux-x86/build-wrapper-linux-x86-64 --out-dir bw-outputs)
-fi
-
-# configure
-./configure.py "${CFG_FLAGS[@]}"
-
-# pre-build ccache stats
-ccache --show-stats
-
-# build!
-
-if [ "$BUILD_MODE" = "docs" ]; then
-    doxygen build/botan.doxy
-    sphinx-build -a -W -c src/build-data/sphinx doc/manual manual-out
-else
-    MAKE_CMD=("${MAKE_PREFIX[@]}" make -j "$BUILD_JOBS")
-    echo "Running" "${MAKE_CMD[@]}"
-    time "${MAKE_CMD[@]}"
-fi
-
-if [ "$BUILD_MODE" = "fuzzers" ] || [ "$BUILD_MODE" = "coverage" ]; then
-    make fuzzers
-    make fuzzer_corpus_zip
-fi
-
-# post-build ccache stats
-ccache --show-stats
-
-# Run SonarQube analysis
-
-if [ "$BUILD_MODE" = "sonarqube" ]; then
-
-    cp src/build-data/sonar-project.properties .
-
-    if [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
-       # => This will run a full analysis of the project and push results to the SonarQube server.
-       #
-       # Analysis is done only on master so that build of branches don't push analyses to the same project and therefore "pollute" the results
-       echo "Starting analysis by SonarQube..."
-       sonar-scanner "-Dsonar.login=$SONAR_TOKEN"
-
-    # PR analysis deactivated at least until custom quality profiles can be created
-    elif false && [ "$TRAVIS_PULL_REQUEST" != "false" ] && [ -n "${GITHUB_TOKEN-}" ]; then
-        # => This will analyse the PR and display found issues as comments in the PR, but it won't push results to the SonarQube server
-        #
-        # For security reasons environment variables are not available on the pull requests
-        # coming from outside repositories
-        # http://docs.travis-ci.com/user/pull-requests/#Security-Restrictions-when-testing-Pull-Requests
-        # That's why the analysis does not need to be executed if the variable GITHUB_TOKEN is not defined.
-        echo "Starting Pull Request analysis by SonarQube..."
-        sonar-scanner -Dsonar.login="$SONAR_TOKEN" \
-                      -Dsonar.analysis.mode=preview \
-                      -Dsonar.github.oauth="$GITHUB_TOKEN" \
-                      -Dsonar.github.repository="$TRAVIS_REPO_SLUG" \
-                      -Dsonar.github.pullRequest="$TRAVIS_PULL_REQUEST"
-    fi
-       # When neither on master branch nor on a non-external pull request => nothing to do
-    fi
-
-if [ "$BUILD_MODE" = "sonarqube" ] || [ "$BUILD_MODE" = "docs" ] || 
-       ( [ "${BUILD_MODE:0:5}" = "cross" ] && [ "$TRAVIS_OS_NAME" = "osx" ] ); then
-    echo "Running tests disabled on this build type"
-
-elif [ "$BUILD_MODE" != "fuzzers" ]; then
-    TEST_CMD=("${TEST_PREFIX[@]}" $TEST_EXE "${TEST_FLAGS[@]}")
-    echo "Running" "${TEST_CMD[@]}"
-    time "${TEST_CMD[@]}"
-fi
-
-if [ "$BUILD_MODE" = "fuzzers" ] || [ "$BUILD_MODE" = "coverage" ]; then
-    # Test each fuzzer against its corpus
-    LD_LIBRARY_PATH=. ./src/scripts/test_fuzzers.py fuzzer_corpus build/fuzzer
-fi
-
-if [ "$BUILD_MODE" = "static" ] || [ "$BUILD_MODE" = "shared" ]
-then
-    echo "Running cli tests ..."
-    ./src/scripts/cli_tests.py "$CLI_EXE"
-fi
-
-# Run Python tests (need shared libs)
-if [ "$BUILD_MODE" = "shared" ] || [ "$BUILD_MODE" = "coverage" ];
-then
-    # TODO: find all things in PATH that begin with python- and execute them :)
-    for py in python2 python3
-    do
-        $py --version
-        LD_LIBRARY_PATH=. $py src/python/botan2.py
-    done
-fi
-
-if [ "$BUILD_MODE" != "docs" ]; then
-    # Test make install
-    make install
-fi
diff --git a/src/scripts/ci/travis/install.sh b/src/scripts/ci/travis/install.sh
index bea57cc82..5e65e12d6 100755
--- a/src/scripts/ci/travis/install.sh
+++ b/src/scripts/ci/travis/install.sh
@@ -2,41 +2,52 @@
 set -ev
 which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
 
-if [ "$BUILD_MODE" = "coverage" ]; then
-    wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.13.orig.tar.gz
-    tar -xvf lcov_1.13.orig.tar.gz
-    make PREFIX="/tmp" -C lcov-1.13/ install
-
-    pip install --user coverage
-
-    pip install --user codecov
-fi
-
-if [ "$BUILD_MODE" = "sonarqube" ]; then
-    curl -LsS https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip > build-wrapper-linux-x86.zip
-    unzip build-wrapper-linux-x86.zip
-fi
-
 if [ "$TRAVIS_OS_NAME" = "linux" ]; then
     sudo apt-get -qq update
 
-    if [ "$BUILD_MODE" = "lint" ]; then
+    if [ "$BUILD_MODE" = "valgrind" ]; then
+        sudo apt-get install valgrind
+
+    elif [ "$BUILD_MODE" = "cross-win32" ]; then
+        sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev
+
+        # See https://github.com/travis-ci/travis-ci/issues/6460
+        sudo dpkg --add-architecture i386
+        sudo apt-get -qq update # have to update again due to adding i386 above
+        sudo apt-get install wine
+
+    elif [ "${BUILD_MODE:0:5}" = "cross" ]; then
+         # Need updated qemu
+         sudo add-apt-repository -y ppa:ubuntu-cloud-archive/kilo-staging
+         sudo apt-get -qq update
+         sudo apt-get install qemu
+
+         if [ "$BUILD_MODE" = "cross-arm32" ]; then
+             sudo apt-get install g++-arm-linux-gnueabihf libc6-dev-armhf-cross
+         elif [ "$BUILD_MODE" = "cross-arm64" ]; then
+             sudo apt-get install g++-aarch64-linux-gnu libc6-dev-arm64-cross
+         elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
+             sudo apt-get install g++-powerpc-linux-gnu libc6-dev-powerpc-cross
+         elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
+             sudo apt-get install g++-powerpc64le-linux-gnu libc6-dev-ppc64el-cross
+         fi
+
+    elif [ "$BUILD_MODE" = "lint" ]; then
         pip install --user pylint
         pip3 install --user pylint
-    fi
 
-    if [ "$BUILD_MODE" = "docs" ]; then
-        sudo apt-get install doxygen
+    elif [ "$BUILD_MODE" = "coverage" ]; then
+        # Need new lcov to handle GCC 4.8
+        wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.13.orig.tar.gz
+        tar -xvf lcov_1.13.orig.tar.gz
+        make PREFIX="/tmp" -C lcov-1.13/ install
+        export PATH=/tmp/bin:$PATH
 
-        # The version of Sphinx in 14.04 is too old (1.2.2) and does not support
-        # all C++ features used in the manual. Install python-requests to avoid
-        # problem in Ubuntu packaged version, see
-        # http://stackoverflow.com/questions/32779919/no-module-named-for-requests
-        sudo apt-get remove python-requests python-openssl
-        sudo pip install requests sphinx pyopenssl
-    fi
+        (cd /tmp/bin && ln -s gcov-4.8 gcov)
+
+        pip install --user coverage
+        pip install --user codecov
 
-    if [ "$BUILD_MODE" = "coverage" ]; then
         sudo apt-get install trousers libtspi-dev
 
         # SoftHSMv1 in 14.04 does not work
@@ -44,39 +55,23 @@ if [ "$TRAVIS_OS_NAME" = "linux" ]; then
         wget https://www.randombit.net/softhsm2-trusty-bin.tar.bz2
         tar -C / -xvjf softhsm2-trusty-bin.tar.bz2
         /tmp/softhsm/bin/softhsm2-util --init-token --free --label test --pin 123456 --so-pin 12345678
-    fi
 
-    if [ "$BUILD_MODE" = "valgrind" ] || [ "${BUILD_MODE:0:5}" = "cross" ]; then
-        if [ "$BUILD_MODE" = "valgrind" ]; then
-            sudo apt-get install valgrind
-        elif [ "$BUILD_MODE" = "cross-win32" ]; then
-            sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev
-
-            # See https://github.com/travis-ci/travis-ci/issues/6460
-            sudo dpkg --add-architecture i386
-            sudo apt-get -qq update # have to update again due to adding i386 above
-            sudo apt-get install wine
-        else
-
-            # Need updated qemu
-            sudo add-apt-repository -y ppa:ubuntu-cloud-archive/kilo-staging
-            sudo apt-get -qq update
-            sudo apt-get install qemu
-
-            if [ "$BUILD_MODE" = "cross-arm32" ]; then
-                sudo apt-get install g++-4.8-arm-linux-gnueabihf libc6-dev-armhf-cross
-            elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-                sudo apt-get install g++-4.8-aarch64-linux-gnu libc6-dev-arm64-cross
-            elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
-                sudo apt-get install g++-4.8-powerpc-linux-gnu libc6-dev-powerpc-cross
-            elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
-                sudo apt-get install g++-4.8-powerpc64le-linux-gnu libc6-dev-ppc64el-cross
-            fi
-        fi
+    elif [ "$BUILD_MODE" = "sonarqube" ]; then
+        curl -LsS https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip > build-wrapper-linux-x86.zip
+        unzip build-wrapper-linux-x86.zip
+
+    elif [ "$BUILD_MODE" = "docs" ]; then
+        sudo apt-get install doxygen
+
+        # The version of Sphinx in 14.04 is too old (1.2.2) and does not support
+        # all C++ features used in the manual. Install python-requests to avoid
+        # problem in Ubuntu packaged version, see
+        # http://stackoverflow.com/questions/32779919/no-module-named-for-requests
+        sudo apt-get remove python-requests python-openssl
+        sudo pip install requests sphinx pyopenssl
     fi
-fi
 
-if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
     # Workaround for https://github.com/Homebrew/homebrew/issues/42553
     brew update || brew update
 
diff --git a/src/scripts/ci/travis/lint.sh b/src/scripts/ci/travis/lint.sh
deleted file mode 100755
index 65f885365..000000000
--- a/src/scripts/ci/travis/lint.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-shopt -s expand_aliases
-
-# Disabled rules in Python 2 only
-# superfluous-parens: needed for print statements
-# too-many-locals: variable counting differs from pylint3
-alias python2_pylint='python2 -m pylint --disable=superfluous-parens,too-many-locals'
-alias python3_pylint='python3 -m pylint'
-
-echo "travis_fold:start:pylint_configure"
-python2_pylint configure.py
-python3_pylint configure.py
-echo "travis_fold:end:pylint_configure"
-
-echo "travis_fold:start:pylint_install"
-python2_pylint src/scripts/install.py
-python3_pylint src/scripts/install.py
-echo "travis_fold:end:pylint_configure"
-
-echo "travis_fold:start:pylint_python_unittests"
-python3_pylint src/scripts/python_unittests.py
-echo "travis_fold:end:pylint_python_unittests"
-
-echo "travis_fold:start:pylint_python_unittests_unix"
-python3_pylint src/scripts/python_unittests_unix.py
-echo "travis_fold:end:pylint_python_unittests_unix"
-
-echo "travis_fold:start:pylint_botanpy"
-python2_pylint src/python/botan2.py
-python3_pylint src/python/botan2.py
-echo "travis_fold:end:pylint_botanpy"
diff --git a/src/scripts/ci/travis/main.sh b/src/scripts/ci/travis/main.sh
deleted file mode 100755
index 2f883af06..000000000
--- a/src/scripts/ci/travis/main.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
-
-PARENT_DIR=$(dirname "$0")
-
-if [ "$BUILD_MODE" = "lint" ]; then
-    "$PARENT_DIR"/lint.sh
-else
-
-    ./src/scripts/ci_build.py --build-jobs=2 --with-ccache --os=$TRAVIS_OS_NAME --cc=$CC --cc-bin=$CXX $BUILD_MODE
-fi