diff options
| -rw-r--r-- | .travis.yml | 6 | ||||
| -rw-r--r-- | src/scripts/ci/codecov.yml (renamed from src/scripts/ci/codecov/codecov.yml) | 0 | ||||
| -rwxr-xr-x | src/scripts/ci/travis/after_success.sh | 14 | ||||
| -rwxr-xr-x | src/scripts/ci/travis/build.sh | 216 | ||||
| -rwxr-xr-x | src/scripts/ci/travis/install.sh | 109 | ||||
| -rwxr-xr-x | src/scripts/ci/travis/lint.sh | 34 | ||||
| -rwxr-xr-x | src/scripts/ci/travis/main.sh | 12 | ||||
| -rwxr-xr-x | src/scripts/ci_build.py | 224 | ||||
| -rwxr-xr-x | src/scripts/lcov.sh | 14 |
9 files changed, 209 insertions, 420 deletions
diff --git a/.travis.yml b/.travis.yml index 266c1686a..22eca2147 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,6 @@ env: global: - CCACHE_SIZE=50M - CCACHE_COMPRESS=1 - - BUILD_JOBS=2 # SONAR_TOKEN for accessing the SonarQube server - secure: "VezRbHFg6kllV5WG06M3tG3aHJaC3xrMylJ6RCVbL+uz2JeralVCqV7eIk4fVb9cu83Li+weEa0AJj0wkxpIUJ+vUh5F65L6gSWSbgHP7muOSVsmnEc6KvX4n3av/ZGe4geSmsxqh2pd/2xI1h7KioGRhKeqaZIdjVgWgGJW2iQ=" @@ -98,10 +97,7 @@ install: - ./src/scripts/ci/travis/install.sh script: - - if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then ./src/scripts/ci/travis/main.sh ; fi - -after_success: - - ./src/scripts/ci/travis/after_success.sh + - if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then ./src/scripts/ci_build.py --os=$TRAVIS_OS_NAME --cc=$CC --cc-bin=$CXX $BUILD_MODE; fi # whitelist branches to avoid testing feature branches twice (as branch and as pull request) branches: diff --git a/src/scripts/ci/codecov/codecov.yml b/src/scripts/ci/codecov.yml index db26a6bd5..db26a6bd5 100644 --- a/src/scripts/ci/codecov/codecov.yml +++ b/src/scripts/ci/codecov.yml diff --git a/src/scripts/ci/travis/after_success.sh b/src/scripts/ci/travis/after_success.sh deleted file mode 100755 index 66612fa10..000000000 --- a/src/scripts/ci/travis/after_success.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -set -ev -which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available - -if [ "$BUILD_MODE" = "coverage" ]; then - GCOV="/usr/bin/gcov-4.8" - /tmp/bin/lcov --gcov-tool "$GCOV" --directory . --capture --output-file "$(pwd)/coverage.info.in" - /tmp/bin/lcov --gcov-tool "$GCOV" --remove "$(pwd)/coverage.info.in" 'tests/*' '/usr/*' --output-file "$(pwd)/coverage.info" - /tmp/bin/lcov --gcov-tool "$GCOV" --list "$(pwd)/coverage.info" - - LD_LIBRARY_PATH=. coverage run --branch src/python/botan2.py - - codecov -fi diff --git a/src/scripts/ci/travis/build.sh b/src/scripts/ci/travis/build.sh deleted file mode 100755 index bbf4e986c..000000000 --- a/src/scripts/ci/travis/build.sh +++ /dev/null @@ -1,216 +0,0 @@ -#!/bin/bash -set -ev -which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available - -MAKE_PREFIX=() -TEST_PREFIX=() -TEST_EXE=./botan-test -TEST_FLAGS=() -CLI_EXE=./botan -CFG_FLAGS=(--prefix=/tmp/botan-installation --cc=$CC --os=$TRAVIS_OS_NAME) - -CC_BIN=$CXX - -if [ "$BUILD_MODE" = "static" ] || [ "$BUILD_MODE" = "mini-static" ]; then - CFG_FLAGS+=(--disable-shared --amalgamation) -elif [ "$BUILD_MODE" = "shared" ] || [ "$BUILD_MODE" = "mini-shared" ]; then - # No special flags required for shared lib build - CFG_FLAGS+=() -elif [ "$BUILD_MODE" = "bsi" ]; then - CFG_FLAGS+=(--module-policy=bsi) -elif [ "$BUILD_MODE" = "nist" ]; then - CFG_FLAGS+=(--module-policy=nist) -elif [ "$BUILD_MODE" = "sonarqube" ]; then - # No special flags required - CFG_FLAGS+=() -elif [ "$BUILD_MODE" = "fuzzers" ]; then - CFG_FLAGS+=(--build-fuzzers=test --with-sanitizers --with-debug-info --disable-shared) -elif [ "$BUILD_MODE" = "parallel" ]; then - - if [ "$CC" = "gcc" ]; then - CFG_FLAGS+=(--with-cilkplus) - else - CFG_FLAGS+=(--with-openmp) - fi - -elif [ "$BUILD_MODE" = "coverage" ]; then - CFG_FLAGS+=(--with-coverage --build-fuzzers=test --no-optimizations) -elif [ "$BUILD_MODE" = "sanitizer" ]; then - export ASAN_OPTIONS=detect_leaks=0 - CFG_FLAGS+=(--with-sanitizers --disable-modules=locking_allocator) -elif [ "$BUILD_MODE" = "valgrind" ]; then - CFG_FLAGS+=(--with-valgrind --with-debug-info --disable-modules=locking_allocator) - TEST_PREFIX=(valgrind --error-exitcode=9 -v) -fi - -if [ "$BUILD_MODE" = "mini-static" ] || [ "$BUILD_MODE" = "mini-shared" ]; then - CFG_FLAGS+=(--minimized-build --enable-modules="base,dev_random,system_rng,sha2_32,sha2_64,aes") -elif [ "$BUILD_MODE" = "valgrind" ]; then - # Valgrind on Travis on full build takes too long and the job is killed - # Prune to the most important stuff - CFG_FLAGS+=(--module-policy=modern --enable-modules=tls) - -elif [ "${BUILD_MODE:0:5}" != "cross" ]; then - # Only use external libraries when compiling natively - CFG_FLAGS+=(--with-bzip2 --with-lzma --with-sqlite --with-zlib) - - if [ "$BUILD_MODE" = "coverage" ]; then - CFG_FLAGS+=(--with-tpm) - TEST_FLAGS=(--run-long-tests --run-online-tests --pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so) - fi - - # Avoid OpenSSL when using dynamic checkers, or on OS X where it sporadically - # is not installed on the CI image - if [ "$TRAVIS_OS_NAME" != "osx" ] && [ "$BUILD_MODE" != "sanitizer" ] && [ "$BUILD_MODE" != "valgrind" ]; then - CFG_FLAGS+=(--with-openssl) - fi -fi - -if [ "$TRAVIS_OS_NAME" = "osx" ] && [ "${BUILD_MODE:0:5}" != "cross" ]; then - # The Boost-specific codepaths are tested via the OS X CI - CFG_FLAGS+=(--with-boost) -fi - -if [ "${BUILD_MODE:0:6}" = "cross-" ]; then - - if [ "$TRAVIS_OS_NAME" = "osx" ]; then - CFG_FLAGS+=(--disable-shared) - MAKE_PREFIX=(xcrun --sdk iphoneos) - if [ "$BUILD_MODE" = "cross-arm32" ]; then - CFG_FLAGS+=(--os=ios) - CFG_FLAGS+=(--cpu=armv7 --cc-abi-flags="-arch armv7 -arch armv7s -stdlib=libc++") - elif [ "$BUILD_MODE" = "cross-arm64" ]; then - CFG_FLAGS+=(--os=ios) - CFG_FLAGS+=(--cpu=armv8-a --cc-abi-flags="-arch arm64 -stdlib=libc++") - fi - elif [ "$TRAVIS_OS_NAME" = "linux" ]; then - CFG_FLAGS+=(--disable-modules=ffi) - - if [ "$BUILD_MODE" = "cross-arm32" ]; then - CC_BIN=arm-linux-gnueabihf-g++-4.8 - TEST_PREFIX=(qemu-arm -L /usr/arm-linux-gnueabihf/) - CFG_FLAGS+=(--cpu=armv7) - CFG_FLAGS+=(--module-policy=modern --enable-modules=tls) - elif [ "$BUILD_MODE" = "cross-arm64" ]; then - CC_BIN=aarch64-linux-gnu-g++-4.8 - TEST_PREFIX=(qemu-aarch64 -L /usr/aarch64-linux-gnu/) - CFG_FLAGS+=(--cpu=armv8-a) - CFG_FLAGS+=(--module-policy=modern --enable-modules=tls) - elif [ "$BUILD_MODE" = "cross-ppc32" ]; then - CC_BIN=powerpc-linux-gnu-g++-4.8 - TEST_PREFIX=(qemu-ppc -L /usr/powerpc-linux-gnu/) - CFG_FLAGS+=(--cpu=ppc32) - CFG_FLAGS+=(--module-policy=modern --enable-modules=tls) - elif [ "$BUILD_MODE" = "cross-ppc64" ]; then - CC_BIN=powerpc64le-linux-gnu-g++-4.8 - TEST_PREFIX=(qemu-ppc64le -L /usr/powerpc64le-linux-gnu/) - CFG_FLAGS+=(--cpu=ppc64 --with-endian=little) - CFG_FLAGS+=(--module-policy=modern --enable-modules=tls) - elif [ "$BUILD_MODE" = "cross-win32" ]; then - CC_BIN=i686-w64-mingw32-g++ - # No test prefix needed, PE executes as usual with Wine installed - CFG_FLAGS+=(--os=mingw) - CFG_FLAGS+=(--cpu=x86_32 --cc-abi-flags="-static" --disable-shared) - TEST_EXE=./botan-test.exe - fi - fi -fi - -CFG_FLAGS+=(--cc-bin="ccache $CC_BIN") - -if [ "$BUILD_MODE" = "sonarqube" ]; then - MAKE_PREFIX=(./build-wrapper-linux-x86/build-wrapper-linux-x86-64 --out-dir bw-outputs) -fi - -# configure -./configure.py "${CFG_FLAGS[@]}" - -# pre-build ccache stats -ccache --show-stats - -# build! - -if [ "$BUILD_MODE" = "docs" ]; then - doxygen build/botan.doxy - sphinx-build -a -W -c src/build-data/sphinx doc/manual manual-out -else - MAKE_CMD=("${MAKE_PREFIX[@]}" make -j "$BUILD_JOBS") - echo "Running" "${MAKE_CMD[@]}" - time "${MAKE_CMD[@]}" -fi - -if [ "$BUILD_MODE" = "fuzzers" ] || [ "$BUILD_MODE" = "coverage" ]; then - make fuzzers - make fuzzer_corpus_zip -fi - -# post-build ccache stats -ccache --show-stats - -# Run SonarQube analysis - -if [ "$BUILD_MODE" = "sonarqube" ]; then - - cp src/build-data/sonar-project.properties . - - if [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then - # => This will run a full analysis of the project and push results to the SonarQube server. - # - # Analysis is done only on master so that build of branches don't push analyses to the same project and therefore "pollute" the results - echo "Starting analysis by SonarQube..." - sonar-scanner "-Dsonar.login=$SONAR_TOKEN" - - # PR analysis deactivated at least until custom quality profiles can be created - elif false && [ "$TRAVIS_PULL_REQUEST" != "false" ] && [ -n "${GITHUB_TOKEN-}" ]; then - # => This will analyse the PR and display found issues as comments in the PR, but it won't push results to the SonarQube server - # - # For security reasons environment variables are not available on the pull requests - # coming from outside repositories - # http://docs.travis-ci.com/user/pull-requests/#Security-Restrictions-when-testing-Pull-Requests - # That's why the analysis does not need to be executed if the variable GITHUB_TOKEN is not defined. - echo "Starting Pull Request analysis by SonarQube..." - sonar-scanner -Dsonar.login="$SONAR_TOKEN" \ - -Dsonar.analysis.mode=preview \ - -Dsonar.github.oauth="$GITHUB_TOKEN" \ - -Dsonar.github.repository="$TRAVIS_REPO_SLUG" \ - -Dsonar.github.pullRequest="$TRAVIS_PULL_REQUEST" - fi - # When neither on master branch nor on a non-external pull request => nothing to do - fi - -if [ "$BUILD_MODE" = "sonarqube" ] || [ "$BUILD_MODE" = "docs" ] || - ( [ "${BUILD_MODE:0:5}" = "cross" ] && [ "$TRAVIS_OS_NAME" = "osx" ] ); then - echo "Running tests disabled on this build type" - -elif [ "$BUILD_MODE" != "fuzzers" ]; then - TEST_CMD=("${TEST_PREFIX[@]}" $TEST_EXE "${TEST_FLAGS[@]}") - echo "Running" "${TEST_CMD[@]}" - time "${TEST_CMD[@]}" -fi - -if [ "$BUILD_MODE" = "fuzzers" ] || [ "$BUILD_MODE" = "coverage" ]; then - # Test each fuzzer against its corpus - LD_LIBRARY_PATH=. ./src/scripts/test_fuzzers.py fuzzer_corpus build/fuzzer -fi - -if [ "$BUILD_MODE" = "static" ] || [ "$BUILD_MODE" = "shared" ] -then - echo "Running cli tests ..." - ./src/scripts/cli_tests.py "$CLI_EXE" -fi - -# Run Python tests (need shared libs) -if [ "$BUILD_MODE" = "shared" ] || [ "$BUILD_MODE" = "coverage" ]; -then - # TODO: find all things in PATH that begin with python- and execute them :) - for py in python2 python3 - do - $py --version - LD_LIBRARY_PATH=. $py src/python/botan2.py - done -fi - -if [ "$BUILD_MODE" != "docs" ]; then - # Test make install - make install -fi diff --git a/src/scripts/ci/travis/install.sh b/src/scripts/ci/travis/install.sh index bea57cc82..5e65e12d6 100755 --- a/src/scripts/ci/travis/install.sh +++ b/src/scripts/ci/travis/install.sh @@ -2,41 +2,52 @@ set -ev which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available -if [ "$BUILD_MODE" = "coverage" ]; then - wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.13.orig.tar.gz - tar -xvf lcov_1.13.orig.tar.gz - make PREFIX="/tmp" -C lcov-1.13/ install - - pip install --user coverage - - pip install --user codecov -fi - -if [ "$BUILD_MODE" = "sonarqube" ]; then - curl -LsS https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip > build-wrapper-linux-x86.zip - unzip build-wrapper-linux-x86.zip -fi - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get -qq update - if [ "$BUILD_MODE" = "lint" ]; then + if [ "$BUILD_MODE" = "valgrind" ]; then + sudo apt-get install valgrind + + elif [ "$BUILD_MODE" = "cross-win32" ]; then + sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev + + # See https://github.com/travis-ci/travis-ci/issues/6460 + sudo dpkg --add-architecture i386 + sudo apt-get -qq update # have to update again due to adding i386 above + sudo apt-get install wine + + elif [ "${BUILD_MODE:0:5}" = "cross" ]; then + # Need updated qemu + sudo add-apt-repository -y ppa:ubuntu-cloud-archive/kilo-staging + sudo apt-get -qq update + sudo apt-get install qemu + + if [ "$BUILD_MODE" = "cross-arm32" ]; then + sudo apt-get install g++-arm-linux-gnueabihf libc6-dev-armhf-cross + elif [ "$BUILD_MODE" = "cross-arm64" ]; then + sudo apt-get install g++-aarch64-linux-gnu libc6-dev-arm64-cross + elif [ "$BUILD_MODE" = "cross-ppc32" ]; then + sudo apt-get install g++-powerpc-linux-gnu libc6-dev-powerpc-cross + elif [ "$BUILD_MODE" = "cross-ppc64" ]; then + sudo apt-get install g++-powerpc64le-linux-gnu libc6-dev-ppc64el-cross + fi + + elif [ "$BUILD_MODE" = "lint" ]; then pip install --user pylint pip3 install --user pylint - fi - if [ "$BUILD_MODE" = "docs" ]; then - sudo apt-get install doxygen + elif [ "$BUILD_MODE" = "coverage" ]; then + # Need new lcov to handle GCC 4.8 + wget http://ftp.de.debian.org/debian/pool/main/l/lcov/lcov_1.13.orig.tar.gz + tar -xvf lcov_1.13.orig.tar.gz + make PREFIX="/tmp" -C lcov-1.13/ install + export PATH=/tmp/bin:$PATH - # The version of Sphinx in 14.04 is too old (1.2.2) and does not support - # all C++ features used in the manual. Install python-requests to avoid - # problem in Ubuntu packaged version, see - # http://stackoverflow.com/questions/32779919/no-module-named-for-requests - sudo apt-get remove python-requests python-openssl - sudo pip install requests sphinx pyopenssl - fi + (cd /tmp/bin && ln -s gcov-4.8 gcov) + + pip install --user coverage + pip install --user codecov - if [ "$BUILD_MODE" = "coverage" ]; then sudo apt-get install trousers libtspi-dev # SoftHSMv1 in 14.04 does not work @@ -44,39 +55,23 @@ if [ "$TRAVIS_OS_NAME" = "linux" ]; then wget https://www.randombit.net/softhsm2-trusty-bin.tar.bz2 tar -C / -xvjf softhsm2-trusty-bin.tar.bz2 /tmp/softhsm/bin/softhsm2-util --init-token --free --label test --pin 123456 --so-pin 12345678 - fi - if [ "$BUILD_MODE" = "valgrind" ] || [ "${BUILD_MODE:0:5}" = "cross" ]; then - if [ "$BUILD_MODE" = "valgrind" ]; then - sudo apt-get install valgrind - elif [ "$BUILD_MODE" = "cross-win32" ]; then - sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev - - # See https://github.com/travis-ci/travis-ci/issues/6460 - sudo dpkg --add-architecture i386 - sudo apt-get -qq update # have to update again due to adding i386 above - sudo apt-get install wine - else - - # Need updated qemu - sudo add-apt-repository -y ppa:ubuntu-cloud-archive/kilo-staging - sudo apt-get -qq update - sudo apt-get install qemu - - if [ "$BUILD_MODE" = "cross-arm32" ]; then - sudo apt-get install g++-4.8-arm-linux-gnueabihf libc6-dev-armhf-cross - elif [ "$BUILD_MODE" = "cross-arm64" ]; then - sudo apt-get install g++-4.8-aarch64-linux-gnu libc6-dev-arm64-cross - elif [ "$BUILD_MODE" = "cross-ppc32" ]; then - sudo apt-get install g++-4.8-powerpc-linux-gnu libc6-dev-powerpc-cross - elif [ "$BUILD_MODE" = "cross-ppc64" ]; then - sudo apt-get install g++-4.8-powerpc64le-linux-gnu libc6-dev-ppc64el-cross - fi - fi + elif [ "$BUILD_MODE" = "sonarqube" ]; then + curl -LsS https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip > build-wrapper-linux-x86.zip + unzip build-wrapper-linux-x86.zip + + elif [ "$BUILD_MODE" = "docs" ]; then + sudo apt-get install doxygen + + # The version of Sphinx in 14.04 is too old (1.2.2) and does not support + # all C++ features used in the manual. Install python-requests to avoid + # problem in Ubuntu packaged version, see + # http://stackoverflow.com/questions/32779919/no-module-named-for-requests + sudo apt-get remove python-requests python-openssl + sudo pip install requests sphinx pyopenssl fi -fi -if [ "$TRAVIS_OS_NAME" = "osx" ]; then +elif [ "$TRAVIS_OS_NAME" = "osx" ]; then # Workaround for https://github.com/Homebrew/homebrew/issues/42553 brew update || brew update diff --git a/src/scripts/ci/travis/lint.sh b/src/scripts/ci/travis/lint.sh deleted file mode 100755 index 65f885365..000000000 --- a/src/scripts/ci/travis/lint.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -ev -which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available - -shopt -s expand_aliases - -# Disabled rules in Python 2 only -# superfluous-parens: needed for print statements -# too-many-locals: variable counting differs from pylint3 -alias python2_pylint='python2 -m pylint --disable=superfluous-parens,too-many-locals' -alias python3_pylint='python3 -m pylint' - -echo "travis_fold:start:pylint_configure" -python2_pylint configure.py -python3_pylint configure.py -echo "travis_fold:end:pylint_configure" - -echo "travis_fold:start:pylint_install" -python2_pylint src/scripts/install.py -python3_pylint src/scripts/install.py -echo "travis_fold:end:pylint_configure" - -echo "travis_fold:start:pylint_python_unittests" -python3_pylint src/scripts/python_unittests.py -echo "travis_fold:end:pylint_python_unittests" - -echo "travis_fold:start:pylint_python_unittests_unix" -python3_pylint src/scripts/python_unittests_unix.py -echo "travis_fold:end:pylint_python_unittests_unix" - -echo "travis_fold:start:pylint_botanpy" -python2_pylint src/python/botan2.py -python3_pylint src/python/botan2.py -echo "travis_fold:end:pylint_botanpy" diff --git a/src/scripts/ci/travis/main.sh b/src/scripts/ci/travis/main.sh deleted file mode 100755 index 2f883af06..000000000 --- a/src/scripts/ci/travis/main.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -ev -which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available - -PARENT_DIR=$(dirname "$0") - -if [ "$BUILD_MODE" = "lint" ]; then - "$PARENT_DIR"/lint.sh -else - - ./src/scripts/ci_build.py --build-jobs=2 --with-ccache --os=$TRAVIS_OS_NAME --cc=$CC --cc-bin=$CXX $BUILD_MODE -fi diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py index 01602f41b..86f4d23a4 100755 --- a/src/scripts/ci_build.py +++ b/src/scripts/ci_build.py @@ -1,31 +1,53 @@ #!/usr/bin/python -# CI build script -# (C) 2017 Jack Lloyd -# Botan is released under the Simplified BSD License (see license.txt) +""" +CI build script +(C) 2017 Jack Lloyd +Botan is released under the Simplified BSD License (see license.txt) +""" -import time -import subprocess -import optparse +import os import platform +import subprocess import sys -import os +import time +import optparse # pylint: disable=deprecated-module + +def get_concurrency(): + """ + Get default concurrency level of build + """ + def_concurrency = 2 + + try: + import multiprocessing + return max(def_concurrency, multiprocessing.cpu_count()) + except ImportError: + return def_concurrency def getenv_or_die(var): + """ + Like it says... + """ val = os.getenv(var) if val is None: raise Exception('Required variable %s not set in environment' % (var)) return val -def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir): +def determine_flags(target, target_os, target_cc, cc_bin, use_ccache, root_dir): + # pylint: disable=too-many-branches,too-many-statements,too-many-arguments + + """ + Return the configure.py flags as well as make/test running prefixes + """ is_cross_target = target.startswith('cross-') if target_os not in ['linux', 'osx']: print('Error unknown OS %s' % (target_os)) return 1 - if cc not in ['gcc', 'clang']: - print('Error unknown compiler %s' % (cc)) + if target_cc not in ['gcc', 'clang']: + print('Error unknown compiler %s' % (target_cc)) return 1 if is_cross_target: @@ -38,7 +60,7 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir): test_prefix = [] test_cmd = [os.path.join(root_dir, 'botan-test')] - flags = ['--prefix=/tmp/botan-install', '--cc=%s' % (cc), '--os=%s' % (target_os)] + flags = ['--prefix=/tmp/botan-install', '--cc=%s' % (target_cc), '--os=%s' % (target_os)] if target in ['static', 'mini-static', 'fuzzers'] or target_os in ['ios', 'mingw']: flags += ['--disable-shared'] @@ -56,14 +78,8 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir): if target == 'docs': flags += ['--with-doxygen', '--with-sphinx'] - if target == 'parallel': - if 'cc' == 'gcc': - flags += ['--with-cilkplus'] - else: - flags += ['--with-openmp'] - if target == 'coverage': - flags += ['--with-coverage'] + flags += ['--with-coverage-info'] if target == 'valgrind': flags += ['--with-valgrind'] test_prefix = ['valgrind', '--error-exitcode=9', '-v'] @@ -77,8 +93,15 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir): if target in ['valgrind', 'sanitizer', 'fuzzers']: flags += ['--disable-modules=locking_allocator'] + if target == 'parallel': + if 'cc' == 'gcc': + flags += ['--with-cilkplus'] + else: + flags += ['--with-openmp'] + if target == 'sonarqube': - make_prefix = [os.path.join(root_dir, 'build-wrapper-linux-x86/build-wrapper-linux-x86-64'), '--out-dir', 'bw-outputs'] + make_prefix = [os.path.join(root_dir, 'build-wrapper-linux-x86/build-wrapper-linux-x86-64'), + '--out-dir', 'bw-outputs'] test_cmd = ['sonar-scanner', '-Dsonar.login=%s' % (getenv_or_die('SONAR_TOKEN'))] if target_os == 'linux' and (target == 'valgrind' or is_cross_target): @@ -96,29 +119,28 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir): flags += ['--cpu=armv8-a', '--cc-abi-flags=-arch arm64 -stdlib=libc++'] else: raise Exception("Unknown cross target '%s' for iOS" % (target)) + elif target == 'cross-win32': + flags += ['--cpu=x86_32', '--cc-abi-flags=-static'] + cc_bin = 'i686-w64-mingw32-g++' + test_cmd = os.path.join(root_dir, 'botan-test.exe') + # No runtime prefix required for Wine else: - if target == 'cross-arm32': flags += ['--cpu=armv7'] - cc_bin = 'arm-linux-gnueabihf-g++-4.8' + cc_bin = 'arm-linux-gnueabihf-g++' test_prefix = ['qemu-arm', '-L', '/usr/arm-linux-gnueabihf/'] elif target == 'cross-arm64': flags += ['--cpu=armv8-a'] - cc_bin = 'aarch64-linux-gnu-g++-4.8' + cc_bin = 'aarch64-linux-gnu-g++' test_prefix = ['qemu-aarch64', '-L', '/usr/aarch64-linux-gnu/'] elif target == 'cross-ppc32': flags += ['--cpu=ppc32'] - cc_bin = 'powerpc-linux-gnu-g++-4.8' + cc_bin = 'powerpc-linux-gnu-g++' test_prefix = ['qemu-ppc', '-L', '/usr/powerpc-linux-gnu/'] elif target == 'cross-ppc64': flags += ['--cpu=ppc64', '--with-endian=little'] - cc_bin = 'powerpc64le-linux-gnu-g++-4.8' + cc_bin = 'powerpc64le-linux-gnu-g++' test_prefix = ['qemu-ppc64le', '-L', '/usr/powerpc64le-linux-gnu/'] - elif target == 'cross-win32': - flags += ['--cpu=x86_32', '--cc-abi-flags=-static'] - cc_bin = 'i686-w64-mingw32-g++' - test_cmd = os.path.join(root_dir, 'botan-test.exe') - # No runtime prefix required for Wine else: raise Exception("Unknown cross target '%s' for Linux" % (target)) else: @@ -127,7 +149,8 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir): if target == 'coverage': flags += ['--with-tpm'] - test_cmd += ['--run-long-tests', '--run-online-tests', '--pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so'] + test_cmd += ['--run-long-tests', '--run-online-tests', + '--pkcs11-lib=/tmp/softhsm/lib/softhsm/libsofthsm2.so'] if target_os == 'osx': # Test Boost on OS X @@ -146,15 +169,17 @@ def determine_flags(target, target_os, cc, cc_bin, use_ccache, root_dir): return flags, run_test_command, make_prefix -def run_cmd(cmd): - +def run_cmd(cmd, root_dir): + """ + Execute a command, die if it failed + """ print("Running '%s':\n" % (' '.join(cmd))) sys.stdout.flush() start = time.time() - # TODO pass LD_LIBRARY_PATH=. - proc = subprocess.Popen(cmd, close_fds=True) + env={'LD_LIBRARY_PATH': root_dir, 'PATH': os.getenv('PATH')} + proc = subprocess.Popen(cmd, close_fds=True, env=env) proc.communicate() time_taken = time.time() - start @@ -165,10 +190,10 @@ def run_cmd(cmd): if proc.returncode != 0: raise Exception("Command failed with error code %d" % (proc.returncode)) -def main(args=None): - if args is None: - args = sys.argv - +def setup_option_parser(): + """ + Return an OptionParser instance + """ parser = optparse.OptionParser() parser.add_option('--os', default=platform.system().lower(), @@ -183,21 +208,46 @@ def main(args=None): parser.add_option('--branch', metavar='B', default=None, help='Specify branch being built') + parser.add_option('--add-travis-folds', action='store_true', default=False, + help='Add fold markers for Travis UI') + parser.add_option('--dry-run', action='store_true', default=False, help='Just show commands to be executed') - parser.add_option('--build-jobs', metavar='J', default='2', + parser.add_option('--build-jobs', metavar='J', default=get_concurrency(), help='Set number of jobs to run in parallel (default %default)') - parser.add_option('--without-ccache', dest='use_ccache', action='store_false', default=True, + + parser.add_option('--with-ccache', dest='use_ccache', action='store_true', default=None, + help='Enable using ccache') + parser.add_option('--without-ccache', dest='use_ccache', action='store_false', help='Disable using ccache') + return parser + +def have_prog(prog): + """ + Check if some named program exists in the path + """ + for path in os.environ['PATH'].split(os.pathsep): + exe_file = os.path.join(path, prog) + if os.path.exists(exe_file) and os.access(exe_file, os.X_OK): + return True +def main(args=None): + # pylint: disable=too-many-branches + """ + Parse options, do the things + """ + if args is None: + args = sys.argv + + parser = setup_option_parser() (options, args) = parser.parse_args(args) if len(args) != 2: print('Usage: %s [options] target' % (args[0])) return 1 - if options.use_ccache == None: - options.use_ccache = True + if options.use_ccache is None: + options.ccache = have_prog('ccache') target = args[1] @@ -206,50 +256,88 @@ def main(args=None): if os.access(root_dir, os.R_OK) != True: raise Exception('Bad root dir setting, dir %s not readable', root_dir) - config_flags, run_test_command, make_prefix = determine_flags( - target, options.os, options.cc, options.cc_bin, options.use_ccache, root_dir) - cmds = [] - cmds.append([os.path.join(root_dir, 'configure.py')] + config_flags) + if target == 'lint': + + py_scripts = [ + 'configure.py', + 'src/python/botan2.py', + 'src/scripts/ci_build.py', + 'src/scripts/install.py', + 'src/scripts/python_unittests.py', + 'src/scripts/python_unittests_unix.py'] + + for target in py_scripts: + target_path = os.path.join(root_dir, target) + + # Some disabled rules specific to Python2 + # superfluous-parens: needed for Python3 compatible print statements + # too-many-locals: variable counting differs from pylint3 + + py2_flags = '--disable=superfluous-parens,too-many-locals' + cmds.append(['python2', '-m', 'pylint', py2_flags, target_path]) + cmds.append(['python3', '-m', 'pylint', target_path]) - if target == 'docs': - cmds.append(['make', '-C', root_dir, 'docs']) else: - if options.use_ccache: - cmds.append(['ccache', '--show-stats']) + config_flags, run_test_command, make_prefix = determine_flags( + target, options.os, options.cc, options.cc_bin, options.use_ccache, root_dir) + + cmds.append([os.path.join(root_dir, 'configure.py')] + config_flags) - cmds.append(make_prefix + ['make', '-C', root_dir, '-j', str(options.build_jobs)]) + if target == 'docs': + cmds.append(['make', '-C', root_dir, 'docs']) + else: + if options.use_ccache: + cmds.append(['ccache', '--show-stats']) + + cmds.append(make_prefix + ['make', '-C', root_dir, '-j', str(options.build_jobs)]) + + if target in ['coverage', 'fuzzers']: + cmds.append(make_prefix + ['make', '-C', root_dir, 'fuzzers', 'fuzzer_corpus_zip']) + + if options.use_ccache: + cmds.append(['ccache', '--show-stats']) + + if run_test_command != None: + cmds.append(run_test_command) if target in ['coverage', 'fuzzers']: - cmds.append(make_prefix + ['make', '-C', root_dir, 'fuzzers', 'fuzzer_corpus_zip']) + cmds.append([os.path.join(root_dir, 'src/scripts/test_fuzzers.py'), + os.path.join(root_dir, 'fuzzer_corpus'), + os.path.join(root_dir, 'build/fuzzer')]) - if options.use_ccache: - cmds.append(['ccache', '--show-stats']) + if target in ['static', 'shared']: + cmds.append([os.path.join(root_dir, 'src/scripts/cli_tests.py'), + os.path.join(root_dir, 'botan')]) - if run_test_command != None: - cmds.append(run_test_command) + if target in ['shared', 'coverage']: + cmds.append(['python2', os.path.join(root_dir, 'src/python/botan2.py')]) + cmds.append(['python3', os.path.join(root_dir, 'src/python/botan2.py')]) - if target in ['coverage', 'fuzzers']: - cmds.append([os.path.join(root_dir, 'src/scripts/test_fuzzers.py'), - os.path.join(root_dir, 'fuzzer_corpus'), - os.path.join(root_dir, 'build/fuzzer')]) + if target != 'docs': + cmds.append(['make', 'install']) - if target in ['static', 'shared']: - cmds.append([os.path.join(root_dir, 'src/scripts/cli_tests.py'), os.path.join(root_dir, 'botan')]) + if target in ['coverage']: + cmds.append(['lcov', '--capture', '--directory', options.root_dir, '--output-file', 'coverage.info.raw']) + cmds.append(['lcov', '--remove', 'coverage.info.raw', '/usr/*', '--output-file', 'coverage.info']) + cmds.append(['lcov', '--list', 'coverage.info']) - if target in ['shared', 'coverage']: - cmds.append(['python2', os.path.join(root_dir, 'src/python/botan2.py')]) - cmds.append(['python3', os.path.join(root_dir, 'src/python/botan2.py')]) + if have_prog('coverage'): + cmds.append(['coverage', 'run', '--branch', os.path.join(root_dir,'src/python/botan2.py')]) - if target != 'docs': - cmds.append(['make', 'install']) + if have_prog('codecov'): + # If codecov exists assume we are on Travis and report to codecov.io + cmds.append(['codecov']) + else: + # Otherwise generate a local HTML report + cmds.append(['genhtml', 'coverage.info', '--output-directory', 'lcov-out']) for cmd in cmds: if options.dry_run: print('$ ' + ' '.join(cmd)) else: - run_cmd(cmd) + run_cmd(cmd, root_dir) return 0 diff --git a/src/scripts/lcov.sh b/src/scripts/lcov.sh deleted file mode 100755 index 9de253069..000000000 --- a/src/scripts/lcov.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -./configure.py --with-debug-info --with-coverage-info --with-bzip2 --with-lzma --with-sqlite --with-zlib --with-sqlite3 - -make -l4 -j$(nproc) -k -./botan-test --pkcs11-lib=/usr/lib/libsofthsm2.so --run-online-tests - -#LCOV_OPTIONS="--rc lcov_branch_coverage=1" -LCOV_OPTIONS="" - -rm -f coverage.info coverage.info.raw -lcov $LCOV_OPTIONS --capture --directory . --output-file coverage.info.raw -lcov $LCOV_OPTIONS --remove coverage.info.raw '/usr/*' --output-file coverage.info -genhtml $LCOV_OPTIONS coverage.info --output-directory lcov-out |