Update Travis build to use Xenial

Drops Mingw i686 build as it runs into a problem I could not debug,
when unit_tls.cpp is compiled with -O2 it crashes with a useless
backtrace. If compiled with -g or with lower optimizations it is ok.
Replaced with Mingw x86-64.

Add mips64 build.

3 files changed, 56 insertions, 84 deletions

diff --git a/src/scripts/ci/setup_travis.sh b/src/scripts/ci/setup_travis.sh
index f03947476..95831f7fe 100755
--- a/src/scripts/ci/setup_travis.sh
+++ b/src/scripts/ci/setup_travis.sh
@@ -1,87 +1,63 @@
 #!/bin/bash
-set -ev
-which shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
 
-git clone --depth 1 https://github.com/randombit/botan-ci-tools
+# Travis CI setup script for Botan build
+#
+# (C) 2015,2017 Simon Warta
+# (C) 2016,2017,2018 Jack Lloyd
 
-if [ "$TRAVIS_OS_NAME" = "linux" ]; then
+command -v shellcheck > /dev/null && shellcheck "$0" # Run shellcheck on this if available
 
-    # ccache in Trusty is too old, use version from Xenial
-    sudo dpkg -i botan-ci-tools/ubuntu/ccache_3.2.4-1_amd64.deb
+set -ev
+
+if [ "$TRAVIS_OS_NAME" = "linux" ]; then
 
     if [ "$BUILD_MODE" = "valgrind" ]; then
         sudo apt-get -qq update
         sudo apt-get install valgrind
 
-    elif [ "$BUILD_MODE" = "cross-win32" ]; then
-        # See https://github.com/travis-ci/travis-ci/issues/6460
-        sudo dpkg --add-architecture i386
-        sudo apt-get -qq update # have to run this after --add-architecture
-        sudo apt-get install wine g++-mingw-w64-i686 mingw-w64-i686-dev
+    elif [ "$BUILD_MODE" = "cross-win64" ]; then
+        sudo apt-get -qq update
+        sudo apt-get install wine g++-mingw-w64-x86-64
 
     elif [ "${BUILD_MODE:0:5}" = "cross" ]; then
-         # Need updated qemu
-         sudo add-apt-repository -y ppa:ubuntu-cloud-archive/kilo-staging
          sudo apt-get -qq update
-         sudo apt-get install qemu-user
 
          if [ "$BUILD_MODE" = "cross-arm32" ]; then
-             sudo apt-get install g++-arm-linux-gnueabihf libc6-dev-armhf-cross
+             sudo apt-get install qemu-user g++-arm-linux-gnueabihf
          elif [ "$BUILD_MODE" = "cross-arm64" ]; then
-             sudo apt-get install g++-aarch64-linux-gnu libc6-dev-arm64-cross
+             sudo apt-get install qemu-user g++-aarch64-linux-gnu
          elif [ "$BUILD_MODE" = "cross-ppc32" ]; then
-             sudo apt-get install g++-powerpc-linux-gnu libc6-dev-powerpc-cross
+             sudo apt-get install qemu-user g++-powerpc-linux-gnu
          elif [ "$BUILD_MODE" = "cross-ppc64" ]; then
-             sudo apt-get install g++-powerpc64le-linux-gnu libc6-dev-ppc64el-cross
+             sudo apt-get install qemu-user g++-powerpc64le-linux-gnu
+         elif [ "$BUILD_MODE" = "cross-mips64" ]; then
+             sudo apt-get install qemu-user g++-mips64-linux-gnuabi64
          fi
 
     elif [ "$BUILD_MODE" = "lint" ]; then
-        pip install --user pylint==1.9.2
+        sudo apt-get -qq update
+        sudo apt-get install pylint
 
     elif [ "$BUILD_MODE" = "coverage" ]; then
         sudo apt-get -qq update
-        sudo apt-get install trousers libtspi-dev
+        sudo apt-get install trousers libtspi-dev lcov python-coverage
 
-        # SoftHSMv1 in 14.04 does not work
+        git clone --depth 1 https://github.com/randombit/botan-ci-tools
+
+        # FIXME use distro softhsm2 package instead
+        # need to figure out ownership problem
         # Installs prebuilt SoftHSMv2 binaries into /tmp
         tar -C / -xvjf botan-ci-tools/softhsm2-trusty-bin.tar.bz2
         /tmp/softhsm/bin/softhsm2-util --init-token --free --label test --pin 123456 --so-pin 12345678
 
-        # need updated lcov for gcc 4.8 coverage format
-        sudo dpkg -i botan-ci-tools/ubuntu/lcov_1.12-2_all.deb
-
-        (cd /home/travis/bin && ln -s gcov-4.8 gcov)
-
-        pip install --user coverage
         pip install --user codecov==2.0.10
 
-    elif [ "$BUILD_MODE" = "sonar" ]; then
-        sudo apt-get -qq update
-        sudo apt-get install trousers libtspi-dev
-        # installed llvm-3.4 conflicts with clang-3.9 in /usr/local
-        # we need a more recent llvm-cov for coverage reports
-        sudo apt-get remove llvm
-
-        tar -C / -xvjf botan-ci-tools/softhsm2-trusty-bin.tar.bz2
-        /tmp/softhsm/bin/softhsm2-util --init-token --free --label test --pin 123456 --so-pin 12345678
-
-        wget https://sonarqube.com/static/cpp/build-wrapper-linux-x86.zip
-        unzip build-wrapper-linux-x86.zip
-
     elif [ "$BUILD_MODE" = "docs" ]; then
         sudo apt-get -qq update
         sudo apt-get install doxygen python-docutils
 
-        # The version of Sphinx in 14.04 is too old (1.2.2) and does not support
-        # all C++ features used in the manual. Install python-requests to avoid
-        # problem in Ubuntu packaged version, see
-        # http://stackoverflow.com/questions/32779919/no-module-named-for-requests
-        #
-        # Reinstall roman due to https://github.com/sphinx-doc/sphinx/issues/5022
-        #
-
-        sudo apt-get remove python-requests python-openssl python-roman
-        sudo pip install requests pyopenssl roman sphinx==1.7.9
+        # Version of Sphinx in 16.04 is too old and dies on enum definitions
+        sudo pip install sphinx==1.7.9
     fi
 
 elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
diff --git a/src/scripts/ci/travis.yml b/src/scripts/ci/travis.yml
index 991fbbf6f..b2ac2539e 100644
--- a/src/scripts/ci/travis.yml
+++ b/src/scripts/ci/travis.yml
@@ -4,7 +4,7 @@ os:
   - linux
   - osx
 
-dist: trusty
+dist: xenial
 sudo: required
 
 compiler:
@@ -19,14 +19,14 @@ env:
 
   matrix:
     - BUILD_MODE="shared"
-#    - BUILD_MODE="sonar"
     - BUILD_MODE="fuzzers"
     - BUILD_MODE="coverage"
     - BUILD_MODE="cross-ppc32"
     - BUILD_MODE="cross-ppc64"
     - BUILD_MODE="cross-arm32"
     - BUILD_MODE="cross-arm64"
-    - BUILD_MODE="cross-win32"
+    - BUILD_MODE="cross-mips64"
+    - BUILD_MODE="cross-win64"
     - BUILD_MODE="valgrind"
     - BUILD_MODE="static"
     - BUILD_MODE="mini-shared"
@@ -41,15 +41,12 @@ matrix:
     - os: osx
       compiler: gcc
 
-    - os: osx
-      env: BUILD_MODE="sonar"
-
     # XCode 9.4 ICEs on 32-bit ARM builds
     - os: osx
       env: BUILD_MODE="cross-arm32"
 
     # Run static, docs, lint, bsi, nist, fuzzers, coverage, valgrind,
-    # sonar, sanitizer, minimized and non-ARM cross builds only on Linux/gcc.
+    # sanitizer, minimized and non-ARM cross builds only on Linux/gcc.
     # The sanitizer builds under Clang run the tests very slowly and
     # cause CI timeouts.
 
@@ -72,15 +69,13 @@ matrix:
     - compiler: clang
       env: BUILD_MODE="mini-shared"
     - compiler: clang
-      env: BUILD_MODE="cross-win32"
+      env: BUILD_MODE="cross-win64"
     - compiler: clang
       env: BUILD_MODE="cross-ppc32"
     - compiler: clang
       env: BUILD_MODE="cross-ppc64"
-
-    # Use clang coverage for Sonar
-    - compiler: gcc
-      env: BUILD_MODE="sonar"
+    - compiler: clang
+      env: BUILD_MODE="cross-mips64"
 
    # No ARM Clang compiler on Linux
     - os: linux
@@ -102,21 +97,15 @@ branches:
     - master
     - release-2
     - coverity_scan
+    - travis-xenial
 
 git:
   depth: 5
 
 cache:
   ccache: true
-  directories:
-    - $HOME/.sonar/cache
 
 addons:
-  #sonarcloud:
-  #  organization: "randombit-github"
-  #  token:
-  #    secure: "JYwDxEZyCJ1aKLjAYB9YlyyIRkjeaY48iYqcMhmidx6/ploWXMbUzVpv5plLFbMGgeMy9sqa2I7Pbo7u3OdcBbbsogGtk/jomnjiauuNS6KcCHEEcVjBILfadxGJPL9F8/0KbcS2vnPpz/jVlRp18Uemud0yGGXXcDaY471sA50="
-
   coverity_scan:
     project:
       name: "randombit/botan"
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
index 8c8e9912b..823d2255d 100755
--- a/src/scripts/ci_build.py
+++ b/src/scripts/ci_build.py
@@ -41,17 +41,18 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin, ccache, ro
     if is_cross_target:
         if target_os == 'osx':
             target_os = 'ios'
-        elif target == 'cross-win32':
+        elif target == 'cross-win64':
             target_os = 'mingw'
 
     make_prefix = []
     test_prefix = []
     test_cmd = [os.path.join(root_dir, 'botan-test')]
 
-    fast_tests = ['block', 'aead', 'hash', 'stream', 'mac', 'modes',
+    fast_tests = ['block', 'aead', 'hash', 'stream', 'mac', 'modes', 'kdf',
                   'hmac_drbg', 'hmac_drbg_unit',
                   'tls', 'ffi',
-                  'rsa_sign', 'rsa_verify', 'dh_kat', 'ecdsa_sign', 'curve25519_scalar',
+                  'rsa_sign', 'rsa_verify', 'dh_kat',
+                  'ecc_randomized', 'ecdh_kat', 'ecdsa_sign', 'curve25519_scalar',
                   'simd_32', 'os_utils', 'util', 'util_dates']
 
     install_prefix = os.path.join(tempfile.gettempdir(), 'botan-install')
@@ -94,7 +95,8 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin, ccache, ro
     if target == 'coverage':
         flags += ['--with-coverage-info', '--test-mode']
     if target == 'valgrind':
-        flags += ['--with-valgrind']
+        # valgrind in 16.04 has a bug with rdrand handling
+        flags += ['--with-valgrind', '--disable-rdrand']
         test_prefix = ['valgrind', '--error-exitcode=9', '-v', '--leak-check=full', '--show-reachable=yes']
         test_cmd += fast_tests
     if target == 'fuzzers':
@@ -144,9 +146,11 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin, ccache, ro
                 flags += ['--cpu=arm64', '--cc-abi-flags=-arch arm64 -stdlib=libc++']
             else:
                 raise Exception("Unknown cross target '%s' for iOS" % (target))
-        elif target == 'cross-win32':
-            cc_bin = 'i686-w64-mingw32-g++'
-            flags += ['--cpu=x86_32', '--cc-abi-flags=-static', '--ar-command=i686-w64-mingw32-ar']
+        elif target == 'cross-win64':
+            # MinGW in 16.04 is lacking std::mutex for unknown reason
+            cc_bin = 'x86_64-w64-mingw32-g++'
+            flags += ['--cpu=x86_64', '--cc-abi-flags=-static',
+                      '--ar-command=x86_64-w64-mingw32-ar', '--without-os-feature=threads']
             test_cmd = [os.path.join(root_dir, 'botan-test.exe')]
             # No runtime prefix required for Wine
         else:
@@ -168,7 +172,11 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin, ccache, ro
             elif target == 'cross-ppc64':
                 flags += ['--cpu=ppc64', '--with-endian=little']
                 cc_bin = 'powerpc64le-linux-gnu-g++'
-                test_prefix = ['qemu-ppc64le', '-L', '/usr/powerpc64le-linux-gnu/']
+                test_prefix = ['qemu-ppc64le', '-cpu', 'POWER8', '-L', '/usr/powerpc64le-linux-gnu/']
+            elif target == 'cross-mips64':
+                flags += ['--cpu=mips64', '--with-endian=big']
+                cc_bin = 'mips64-linux-gnuabi64-g++'
+                test_prefix = ['qemu-mips64', '-L', '/usr/mips64-linux-gnuabi64/']
             else:
                 raise Exception("Unknown cross target '%s' for Linux" % (target))
     else:
@@ -381,7 +389,7 @@ def main(args=None):
             raise Exception('No python interpreters found cannot lint')
 
         pylint_rc = '--rcfile=%s' % (os.path.join(root_dir, 'src/configs/pylint.rc'))
-        pylint_flags = [pylint_rc, '--reports=no', '--score=no']
+        pylint_flags = [pylint_rc, '--reports=no']
 
         # Some disabled rules specific to Python2
         # superfluous-parens: needed for Python3 compatible print statements
@@ -407,14 +415,13 @@ def main(args=None):
             'src/scripts/python_unittests.py',
             'src/scripts/python_unittests_unix.py']
 
-        for target in py_scripts:
-            target_path = os.path.join(root_dir, target)
+        full_paths = [os.path.join(root_dir, s) for s in py_scripts]
 
-            if use_python2:
-                cmds.append(['python2', '-m', 'pylint'] + pylint_flags + [py2_flags, target_path])
+        if use_python2:
+            cmds.append(['python2', '-m', 'pylint'] + pylint_flags + [py2_flags] + full_paths)
 
-            if use_python3 and options.use_pylint3:
-                cmds.append(['python3', '-m', 'pylint'] + pylint_flags + [py3_flags, target_path])
+        if use_python3 and options.use_pylint3:
+            cmds.append(['python3', '-m', 'pylint'] + pylint_flags + [py3_flags] + full_paths)
 
     else:
         config_flags, run_test_command, make_prefix = determine_flags(