Script the Windows CI build

9 files changed, 128 insertions, 70 deletions

diff --git a/configure.py b/configure.py
index 121bef0af..2b64798d3 100755
--- a/configure.py
+++ b/configure.py
@@ -1131,22 +1131,22 @@ class CompilerInfo(InfoObject): # pylint: disable=too-many-instance-attributes
 
         if options.with_coverage_info:
             if self.coverage_flags == '':
-                raise InternalError('No coverage handling for %s' % (self.basename))
+                raise UserError('No coverage handling for %s' % (self.basename))
             abi_link.append(self.coverage_flags)
 
         if options.with_sanitizers:
             if self.sanitizer_flags == '':
-                raise InternalError('No sanitizer handling for %s' % (self.basename))
+                raise UserError('No sanitizer handling for %s' % (self.basename))
             abi_link.append(self.sanitizer_flags)
 
         if options.with_openmp:
             if 'openmp' not in self.mach_abi_linking:
-                raise InternalError('No support for OpenMP for %s' % (self.basename))
+                raise UserError('No support for OpenMP for %s' % (self.basename))
             abi_link.append(self.mach_abi_linking['openmp'])
 
         if options.with_cilkplus:
             if 'cilkplus' not in self.mach_abi_linking:
-                raise InternalError('No support for Cilk Plus for %s' % (self.basename))
+                raise UserError('No support for Cilk Plus for %s' % (self.basename))
             abi_link.append(self.mach_abi_linking['cilkplus'])
 
         abi_flags = ' '.join(sorted(abi_link))
diff --git a/src/build-data/arch/x86_64.txt b/src/build-data/arch/x86_64.txt
index e77ae4d5e..d70eaae39 100644
--- a/src/build-data/arch/x86_64.txt
+++ b/src/build-data/arch/x86_64.txt
@@ -9,6 +9,7 @@ amd64
 x86-64
 em64t
 x64
+x86_amd64
 </aliases>
 
 <submodels>
diff --git a/src/build-data/makefile/nmake.in b/src/build-data/makefile/nmake.in
index 83d3e3f2b..f5ecde341 100644
--- a/src/build-data/makefile/nmake.in
+++ b/src/build-data/makefile/nmake.in
@@ -42,7 +42,7 @@ CLIOBJS       = %{cli_objs}
 TESTOBJS      = %{test_objs}
 
 # First make target. Will be used by default
-all: $(CLI) $(TEST)
+all: libs cli tests
 
 # Build Commands
 %{lib_build_cmds}
@@ -51,8 +51,6 @@ all: $(CLI) $(TEST)
 
 %{test_build_cmds}
 
-libs: $(LIBRARIES)
-
 # Link Commands
 $(CLI): $(LIBRARIES) $(CLIOBJS)
 	$(CLI_LINK_CMD) /OUT:$@ $(CLIOBJS) $(LIB_FILENAME) $(CLI_LINKS_TO)
@@ -62,6 +60,10 @@ $(TEST): $(LIBRARIES) $(TESTOBJS)
 	$(TEST_LINK_CMD) /OUT:$@ $(TESTOBJS) $(LIB_FILENAME) $(TEST_LINKS_TO)
 	$(TEST_POST_LINK_CMD)
 
+libs: $(LIBRARIES)
+cli: $(CLI)
+tests: $(TEST)
+
 !If "$(SO_OBJ_FLAGS)" == ""
 # static lib
 $(LIB_FILENAME): $(LIBOBJS)
diff --git a/src/lib/utils/http_util/info.txt b/src/lib/utils/http_util/info.txt
index 63e569f64..fe9fc3ea7 100644
--- a/src/lib/utils/http_util/info.txt
+++ b/src/lib/utils/http_util/info.txt
@@ -1,7 +1,3 @@
 <defines>
 HTTP_UTIL -> 20131128
 </defines>
-
-<libs>
-windows -> Ws2_32.lib
-</libs>
\ No newline at end of file
diff --git a/src/lib/utils/info.txt b/src/lib/utils/info.txt
index da39201d9..8bbe2925b 100644
--- a/src/lib/utils/info.txt
+++ b/src/lib/utils/info.txt
@@ -43,4 +43,5 @@ stl_util.h
 <libs>
 linux -> rt
 mingw -> ws2_32
+windows -> ws2_32.lib
 </libs>
diff --git a/src/scripts/ci/appveyor.yml b/src/scripts/ci/appveyor.yml
index b6df94a6f..09c38f42b 100644
--- a/src/scripts/ci/appveyor.yml
+++ b/src/scripts/ci/appveyor.yml
@@ -3,7 +3,9 @@
 #
 # Build jobs
 #  1. six basic builds: 32/64bit on MSVS2013/2015/2017
-#  2. add static lib, amalgamation and debug on MSVC2017
+#  2. MSVC2017 static lib with with amalgamation
+#  3. MSVC2017 with debug/sanitizers
+#  4. MSVC2015 for Windows RT (TODO)
 #
 # Note: Avoid the AppVeyor settings `platform` and `configuration` since excluding
 # from the build matrix is not supported (https://github.com/appveyor/ci/issues/386)
@@ -11,69 +13,57 @@
 clone_depth: 5
 
 environment:
+
   matrix:
     # 1
     - MSVS: 2013
       PLATFORM: x86
+      TARGET: shared
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
     - MSVS: 2013
       PLATFORM: x86_amd64
+      TARGET: shared
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+
     - MSVS: 2015
       PLATFORM: x86
+      TARGET: shared
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
     - MSVS: 2015
       PLATFORM: x86_amd64
+      TARGET: shared
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+
     - MSVS: 2017
       PLATFORM: x86
+      TARGET: shared
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
     - MSVS: 2017
       PLATFORM: x86_amd64
+      TARGET: shared
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
 
     # 2
     - MSVS: 2017
       PLATFORM: x86_amd64
-      CONFIG: --disable-shared
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-    - MSVS: 2017
-      PLATFORM: x86_amd64
-      CONFIG: --amalgamation
+      TARGET: static
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
 
     # 3
     - MSVS: 2017
       PLATFORM: x86_amd64
-      CONFIG: --with-debug
+      TARGET: sanitizer
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
 
 install:
-  # Check setup
-  - echo Current build setup MSVS="%MSVS%" PLATFORM="%PLATFORM%" CONFIG="%CONFIG%"
-
-  # Choose compiler
-  - if %MSVS% == 2013 (
-       call "%ProgramFiles(x86)%\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" %PLATFORM%
-    )
-  - if %MSVS% == 2015 (
-       call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %PLATFORM%
-    )
-  - if %MSVS% == 2017 (
-       call "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" %PLATFORM%
-    )
-  - cl # check compiler version
-  - appveyor DownloadFile http://download.qt.io/official_releases/jom/jom.zip -FileName jom.zip
-  - 7z e jom.zip
+  - call src\scripts\ci\setup_appveyor.bat
 
 build_script:
-  - python configure.py --cc=msvc --cpu=%PLATFORM% %CONFIG%
-  - jom -j3
-  - botan-test
-  - nmake install
+  - python src\scripts\ci_build.py --os=windows --cc=msvc --without-python3 --make-tool=jom --cpu=%PLATFORM% %TARGET%
 
 # whitelist branches to avoid testing feature branches twice (as branch and as pull request)
 branches:
   only:
     - master
     - release-2
+
diff --git a/src/scripts/ci/setup_appveyor.bat b/src/scripts/ci/setup_appveyor.bat
new file mode 100644
index 000000000..e3f081773
--- /dev/null
+++ b/src/scripts/ci/setup_appveyor.bat
@@ -0,0 +1,12 @@
+
+echo Current build setup MSVS="%MSVS%" PLATFORM="%PLATFORM%" TARGET="%TARGET%"
+
+if %MSVS% == 2013 call "%ProgramFiles(x86)%\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" %PLATFORM%
+if %MSVS% == 2015 call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %PLATFORM%
+if %MSVS% == 2017 call "%ProgramFiles(x86)%\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" %PLATFORM%
+
+rem check compiler version
+cl
+
+appveyor DownloadFile http://download.qt.io/official_releases/jom/jom.zip -FileName jom.zip
+7z e jom.zip
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
index 7267ecff7..1ac264d0e 100755
--- a/src/scripts/ci_build.py
+++ b/src/scripts/ci_build.py
@@ -11,6 +11,7 @@ import platform
 import subprocess
 import sys
 import time
+import tempfile
 import optparse # pylint: disable=deprecated-module
 
 def get_concurrency():
@@ -25,17 +26,17 @@ def get_concurrency():
     except ImportError:
         return def_concurrency
 
-def determine_flags(target, target_os, target_cc, cc_bin, use_ccache, root_dir):
-    # pylint: disable=too-many-branches,too-many-statements,too-many-arguments
+def determine_flags(target, target_os, target_cpu, target_cc, cc_bin, ccache, root_dir):
+    # pylint: disable=too-many-branches,too-many-statements,too-many-arguments,too-many-locals
 
     """
     Return the configure.py flags as well as make/test running prefixes
     """
     is_cross_target = target.startswith('cross-')
 
-    if target_os not in ['linux', 'osx']:
+    if target_os not in ['linux', 'osx', 'windows']:
         print('Error unknown OS %s' % (target_os))
-        return 1
+        return (None, None, None)
 
     if is_cross_target:
         if target_os == 'osx':
@@ -53,13 +54,19 @@ def determine_flags(target, target_os, target_cc, cc_bin, use_ccache, root_dir):
                   'rsa_sign', 'rsa_verify', 'dh_kat', 'ecdsa_sign', 'curve25519_scalar',
                   'simd_32', 'os_utils', 'util', 'util_dates']
 
-    flags = ['--prefix=/tmp/botan-install', '--cc=%s' % (target_cc), '--os=%s' % (target_os)]
+    install_prefix = os.path.join(tempfile.gettempdir(), 'botan-install')
+    flags = ['--prefix=%s' % (install_prefix),
+             '--cc=%s' % (target_cc),
+             '--os=%s' % (target_os)]
+
+    if target_cpu != None:
+        flags += ['--cpu=%s' % (target_cpu)]
 
     if target in ['static', 'mini-static', 'fuzzers'] or target_os in ['ios', 'mingw']:
         flags += ['--disable-shared']
 
     if target in ['mini-static', 'mini-shared']:
-        flags += ['--minimized-build', '--enable-modules=dev_random,system_rng,sha2_32,sha2_64,aes']
+        flags += ['--minimized-build', '--enable-modules=system_rng,sha2_32,sha2_64,aes']
 
     if target == 'static':
         # Arbitrarily test amalgamation on static lib builds
@@ -86,7 +93,12 @@ def determine_flags(target, target_os, target_cc, cc_bin, use_ccache, root_dir):
     if target in ['fuzzers', 'coverage']:
         flags += ['--build-fuzzers=test']
     if target in ['fuzzers', 'sanitizer']:
-        flags += ['--with-sanitizers']
+
+        # On VC iterator debugging comes from generic debug mode
+        if target_cc == 'msvc':
+            flags += ['--with-debug-info']
+        else:
+            flags += ['--with-sanitizers']
     if target in ['valgrind', 'sanitizer', 'fuzzers']:
         flags += ['--disable-modules=locking_allocator']
 
@@ -97,6 +109,9 @@ def determine_flags(target, target_os, target_cc, cc_bin, use_ccache, root_dir):
             flags += ['--with-openmp']
 
     if target == 'sonar':
+        if os != 'linux':
+            raise Exception('Only Linux supported in Sonar target currently')
+
         make_prefix = [os.path.join(root_dir, 'build-wrapper-linux-x86/build-wrapper-linux-x86-64'),
                        '--out-dir', 'bw-outputs']
 
@@ -144,7 +159,9 @@ def determine_flags(target, target_os, target_cc, cc_bin, use_ccache, root_dir):
                 raise Exception("Unknown cross target '%s' for Linux" % (target))
     else:
         # Flags specific to native targets
-        flags += ['--with-bzip2', '--with-sqlite', '--with-zlib']
+
+        if target_os in ['osx', 'linux']:
+            flags += ['--with-bzip2', '--with-sqlite', '--with-zlib']
 
         if target_os == 'osx':
             # Test Boost on OS X
@@ -166,7 +183,12 @@ def determine_flags(target, target_os, target_cc, cc_bin, use_ccache, root_dir):
             if os.access(softhsm_lib, os.R_OK):
                 test_cmd += ['--pkcs11-lib=%s' % (softhsm_lib)]
 
-    flags += ['--cc-bin=%s%s' % ('ccache ' if use_ccache else '', cc_bin)]
+    if ccache is None:
+        flags += ['--cc-bin=%s' % (cc_bin)]
+    elif ccache == 'clcache':
+        flags += ['--cc-bin=%s' % (ccache)]
+    else:
+        flags += ['--cc-bin=%s %s' % (ccache, cc_bin)]
 
     if test_cmd is None:
         run_test_command = None
@@ -213,6 +235,19 @@ def parse_args(args):
     parser.add_option('--root-dir', metavar='D', default='.',
                       help='Set directory to execute from (default %default)')
 
+    parser.add_option('--make-tool', metavar='TOOL', default='make',
+                      help='Specify tool to run to build source (default %default)')
+
+    parser.add_option('--cpu', default=None,
+                      help='Specify a target CPU platform')
+
+    parser.add_option('--with-debug', action='store_true', default=False,
+                      help='Include debug information')
+    parser.add_option('--amalgamation', action='store_true', default=False,
+                      help='Build via amalgamation')
+    parser.add_option('--disable-shared', action='store_true', default=False,
+                      help='Disable building shared libraries')
+
     parser.add_option('--branch', metavar='B', default=None,
                       help='Specify branch being built')
 
@@ -224,10 +259,8 @@ def parse_args(args):
     parser.add_option('--build-jobs', metavar='J', default=get_concurrency(),
                       help='Set number of jobs to run in parallel (default %default)')
 
-    parser.add_option('--with-ccache', dest='use_ccache', action='store_true', default=None,
-                      help='Enable using ccache')
-    parser.add_option('--without-ccache', dest='use_ccache', action='store_false',
-                      help='Disable using ccache')
+    parser.add_option('--compiler-cache', default=None,
+                      help='Set a compiler cache to use (ccache, clcache)')
 
     parser.add_option('--with-python3', dest='use_python3', action='store_true', default=None,
                       help='Enable using python3')
@@ -257,8 +290,7 @@ def main(args=None):
         print('Usage: %s [options] target' % (args[0]))
         return 1
 
-    if options.use_ccache is None:
-        options.use_ccache = have_prog('ccache')
+    py_interp = 'python'
 
     use_python2 = have_prog('python2')
 
@@ -266,16 +298,29 @@ def main(args=None):
         use_python3 = have_prog('python3')
     else:
         use_python3 = options.use_python3
+        if use_python3:
+            py_interp = 'python3'
 
     if options.cc_bin is None:
         if options.cc == 'gcc':
             options.cc_bin = 'g++'
         elif options.cc == 'clang':
             options.cc_bin = 'clang++'
+        elif options.cc == 'msvc':
+            options.cc_bin = 'cl'
         else:
             print('Error unknown compiler %s' % (options.cc))
             return 1
 
+    if options.compiler_cache is None and options.cc != 'msvc':
+        # Autodetect ccache
+        options.compiler_cache = have_prog('ccache')
+
+    if options.compiler_cache == 'clcache' and target in ['sanitizer']:
+        # clcache doesn't support /Zi so using it just adds overhead with
+        # no benefit
+        options.compiler_cache = None
+
     target = args[1]
 
     if target == 'sonar' and os.getenv('SONAR_TOKEN') is None:
@@ -317,38 +362,48 @@ def main(args=None):
 
     else:
         config_flags, run_test_command, make_prefix = determine_flags(
-            target, options.os, options.cc, options.cc_bin, options.use_ccache, root_dir)
+            target, options.os, options.cpu, options.cc, options.cc_bin, options.compiler_cache, root_dir)
+
+        cmds.append([py_interp, os.path.join(root_dir, 'configure.py')] + config_flags)
 
-        cmds.append([os.path.join(root_dir, 'configure.py')] + config_flags)
+        make_cmd = [options.make_tool]
+        if root_dir != '.':
+            make_cmd += ['-C', root_dir]
+        if options.build_jobs > 1:
+            make_cmd += ['-j%d' % (options.build_jobs)]
 
         if target == 'docs':
-            cmds.append(['make', '-C', root_dir, 'docs'])
+            cmds.append(make_cmd + ['docs'])
         else:
-            if options.use_ccache:
+            if options.compiler_cache == 'ccache':
                 cmds.append(['ccache', '--show-stats'])
+            elif options.compiler_cache == 'clcache':
+                cmds.append(['clcache', '-s'])
 
             make_targets = ['libs', 'cli', 'tests']
             if target in ['coverage', 'fuzzers']:
                 make_targets += ['fuzzers', 'fuzzer_corpus_zip']
 
-            cmds.append(make_prefix +
-                        ['make', '-j', str(options.build_jobs), '-C', root_dir] +
-                        make_targets)
+            cmds.append(make_prefix + make_cmd + make_targets)
 
-            if options.use_ccache:
+            if options.compiler_cache == 'ccache':
                 cmds.append(['ccache', '--show-stats'])
+            elif options.compiler_cache == 'clcache':
+                cmds.append(['clcache', '-s'])
 
         if run_test_command != None:
             cmds.append(run_test_command)
 
         if target in ['coverage', 'fuzzers']:
-            cmds.append([os.path.join(root_dir, 'src/scripts/test_fuzzers.py'),
+            cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/test_fuzzers.py'),
                          os.path.join(root_dir, 'fuzzer_corpus'),
                          os.path.join(root_dir, 'build/fuzzer')])
 
-        if target in ['static', 'shared']:
-            cmds.append([os.path.join(root_dir, 'src/scripts/cli_tests.py'),
-                         os.path.join(root_dir, 'botan')])
+        if target in ['static', 'shared'] and options.os != 'windows':
+            botan_exe = os.path.join(root_dir, 'botan-cli.exe' if options.os == 'windows' else 'botan')
+            cmds.append([py_interp,
+                         os.path.join(root_dir, 'src/scripts/cli_tests.py'),
+                         botan_exe])
 
         botan_py = os.path.join(root_dir, 'src/python/botan2.py')
 
@@ -360,7 +415,7 @@ def main(args=None):
                 cmds.append(['python3', botan_py])
 
         if target != 'docs':
-            cmds.append(['make', '-C', root_dir, 'install'])
+            cmds.append(make_cmd + ['install'])
 
         if target in ['coverage']:
 
diff --git a/src/scripts/cli_tests.py b/src/scripts/cli_tests.py
index ba0d9affb..5af8cc07b 100755
--- a/src/scripts/cli_tests.py
+++ b/src/scripts/cli_tests.py
@@ -7,6 +7,7 @@ import argparse
 import re
 import subprocess
 import sys
+import os.path
 
 import vecparser
 
@@ -127,10 +128,10 @@ if __name__ == '__main__':
 
     cli_binary = args.cli_binary
 
-    vecfile_cfb = vecparser.VecDocument("src/tests/data/modes/cfb.vec")
-    vecfile_gcm = vecparser.VecDocument("src/tests/data/aead/gcm.vec")
-    vecfile_ocb = vecparser.VecDocument("src/tests/data/aead/ocb.vec")
-    vecfile_xts = vecparser.VecDocument("src/tests/data/modes/xts.vec")
+    vecfile_cfb = vecparser.VecDocument(os.path.join('src', 'tests', 'data', 'modes', 'cfb.vec'))
+    vecfile_gcm = vecparser.VecDocument(os.path.join('src', 'tests', 'data', 'aead', 'gcm.vec'))
+    vecfile_ocb = vecparser.VecDocument(os.path.join('src', 'tests', 'data', 'aead', 'ocb.vec'))
+    vecfile_xts = vecparser.VecDocument(os.path.join('src', 'tests', 'data', 'modes', 'xts.vec'))
     #data = vecfile.get_data()
     #for algo in data:
     #    print(algo)