Detect the compiler version using the preprocessor instead of command line

Fixes #1314

5 files changed, 99 insertions, 234 deletions

diff --git a/configure.py b/configure.py
index 82f6faabf..6bd27dbb0 100755
--- a/configure.py
+++ b/configure.py
@@ -1020,6 +1020,7 @@ class CompilerInfo(InfoObject): # pylint: disable=too-many-instance-attributes
                 'add_lib_dir_option': '-L',
                 'add_lib_option': '-l',
                 'add_framework_option': '-framework ',
+                'preproc_flags': '-E',
                 'compile_flags': '-c',
                 'debug_info_flags': '-g',
                 'optimization_flags': '',
@@ -1059,6 +1060,7 @@ class CompilerInfo(InfoObject): # pylint: disable=too-many-instance-attributes
         self.optimization_flags = lex.optimization_flags
         self.output_to_exe = lex.output_to_exe
         self.output_to_object = lex.output_to_object
+        self.preproc_flags = lex.preproc_flags
         self.sanitizer_flags = lex.sanitizer_flags
         self.shared_flags = lex.shared_flags
         self.size_optimization_flags = lex.size_optimization_flags
@@ -2485,82 +2487,6 @@ class AmalgamationGenerator(object):
         return (sorted(amalgamation_sources), sorted(amalgamation_headers))
 
 
-class CompilerDetector(object):
-    _msvc_version_source = os.path.join(
-        os.path.dirname(os.path.realpath(__file__)), "src", "build-data", "compiler_detection", "msvc_version.c")
-    _version_flags = {
-        'msvc': ['/nologo', '/EP', _msvc_version_source],
-        'gcc': ['-v'],
-        'clang': ['-v'],
-    }
-    _version_patterns = {
-        'msvc': r'^([0-9]{2})([0-9]{2})',
-        'gcc': r'gcc version ([0-9]+)\.([0-9]+)\.[0-9]+',
-        'clang': r'clang version ([0-9]+)\.([0-9])[ \.]',
-    }
-
-    def __init__(self, cc_name, cc_bin, os_name):
-        self._cc_name = cc_name
-        self._cc_bin = cc_bin
-        self._os_name = os_name
-
-    def get_version(self):
-        try:
-            cmd = self._cc_bin.split(' ') + CompilerDetector._version_flags[self._cc_name]
-        except KeyError:
-            logging.info("No compiler version detection available for %s" % (self._cc_name))
-            return None
-
-        try:
-            stdout, stderr = subprocess.Popen(
-                cmd,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                universal_newlines=True).communicate()
-            cc_output = stdout + "\n" + stderr
-        except OSError as e:
-            logging.warning('Could not execute %s for version check: %s' % (cmd, e))
-            return None
-
-        return self.version_from_compiler_output(cc_output)
-
-    def version_from_compiler_output(self, cc_output):
-        cc_version = None
-
-        match = re.search(CompilerDetector._version_patterns[self._cc_name], cc_output, flags=re.MULTILINE)
-        if match:
-            major = int(match.group(1))
-            minor = int(match.group(2))
-            cc_version = "%d.%d" % (major, minor)
-
-        if cc_version is None and self._cc_name == 'clang' and self._os_name in ['darwin', 'ios']:
-            # For appleclang version >= X, return minimal clang version Y
-            appleclang_to_clang_version = {
-                703: '3.8',
-                800: '3.9',
-                # 802 has no support for clang 4.0 flags -Og and -MJ, 900 has.
-                802: '3.9',
-                900: '4.0',
-            }
-
-            # All appleclang versions without "based on LLVM" note are at least clang 3.7
-            # https://en.wikipedia.org/wiki/Xcode#Latest_versions
-            cc_version = '3.7'
-            match = re.search(r'Apple LLVM version [0-9\.]+ \(clang-([0-9]+)\.', cc_output)
-            if match:
-                user_appleclang = int(match.group(1))
-                for appleclang, clang in sorted(appleclang_to_clang_version.items()):
-                    if user_appleclang >= appleclang:
-                        cc_version = clang
-                logging.info('Mapping Apple Clang version %s to LLVM version %s' % (user_appleclang, cc_version))
-
-        if not cc_version:
-            logging.warning("Tried to get %s version, but output '%s' does not match expected version format" % (
-                self._cc_name, cc_output))
-
-        return cc_version
-
-
 def have_program(program):
     """
     Test for the existence of a program
@@ -2682,7 +2608,7 @@ def set_defaults_for_unset_options(options, info_arch, info_cc): # pylint: disab
         logging.info('Guessing target OS is %s (use --os to set)' % (options.os))
 
     def deduce_compiler_type_from_cc_bin(cc_bin):
-        if cc_bin.find('clang') != -1:
+        if cc_bin.find('clang') != -1 or cc_bin in ['emcc', 'em++']:
             return 'clang'
         if cc_bin.find('-g++') != -1:
             return 'gcc'
@@ -2851,21 +2777,43 @@ def prepare_configure_build(info_modules, source_paths, options,
     return using_mods, build_config, template_vars
 
 
-def calculate_cc_min_version(options, cc, osinfo):
-    if options.cc_min_version:
-        return options.cc_min_version
-    else:
-        cc_version = CompilerDetector(
-            cc.basename,
-            options.compiler_binary or cc.binary_name,
-            osinfo.basename
-        ).get_version()
-        if cc_version:
-            logging.info('Auto-detected compiler version %s' % (cc_version))
-            return cc_version
-        else:
-            logging.warning("Detecting compiler version failed. Use --cc-min-version to set")
-            return "0.0"
+def calculate_cc_min_version(options, ccinfo, source_paths):
+    version_patterns = {
+        'msvc': r'^MSVC ([0-9]{2})([0-9]{2})$',
+        'gcc': r'^GCC ([0-9]+) ([0-9]+)$',
+        'clang': r'^CLANG ([0-9]+) ([0-9])$',
+    }
+
+    if ccinfo.basename not in version_patterns:
+        logging.info("No compiler version detection available for %s" % (ccinfo.basename))
+        return "0.0"
+
+    detect_version_source = os.path.join(source_paths.build_data_dir, "detect_version.cpp")
+
+    cc_bin = options.compiler_binary or ccinfo.binary_name
+
+    cmd = cc_bin.split(' ') + [ccinfo.preproc_flags, detect_version_source]
+
+    try:
+        stdout, stderr = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True).communicate()
+        cc_output = stdout + "\n" + stderr
+    except OSError as e:
+        logging.warning('Could not execute %s for version check: %s' % (cmd, e))
+        return "0.0"
+
+    match = re.search(version_patterns[ccinfo.basename], cc_output, flags=re.MULTILINE)
+    if match is None:
+        logging.warning("Tried to get %s version, but output '%s' does not match expected version format" % (
+            ccinfo.basename, cc_output))
+        return "0.0"
+
+    cc_version = "%d.%d" % (int(match.group(1)), int(match.group(2)))
+    logging.info('Auto-detected compiler version %s' % (cc_version))
+    return cc_version
 
 def main_action_configure_build(info_modules, source_paths, options,
                                 cc, cc_min_version, arch, osinfo, module_policy):
@@ -3007,7 +2955,7 @@ def main(argv):
     arch = info_arch[options.arch]
     osinfo = info_os[options.os]
     module_policy = info_module_policies[options.module_policy] if options.module_policy else None
-    cc_min_version = calculate_cc_min_version(options, cc, osinfo)
+    cc_min_version = options.cc_min_version or calculate_cc_min_version(options, cc, source_paths)
 
     logging.info('Target is %s:%s-%s-%s-%s' % (
         options.compiler, cc_min_version, options.os, options.arch, options.cpu))
diff --git a/src/build-data/cc/msvc.txt b/src/build-data/cc/msvc.txt
index b816d33f8..66949970d 100644
--- a/src/build-data/cc/msvc.txt
+++ b/src/build-data/cc/msvc.txt
@@ -21,6 +21,8 @@ size_optimization_flags "/O1 /Os"
 # for using a PDB file:
 debug_info_flags "/Zi /FS"
 
+preproc_flags "/nologo /EP"
+
 lang_flags "/EHs /GR"
 warning_flags "/W4 /wd4250 /wd4251 /wd4275"
 
diff --git a/src/build-data/compiler_detection/msvc_version.c b/src/build-data/compiler_detection/msvc_version.c
deleted file mode 100644
index 054f69d28..000000000
--- a/src/build-data/compiler_detection/msvc_version.c
+++ /dev/null
@@ -1,8 +0,0 @@
-// _MSC_VER Defined as an integer literal that encodes the major and minor
-// number elements of the compiler's version number. The major number is
-// the first element of the period-delimited version number and the minor
-// number is the second element. For example, if the version number of the
-// Visual C++ compiler is 17.00.51106.1, the _MSC_VER macro evaluates to 1700.
-// https://msdn.microsoft.com/en-us/library/b0084kay.aspx
-
-_MSC_VER
diff --git a/src/build-data/detect_version.cpp b/src/build-data/detect_version.cpp
new file mode 100644
index 000000000..55e5db98d
--- /dev/null
+++ b/src/build-data/detect_version.cpp
@@ -0,0 +1,56 @@
+/*
+* This file is preprocessed to produce output that is examined by
+* configure.py to determine the compilers version number.
+*/
+
+#if defined(_MSC_VER)
+
+// _MSC_VER Defined as an integer literal that encodes the major and minor
+// number elements of the compiler's version number. The major number is
+// the first element of the period-delimited version number and the minor
+// number is the second element. For example, if the version number of the
+// Visual C++ compiler is 17.00.51106.1, the _MSC_VER macro evaluates to 1700.
+// https://msdn.microsoft.com/en-us/library/b0084kay.aspx
+
+MSVC _MSC_VER
+
+#elif defined(__clang__)
+
+#if defined(__apple_build_version__)
+
+/*
+Map Apple XCode versions back to standard Clang
+
+This is not a complete map, since we don't support any versions of
+Clang before 3.5 in any case, and it arbitrarily maps any version with
+XCode >= 9 to Clang 4.0. This is fine because we don't currently need
+any features not available in Clang 4.0
+*/
+
+#if __clang_major__ >= 9
+CLANG 4 0
+#elif __clang__major__ == 8
+CLANG 3 9
+#elif __clang__major__ == 7 && __clang__minor__ == 3
+CLANG 3 8
+#elif __clang__major__ == 7
+CLANG 3 7
+#else
+CLANG 3 5
+#endif
+
+#else
+
+CLANG __clang_major__ __clang_minor__
+
+#endif
+
+#elif defined(__GNUG__)
+
+GCC __GNUC__ __GNUC_MINOR__
+
+#else
+
+UNKNOWN 0 0
+
+#endif
diff --git a/src/scripts/python_unittests.py b/src/scripts/python_unittests.py
index 337f5369e..a6a22f3f0 100755
--- a/src/scripts/python_unittests.py
+++ b/src/scripts/python_unittests.py
@@ -15,7 +15,6 @@ import unittest
 
 sys.path.append("../..") # Botan repo root
 from configure import AmalgamationHelper # pylint: disable=wrong-import-position
-from configure import CompilerDetector # pylint: disable=wrong-import-position
 from configure import ModulesChooser # pylint: disable=wrong-import-position
 
 class AmalgamationHelperTests(unittest.TestCase):
@@ -62,138 +61,6 @@ class AmalgamationHelperTests(unittest.TestCase):
         self.assertEqual(AmalgamationHelper.is_any_include("#include <botan/oids.h> // comment"),
                          "botan/oids.h")
 
-class CompilerDetection(unittest.TestCase):
-
-    def test_gcc_invalid(self):
-        detector = CompilerDetector("gcc", "g++", "linux")
-
-        compiler_out = ""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), None)
-        compiler_out = "gcc version 20170406 (Ubuntu 6.3.0-12ubuntu2)"
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), None)
-
-    def test_clang_invalid(self):
-        detector = CompilerDetector("clang", "clang++", "linux")
-
-        compiler_out = ""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), None)
-        compiler_out = "clang version 20170406."
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), None)
-
-    def test_msvc_invalid(self):
-        detector = CompilerDetector("msvc", "cl.exe", "windows")
-
-        compiler_out = ""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), None)
-        compiler_out = "Microsoft (R) C/C++ Optimizing Compiler Version 19.00.24213.1 for x86"
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), None)
-
-    def test_gcc_version(self):
-        detector = CompilerDetector("gcc", "g++", "linux")
-        compiler_out = """Using built-in specs.
-COLLECT_GCC=/usr/bin/gcc
-COLLECT_LTO_WRAPPER=/usr/lib/gcc/x86_64-linux-gnu/6/lto-wrapper
-Target: x86_64-linux-gnu
-Configured with: ../src/configure -v --with-pkgversion='Ubuntu 6.3.0-12ubuntu2' --with-bugurl=file:///usr/share/doc/gcc-6/README.Bugs --enable-languages=c,ada,c++,java,go,d,fortran,objc,obj-c++ --prefix=/usr --program-suffix=-6 --program-prefix=x86_64-linux-gnu- --enable-shared --enable-linker-build-id --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix --libdir=/usr/lib --enable-nls --with-sysroot=/ --enable-clocale=gnu --enable-libstdcxx-debug --enable-libstdcxx-time=yes --with-default-libstdcxx-abi=new --enable-gnu-unique-object --disable-vtable-verify --enable-libmpx --enable-plugin --enable-default-pie --with-system-zlib --disable-browser-plugin --enable-java-awt=gtk --enable-gtk-cairo --with-java-home=/usr/lib/jvm/java-1.5.0-gcj-6-amd64/jre --enable-java-home --with-jvm-root-dir=/usr/lib/jvm/java-1.5.0-gcj-6-amd64 --with-jvm-jar-dir=/usr/lib/jvm-exports/java-1.5.0-gcj-6-amd64 --with-arch-directory=amd64 --with-ecj-jar=/usr/share/java/eclipse-ecj.jar --with-target-system-zlib --enable-objc-gc=auto --enable-multiarch --disable-werror --with-arch-32=i686 --with-abi=m64 --with-multilib-list=m32,m64,mx32 --enable-multilib --with-tune=generic --enable-checking=release --build=x86_64-linux-gnu --host=x86_64-linux-gnu --target=x86_64-linux-gnu
-Thread model: posix
-gcc version 6.3.0 20170406 (Ubuntu 6.3.0-12ubuntu2)"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "6.3")
-
-    def test_clang_version(self):
-        detector = CompilerDetector("clang", "clang++", "linux")
-        compiler_out = """clang version 4.0.0-1ubuntu1 (tags/RELEASE_400/rc1)
-Target: x86_64-pc-linux-gnu
-Thread model: posix
-InstalledDir: /usr/bin
-Found candidate GCC installation: /usr/bin/../lib/gcc/i686-linux-gnu/6
-Found candidate GCC installation: /usr/bin/../lib/gcc/i686-linux-gnu/6.3.0
-Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/4.9
-Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/4.9.4
-Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/5
-Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.1
-Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/6
-Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/6.3.0
-Found candidate GCC installation: /usr/lib/gcc/i686-linux-gnu/6
-Found candidate GCC installation: /usr/lib/gcc/i686-linux-gnu/6.3.0
-Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/4.9
-Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/4.9.4
-Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/5
-Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/5.4.1
-Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/6
-Found candidate GCC installation: /usr/lib/gcc/x86_64-linux-gnu/6.3.0
-Selected GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/6.3.0
-Candidate multilib: .;@m64
-Selected multilib: .;@m64"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "4.0")
-
-    def test_clang_version_two_digit(self):
-        # Output from Crystax NDK. Note: there is a trailing whitespace behind the version
-        # number for whatever reason. But let's keep the original output unchanged.
-        # pylint: disable=trailing-whitespace
-        detector = CompilerDetector("clang", "clang++", "android")
-        compiler_out = """clang version 3.7 
-Target: i686-none-linux-android
-Thread model: posix
-Found candidate GCC installation: /foo/crystax-toolchains/x86/bin/../lib/gcc/i686-linux-android/4.9
-Selected GCC installation: /foo/crystax-toolchains/x86/bin/../lib/gcc/i686-linux-android/4.9
-Candidate multilib: .;@m32
-Selected multilib: .;@m32"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "3.7")
-
-    def test_clang_version_appleclang(self):
-        detector = CompilerDetector("clang", "clang++", "darwin")
-        compiler_out = """Apple LLVM version 8.1.0 (clang-802.0.42)
-Target: x86_64-apple-darwin16.7.0
-Thread model: posix
-InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "3.9")
-
-        compiler_out = """Apple LLVM version 9.0.0 (clang-900.0.35)
-Target: x86_64-apple-darwin16.7.0
-Thread model: posix
-InstalledDir: /Applications/Xcode-beta.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "4.0")
-
-    def test_clang_version_appleclang_intermediate(self):
-        # fake versions in between the knwon ones
-        # clang-700.0.0 is lower than all known versions
-        # clang-802.1.0 is a minor update of known clang-802
-        # clang-1111.9.99 is a random future value
-        detector = CompilerDetector("clang", "clang++", "darwin")
-
-        compiler_out = """Apple LLVM version 7.0.0 (clang-700.0.0)
-Target: x86_64-apple-darwin16.7.0
-Thread model: posix
-InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "3.7")
-
-        compiler_out = """Apple LLVM version 8.1.1 (clang-802.1.0)
-Target: x86_64-apple-darwin16.7.0
-Thread model: posix
-InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "3.9")
-
-        compiler_out = """Apple LLVM version 11.11.0 (clang-1111.9.99)
-Target: x86_64-apple-darwin16.7.0
-Thread model: posix
-InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "4.0")
-
-    def test_msvc_version(self):
-        detector = CompilerDetector("msvc", "cl.exe", "windows")
-        compiler_out = """msvc_version.c
-
-1900
-"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "19.0")
-
-        compiler_out = """msvc_version.c
-
-1910
-"""
-        self.assertEqual(detector.version_from_compiler_output(compiler_out), "19.10")
-
-
 class ModulesChooserResolveDependencies(unittest.TestCase):
     def test_base(self):
         available_modules = set(["A", "B"])