Split amalagamation into ISA specific objects

6 files changed, 83 insertions, 52 deletions

diff --git a/configure.py b/configure.py
index aa863f639..5c2869d01 100755
--- a/configure.py
+++ b/configure.py
@@ -135,6 +135,7 @@ class BuildConfigurationInformation(object):
         self.botan_include_dir = os.path.join(self.include_dir, 'botan')
         self.internal_include_dir = os.path.join(self.botan_include_dir, 'internal')
 
+        self.modules = modules
         self.sources = sorted(flatten([mod.sources() for mod in modules]))
         self.internal_headers = sorted(flatten([m.internal_headers() for m in modules]))
 
@@ -1139,14 +1140,11 @@ def create_template_vars(build_config, options, modules, cc, arch, osinfo):
 
                 return get_isa_specific_flags(cc, isas)
 
-        return ''
+        if src.startswith('botan_all_'):
+            isa =  src.replace('botan_all_','').replace('.cpp', '').split('_')
+            return get_isa_specific_flags(cc, isa)
 
-    def all_isa_specific_flags():
-        all_isas = set()
-        for mod in modules:
-            for isa in mod.need_isa:
-                all_isas.add(isa)
-        return get_isa_specific_flags(cc, all_isas)
+        return ''
 
     """
     Form snippets of makefile for building each source file
@@ -1257,7 +1255,7 @@ def create_template_vars(build_config, options, modules, cc, arch, osinfo):
 
         'cc': (options.compiler_binary or cc.binary_name) + cc.mach_abi_link_flags(options),
 
-        'lib_opt': cc.opt_flags('lib', options) + (all_isa_specific_flags() if options.via_amalgamation else ''),
+        'lib_opt': cc.opt_flags('lib', options),
         'app_opt': cc.opt_flags('app', options),
         'lang_flags': cc.lang_flags,
         'warn_flags': warning_flags(cc.warning_flags,
@@ -1653,7 +1651,7 @@ def setup_build(build_config, options, template_vars):
 """
 Generate the amalgamation
 """
-def generate_amalgamation(build_config):
+def generate_amalgamation(build_config, options):
     def strip_header_goop(header_name, contents):
         header_guard = re.compile('^#define BOTAN_.*_H__$')
 
@@ -1693,11 +1691,11 @@ def generate_amalgamation(build_config):
                 self.file_contents[os.path.basename(f)] = contents
 
             self.contents = ''
-            for name in self.file_contents:
+            for name in sorted(self.file_contents):
                 self.contents += ''.join(list(self.header_contents(name)))
 
             self.header_includes = ''
-            for std_header in self.all_std_includes:
+            for std_header in sorted(self.all_std_includes):
                 self.header_includes += '#include <%s>\n' % (std_header)
             self.header_includes += '\n'
 
@@ -1728,17 +1726,18 @@ def generate_amalgamation(build_config):
     amalg_basename = 'botan_all'
 
     header_name = '%s.h' % (amalg_basename)
-    source_name = '%s.cpp' % (amalg_basename)
+    header_int_name = '%s_internal.h' % (amalg_basename)
 
-    logging.info('Writing amalgamation to %s and %s' % (header_name, source_name))
+    logging.info('Writing amalgamation header to %s' % (header_name))
 
     botan_h = open(header_name, 'w')
+    botan_int_h = open(header_int_name, 'w')
 
     pub_header_amalag = Amalgamation_Generator(build_config.public_headers)
 
     amalg_header = """/*
 * Botan %s Amalgamation
-* (C) 1999-2013,2014 Jack Lloyd and others
+* (C) 1999-2013,2014,2015 Jack Lloyd and others
 *
 * Botan is released under the Simplified BSD License (see license.txt)
 */
@@ -1760,36 +1759,63 @@ def generate_amalgamation(build_config):
         [s for s in build_config.internal_headers
          if s.find('asm_macr_') == -1])
 
+    botan_int_h.write("""
+#ifndef BOTAN_AMALGAMATION_INTERNAL_H__
+#define BOTAN_AMALGAMATION_INTERNAL_H__
+
+""")
+    botan_int_h.write(internal_headers.header_includes)
+    botan_int_h.write(internal_headers.contents)
+    botan_int_h.write("\n#endif\n")
+
     headers_written = pub_header_amalag.all_std_includes.union(internal_headers.all_std_includes)
 
-    botan_cpp = open(source_name, 'w')
+    src_files = []
+
+    def open_amalg_file(tgt):
+        fsname = '%s%s.cpp' % (amalg_basename, '_' + tgt if tgt else '' )
+        src_files.append(fsname)
+        logging.info('Writing amalgamation source to %s' % (fsname))
+        f = open(fsname, 'w')
+        f.write(amalg_header)
 
-    botan_cpp.write(amalg_header)
+        f.write('\n#include "%s"\n' % (header_name))
+        f.write('#include "%s"\n\n' % (header_int_name))
 
-    botan_cpp.write('\n#include "%s"\n' % (header_name))
+        return f
 
-    botan_cpp.write(internal_headers.header_includes)
-    botan_cpp.write(internal_headers.contents)
+    botan_amalgs = {}
+    botan_amalgs[''] = open_amalg_file('')
 
-    for src in build_config.sources:
-        if src.endswith('.S'):
-            continue
+    for mod in build_config.modules:
+        tgt = ''
 
-        contents = open(src).readlines()
-        for line in contents:
-            if botan_include.search(line):
-                continue
+        if mod.need_isa != []:
+            tgt = '_'.join(sorted(mod.need_isa))
+            if tgt == 'sse2' and options.arch == 'x86_64':
+                tgt = '' # always available
 
-            match = any_include.search(line)
-            if match:
-                header = match.group(1)
-                if header in headers_written:
+        if tgt not in botan_amalgs:
+            botan_amalgs[tgt] = open_amalg_file(tgt)
+        for src in sorted(mod.source):
+            contents = open(src).readlines()
+            for line in contents:
+                if botan_include.search(line):
                     continue
 
-                botan_cpp.write(line)
-                headers_written.add(header)
-            else:
-                botan_cpp.write(line)
+                match = any_include.search(line)
+                if match:
+                    header = match.group(1)
+                    if header in headers_written:
+                        continue
+
+                    botan_amalgs[tgt].write(line)
+                    headers_written.add(header)
+                else:
+                    botan_amalgs[tgt].write(line)
+
+    if options.via_amalgamation:
+        build_config.build_sources = src_files
 
 """
 Test for the existence of a program
@@ -1964,6 +1990,9 @@ def main(argv = None):
     build_config.public_headers.append(
         os.path.join(build_config.build_dir, 'build.h'))
 
+    if options.gen_amalgamation:
+        generate_amalgamation(build_config, options)
+
     template_vars = create_template_vars(build_config, options,
                                          modules_to_use,
                                          cc,
@@ -1973,9 +2002,6 @@ def main(argv = None):
     # Performs the I/O
     setup_build(build_config, options, template_vars)
 
-    if options.gen_amalgamation:
-        generate_amalgamation(build_config)
-
     def release_date(datestamp):
         if datestamp == 0:
             return 'undated'
diff --git a/doc/dev/todo.rst b/doc/dev/todo.rst
index 223c50bbb..2ac15f037 100644
--- a/doc/dev/todo.rst
+++ b/doc/dev/todo.rst
@@ -8,18 +8,27 @@ and could make a quick project.
 Request a new feature by sending a patch to this file or by writing to
 the mailing list.
 
-Basic Crypto
+Symmetric Algorithms, Hashes, ...
 ----------------------------------------
 
 * Bitsliced AES or Camellia
+* Camellia with AES-NI
 * Serpent using AVX2
 * scrypt
 * BLAKE2b
-* EdDSA
 * Skein-MAC
 * ARIA (Korean block cipher, RFCs 5794 and 6209)
 * Extend Cascade_Cipher to support arbitrary number of ciphers
 
+Public Key Crypto, Math
+----------------------------------------
+
+* EdDSA
+* Ed448-Goldilocks
+* Add specialized reductions for other NIST primes
+* Fast new implementations/algorithms for ECC point operations,
+  Montgomery multiplication, multi-exponentiation, ...
+
 TLS
 ----------------------------------------
 
@@ -39,14 +48,6 @@ PKIX
 * OCSP responder logic
 * X.509 attribute certificates (RFC 5755)
 
-Public Key Crypto, Math, Algorithms
-----------------------------------------
-
-* Add specialized reductions for P-256 and P-384
-* Optimizations for BigInt using SSE2, ARM/NEON, AVX2, ...
-* Fast new implementations/algorithms for ECC point operations,
-  Montgomery multiplication, multi-exponentiation, ...
-
 New Protocols
 ----------------------------------------
 
diff --git a/src/build-data/arch/x86_32.txt b/src/build-data/arch/x86_32.txt
index 54f821a84..89802124d 100644
--- a/src/build-data/arch/x86_32.txt
+++ b/src/build-data/arch/x86_32.txt
@@ -69,7 +69,6 @@ sse4.2
 avx2
 bmi2
 aesni
-clmul
 rdrand
 sha
 </isa_extensions>
diff --git a/src/build-data/cc/clang.txt b/src/build-data/cc/clang.txt
index a7d334aee..cbb68bb21 100644
--- a/src/build-data/cc/clang.txt
+++ b/src/build-data/cc/clang.txt
@@ -38,8 +38,7 @@ sse4.1 -> "-msse4.1"
 sse4.2 -> "-msse4.2"
 avx2   -> "-mavx2"
 bmi2   -> "-mbmi2"
-aesni  -> "-maes"
-clmul  -> "-mpclmul"
+aesni  -> "-maes -mpclmul"
 rdrand -> "-mrdrnd"
 sha    -> "-msha"
 altivec -> "-maltivec"
diff --git a/src/build-data/cc/gcc.txt b/src/build-data/cc/gcc.txt
index 8db3fdf61..a22450f3b 100644
--- a/src/build-data/cc/gcc.txt
+++ b/src/build-data/cc/gcc.txt
@@ -43,8 +43,7 @@ sse4.1  -> "-msse4.1"
 sse4.2  -> "-msse4.2"
 avx2    -> "-mavx2"
 bmi2    -> "-mbmi2"
-aesni   -> "-maes"
-clmul   -> "-mpclmul"
+aesni   -> "-maes -mpclmul"
 rdrand  -> "-mrdrnd"
 sha     -> "-msha"
 altivec -> "-maltivec"
diff --git a/src/scripts/install.py b/src/scripts/install.py
index 69d3a4286..0bf74b7a4 100755
--- a/src/scripts/install.py
+++ b/src/scripts/install.py
@@ -177,6 +177,13 @@ def main(args = None):
         shutil.copyfile(cfg['botan_pkgconfig'],
                         os.path.join(pkgconfig_dir, os.path.basename(cfg['botan_pkgconfig'])))
 
+    if 'ffi' in cfg['mod_list'].split('\n'):
+        logging.debug('FFI enabled - installing Python module')
+        def make_py_lib_path(py_ver):
+            print sys.path
+        python_dir = cfg['python_dir']
+        logging.debug('Python dir %s' % (python_dir))
+
     shutil.rmtree(botan_doc_dir, True)
     shutil.copytree(cfg['doc_output_dir'], botan_doc_dir)