Add option --module-policy

A module policy is a file specifying three types of modules: ones which
are required, ones which are prohibited, and ones which should be used
if otherwise available (this is mostly for platform specific modules).

Finally there are whatever modules which exist in the library of which
the policy makes no mention. These will be included if an explicit
dependency of some other module pulls them in (so there is no reason
to mention base, utils, ... in the file) but skipped otherwise.

For example policy 'sane' does not mention 'utils' or 'twofish' either
way. Since utils is a dependency of other modules which are included,
but Twofish does not. However unlike an explicitly prohibited module,
not mentioned can still be requested as part of the build (here with
--enable-module=twofish)

Also fixes some test bugs noticed by compiling in different build
configs. DLIES test didn't check that the KDF and MAC existed. Adds a
typedef for MessageAuthenticationCode because typing it twice in a
single line in the DLIES test made me think it's way too long. :) Also
fix some fuzzer build problems. Due to a copy and paste bug the PKCS
certificate (it was not).

Inspired by GH #439

7 files changed, 383 insertions, 69 deletions

diff --git a/configure.py b/configure.py
index 88c2b1832..e04a0b79a 100755
--- a/configure.py
+++ b/configure.py
@@ -359,6 +359,10 @@ def process_command_line(args):
 
     mods_group = optparse.OptionGroup(parser, 'Module selection')
 
+    mods_group.add_option('--module-policy', dest='module_policy',
+                          help="module policy file (see src/build-data/policy)",
+                          metavar='POL', default=None)
+
     mods_group.add_option('--enable-modules', dest='enabled_modules',
                           metavar='MODS', action='append',
                           help='enable specific modules')
@@ -711,6 +715,11 @@ class ModuleInfo(object):
             return 0
         return 1
 
+class ModulePolicyInfo(object):
+    def __init__(self, infofile):
+        lex_me_harder(infofile, self,
+                      ['required', 'if_available', 'prohibited'], {})
+
 class ArchInfo(object):
     def __init__(self, infofile):
         lex_me_harder(infofile, self,
@@ -1394,7 +1403,7 @@ def create_template_vars(build_config, options, modules, cc, arch, osinfo):
 """
 Determine which modules to load based on options, target, etc
 """
-def choose_modules_to_use(modules, archinfo, ccinfo, options):
+def choose_modules_to_use(modules, module_policy, archinfo, ccinfo, options):
 
     for mod in modules.values():
         mod.dependencies_exist(modules)
@@ -1406,6 +1415,18 @@ def choose_modules_to_use(modules, archinfo, ccinfo, options):
     def cannot_use_because(mod, reason):
         not_using_because.setdefault(reason, []).append(mod)
 
+    def check_usable(module, modname, options):
+        if not module.compatible_os(options.os):
+            cannot_use_because(modname, 'incompatible OS')
+            return False
+        elif not module.compatible_compiler(ccinfo, archinfo.basename):
+            cannot_use_because(modname, 'incompatible compiler')
+            return False
+        elif not module.compatible_cpu(archinfo, options):
+            cannot_use_because(modname, 'incompatible CPU')
+            return False
+        return True
+
     for modname in options.enabled_modules:
         if modname not in modules:
             logging.error("Module not found: %s" % (modname))
@@ -1415,19 +1436,35 @@ def choose_modules_to_use(modules, archinfo, ccinfo, options):
             logging.warning("Disabled module not found: %s" % (modname))
 
     for (modname, module) in modules.items():
+        usable = check_usable(module, modname, options)
+
+        if module_policy is not None:
+
+            if modname in module_policy.required:
+                if not usable:
+                    logging.error('Module policy requires module %s not usable on this platform' % (modname))
+                elif modname in options.disabled_modules:
+                    logging.error('Module %s was disabled but is required by policy' % (modname))
+                to_load.append(modname)
+                continue
+            elif modname in module_policy.if_available:
+                if modname in options.disabled_modules:
+                    cannot_use_because(modname, 'disabled by user')
+                elif usable:
+                    logging.debug('Enabling optional module %s' % (modname))
+                    to_load.append(modname)
+                continue
+            elif modname in module_policy.prohibited:
+                if modname in options.enabled_modules:
+                    logging.error('Module %s was requested but is prohibited by policy' % (modname))
+                cannot_use_because(modname, 'prohibited by module policy')
+                continue
+
         if modname in options.disabled_modules:
             cannot_use_because(modname, 'disabled by user')
         elif modname in options.enabled_modules:
             to_load.append(modname) # trust the user
-
-        elif not module.compatible_os(options.os):
-            cannot_use_because(modname, 'incompatible OS')
-        elif not module.compatible_compiler(ccinfo, archinfo.basename):
-            cannot_use_because(modname, 'incompatible compiler')
-        elif not module.compatible_cpu(archinfo, options):
-            cannot_use_because(modname, 'incompatible CPU')
-
-        else:
+        elif usable:
             if module.load_on == 'never':
                 cannot_use_because(modname, 'disabled as buggy')
             elif module.load_on == 'request':
@@ -1447,7 +1484,7 @@ def choose_modules_to_use(modules, archinfo, ccinfo, options):
                 to_load.append(modname)
 
             elif module.load_on == 'auto':
-                if options.no_autoload:
+                if options.no_autoload or module_policy is not None:
                     maybe_dep.append(modname)
                 else:
                     to_load.append(modname)
@@ -1483,7 +1520,7 @@ def choose_modules_to_use(modules, archinfo, ccinfo, options):
                     cannot_use_because(modname, 'dependency failure')
 
     for not_a_dep in maybe_dep:
-        cannot_use_because(not_a_dep, 'loaded only if needed by dependency')
+        cannot_use_because(not_a_dep, 'only used if needed or requested')
 
     for reason in sorted(not_using_because.keys()):
         disabled_mods = sorted(set([mod for mod in not_using_because[reason]]))
@@ -1510,52 +1547,6 @@ def choose_modules_to_use(modules, archinfo, ccinfo, options):
     return to_load
 
 """
-Load the info files about modules, targets, etc
-"""
-def load_info_files(options):
-
-    def find_files_named(desired_name, in_path):
-        for (dirpath, dirnames, filenames) in os.walk(in_path):
-            if desired_name in filenames:
-                yield os.path.join(dirpath, desired_name)
-
-    modules = dict([(mod.basename, mod) for mod in
-                    [ModuleInfo(info) for info in
-                     find_files_named('info.txt', options.lib_dir)]])
-
-    def list_files_in_build_data(subdir):
-        for (dirpath, dirnames, filenames) in \
-                os.walk(os.path.join(options.build_data, subdir)):
-            for filename in filenames:
-                if filename.endswith('.txt'):
-                    yield os.path.join(dirpath, filename)
-
-    def form_name(filepath):
-        return os.path.basename(filepath).replace('.txt', '')
-
-    archinfo = dict([(form_name(info), ArchInfo(info))
-                     for info in list_files_in_build_data('arch')])
-
-    osinfo   = dict([(form_name(info), OsInfo(info))
-                      for info in list_files_in_build_data('os')])
-
-    ccinfo = dict([(form_name(info), CompilerInfo(info))
-                    for info in list_files_in_build_data('cc')])
-
-    def info_file_load_report(type, num):
-        if num > 0:
-            logging.debug('Loaded %d %s info files' % (num, type))
-        else:
-            logging.warning('Failed to load any %s info files' % (type))
-
-    info_file_load_report('CPU', len(archinfo));
-    info_file_load_report('OS', len(osinfo))
-    info_file_load_report('compiler', len(ccinfo))
-
-    return (modules, archinfo, ccinfo, osinfo)
-
-
-"""
 Choose the link method based on system availablity and user request
 """
 def choose_link_method(options):
@@ -1856,7 +1847,35 @@ def main(argv = None):
     options.build_data = os.path.join(options.src_dir, 'build-data')
     options.makefile_dir = os.path.join(options.build_data, 'makefile')
 
-    (modules, info_arch, info_cc, info_os) = load_info_files(options)
+    def find_files_named(desired_name, in_path):
+        for (dirpath, dirnames, filenames) in os.walk(in_path):
+            if desired_name in filenames:
+                yield os.path.join(dirpath, desired_name)
+
+    modules = dict([(mod.basename, mod) for mod in
+                    [ModuleInfo(info) for info in
+                     find_files_named('info.txt', options.lib_dir)]])
+
+    def load_build_data(descr, subdir, class_t):
+        info = {}
+
+        subdir = os.path.join(options.build_data, subdir)
+
+        for fsname in os.listdir(subdir):
+            if fsname.endswith('.txt'):
+                info[fsname.replace('.txt', '')] = class_t(os.path.join(subdir, fsname))
+        if len(info) == 0:
+            logging.warning('Failed to load any %s files' % (descr))
+        else:
+            logging.debug('Loaded %d %s files' % (len(info), descr))
+
+        return info
+
+    info_arch = load_build_data('CPU info', 'arch', ArchInfo)
+    info_os   = load_build_data('OS info', 'os', OsInfo)
+    info_cc   = load_build_data('compiler info', 'cc', CompilerInfo)
+
+    module_policies = load_build_data('module policy', 'policy', ModulePolicyInfo)
 
     if options.list_modules:
         for k in sorted(modules.keys()):
@@ -1932,6 +1951,12 @@ def main(argv = None):
     cc = info_cc[options.compiler]
     arch = info_arch[options.arch]
     osinfo = info_os[options.os]
+    module_policy = None
+
+    if options.module_policy != None:
+        if options.module_policy not in module_policies:
+            logging.error("Unknown module set %s", options.module_policy)
+        module_policy = module_policies[options.module_policy]
 
     if options.with_visibility is None:
         options.with_visibility = True
@@ -1948,7 +1973,7 @@ def main(argv = None):
         raise Exception('Botan does not support building as shared library on the target os. '
                 'Build static using --disable-shared.')
 
-    loaded_mods = choose_modules_to_use(modules, arch, cc, options)
+    loaded_mods = choose_modules_to_use(modules, module_policy, arch, cc, options)
 
     for m in loaded_mods:
         if modules[m].load_on == 'vendor':
diff --git a/src/build-data/policy/bsi.txt b/src/build-data/policy/bsi.txt
new file mode 100644
index 000000000..9ab68a921
--- /dev/null
+++ b/src/build-data/policy/bsi.txt
@@ -0,0 +1,156 @@
+<required>
+# block
+aes
+
+# modes
+gcm
+cbc
+mode_pad
+
+# stream
+ctr
+
+# hash
+sha2_32
+sha2_64
+keccak
+
+# mac
+cmac
+hmac
+
+# pk_pad
+eme_oaep
+emsa_pssr
+
+# pubkey
+dlies
+dh
+rsa
+dsa
+ecdsa
+ecdh
+
+# rng
+auto_rng
+hmac_rng
+hmac_drbg
+</required>
+
+<if_available>
+# block
+aes_ni
+aes_ssse3
+
+# modes
+clmul
+
+# entropy sources
+beos_stats
+darwin_secrandom
+egd
+proc_walk
+unix_procs
+rdrand
+rdseed
+hres_timer
+dev_random
+system_rng
+cryptoapi_rng
+win32_stats
+
+# utils
+locking_allocator
+simd_altivec
+simd_scalar
+simd_sse2
+</if_available>
+
+<prohibited>
+# block
+blowfish
+camellia
+cascade
+cast
+gost_28147
+idea
+idea_sse2
+kasumi
+lion
+mars
+misty1
+noekeon
+noekeon_simd
+rc2
+rc5
+rc6
+safer
+seed
+serpent
+serpent_simd
+tea
+threefish
+threefish_avx2
+twofish
+xtea
+xtea_simd
+
+# modes
+ccm
+chacha20poly1305
+eax
+ocb
+siv
+cfb
+ecb
+
+# stream
+chacha
+ofb
+rc4
+salsa20
+
+# pubkey
+curve25519
+elgamal
+gost_3410
+mce
+mceies
+nr
+rw
+
+# pk_pad
+#eme_pkcs1 // needed for tls
+eme_raw
+#emsa_pkcs1 // needed for tls
+emsa_raw
+emsa_x931
+emsa1
+emsa1_bsi
+
+# hash
+blake2
+comb4p
+gost_3411
+has160
+md2
+md4
+#md5 // needed for tls
+rmd128
+rmd160
+#sha1 // needed for tls
+#sha1_sse2 // needed for tls
+skein
+tiger
+whirlpool
+
+# mac
+cbc_mac
+poly1305
+siphash
+x919_mac
+
+# rng
+x931_rng
+
+</prohibited>
diff --git a/src/build-data/policy/sane.txt b/src/build-data/policy/sane.txt
new file mode 100644
index 000000000..3482296d6
--- /dev/null
+++ b/src/build-data/policy/sane.txt
@@ -0,0 +1,120 @@
+<required>
+aes
+serpent
+threefish
+chacha
+
+sha2_32
+sha2_64
+blake2
+skein
+keccak
+
+gcm
+ocb
+chacha20poly1305
+
+kdf2
+hkdf
+cmac
+hmac
+poly1305
+siphash
+
+pbkdf2
+
+# required for private key encryption
+pbes2
+
+# required for TLS
+prf_tls
+
+curve25519
+ecdh
+ecdsa
+rsa
+
+eme_oaep
+emsa_pssr
+emsa1
+
+auto_rng
+hmac_rng
+
+ffi
+</required>
+
+<prohibited>
+cast
+des
+gost_28147
+idea
+idea_sse2
+kasumi
+lion
+mars
+misty1
+rc2
+rc4
+rc5
+rc6
+safer
+seed
+tea
+xtea
+xtea_simd
+
+cbc_mac
+x919_mac
+
+# MD5 and SHA1 are broken but not prohibited. They are widely in use
+# in non-crypto contexts and are required by TLS currently
+md2
+md4
+rmd128
+has160
+gost_3411
+
+cfb
+ecb
+ofb
+
+elgamal
+rw
+nr
+gost_3410
+
+emsa_x931
+pbkdf1
+prf_x942
+x931_rng
+
+passhash9
+cryptobox
+unix_procs
+</prohibited>
+
+<if_available>
+clmul
+locking_allocator
+
+sha1_sse2
+aes_ni
+aes_ssse3
+noekeon_simd
+serpent_simd
+threefish_avx2
+
+simd_scalar
+simd_sse2
+simd_altivec
+
+# entropy sources
+rdrand
+rdseed
+hres_timer
+dev_random
+system_rng
+cryptoapi_rng
+win32_stats
+</if_available>
diff --git a/src/lib/mac/mac.h b/src/lib/mac/mac.h
index 90ef4db15..fe3388f3b 100644
--- a/src/lib/mac/mac.h
+++ b/src/lib/mac/mac.h
@@ -53,6 +53,8 @@ class BOTAN_DLL MessageAuthenticationCode : public Buffered_Computation,
       virtual MessageAuthenticationCode* clone() const = 0;
    };
 
+typedef MessageAuthenticationCode MAC;
+
 }
 
 #endif
diff --git a/src/lib/pubkey/dlies/dlies.cpp b/src/lib/pubkey/dlies/dlies.cpp
index 86cd51e19..ba890ac3d 100644
--- a/src/lib/pubkey/dlies/dlies.cpp
+++ b/src/lib/pubkey/dlies/dlies.cpp
@@ -21,6 +21,8 @@ DLIES_Encryptor::DLIES_Encryptor(const PK_Key_Agreement_Key& key,
    m_mac(mac_obj),
    m_mac_keylen(mac_kl)
    {
+   BOTAN_ASSERT_NONNULL(kdf_obj);
+   BOTAN_ASSERT_NONNULL(mac_obj);
    m_my_key = key.public_value();
    }
 
diff --git a/src/tests/test_dlies.cpp b/src/tests/test_dlies.cpp
index 1c7327ab4..ba8142dcb 100644
--- a/src/tests/test_dlies.cpp
+++ b/src/tests/test_dlies.cpp
@@ -42,20 +42,29 @@ class DLIES_KAT_Tests : public Text_Based_Test
          Botan::DH_PrivateKey from(Test::rng(), domain, x1);
          Botan::DH_PrivateKey to(Test::rng(), domain, x2);
 
-         const std::string kdf = "KDF2(SHA-1)";
-         const std::string mac = "HMAC(SHA-1)";
+         const std::string kdf_algo = "KDF2(SHA-1)";
+         const std::string mac_algo = "HMAC(SHA-1)";
          const size_t mac_key_len = 16;
 
          Test::Result result("DLIES");
 
+         std::unique_ptr<Botan::KDF> kdf(Botan::KDF::create(kdf_algo));
+         std::unique_ptr<Botan::MAC> mac(Botan::MAC::create(mac_algo));
+
+         if(!kdf || !mac)
+            {
+            result.test_note("Skipping due to missing KDF or MAC algo");
+            return result;
+            }
+
          Botan::DLIES_Encryptor encryptor(from,
-                                          Botan::KDF::create(kdf).release(),
-                                          Botan::MessageAuthenticationCode::create(mac).release(),
+                                          kdf->clone(),
+                                          mac->clone(),
                                           mac_key_len);
 
          Botan::DLIES_Decryptor decryptor(to,
-                                          Botan::KDF::create(kdf).release(),
-                                          Botan::MessageAuthenticationCode::create(mac).release(),
+                                          kdf.release(),
+                                          mac.release(),
                                           mac_key_len);
 
          encryptor.set_other_key(to.public_value());
diff --git a/src/tests/test_fuzzer.cpp b/src/tests/test_fuzzer.cpp
index 18516a68c..2be8e7c08 100644
--- a/src/tests/test_fuzzer.cpp
+++ b/src/tests/test_fuzzer.cpp
@@ -6,12 +6,12 @@
 
 #include "tests.h"
 #include <chrono>
+#include <botan/internal/filesystem.h>
 
 #if defined(BOTAN_HAS_X509_CERTIFICATES)
   #include <botan/x509cert.h>
   #include <botan/x509_crl.h>
   #include <botan/base64.h>
-  #include <botan/internal/filesystem.h>
 #endif
 
 #if defined(BOTAN_HAS_PUBLIC_KEY_CRYPTO)
@@ -61,8 +61,8 @@ class Fuzzer_Input_Tests : public Test
             {
             try
                {
-               std::unique_ptr<Botan::Private_Key> key(Botan::PKCS8::load_key(vec_file, Test::rng()));
-               Botan::X509_Certificate cert(vec_file);
+               std::unique_ptr<Botan::Private_Key> key(
+                  Botan::PKCS8::load_key(vec_file, Test::rng()));
                }
             catch(std::exception&) {}