Add a script to handle `make clean` target

This removes a lot of logic that cannot be shared between the
nmake (Windows environment) and gnumake (Unix env) makefiles.
Also it cleans up inconsistencies, eg nmake's make distclean did not
remove amalgamation files, but gnumake version did.

7 files changed, 177 insertions, 115 deletions

diff --git a/configure.py b/configure.py
index 23cc1f14b..15018a27a 100755
--- a/configure.py
+++ b/configure.py
@@ -2096,6 +2096,9 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
                                    if options.house_curve else ''
         }
 
+    variables['test_exe'] = os.path.join(variables['out_dir'],
+                                         'botan-test' + variables['program_suffix'])
+
     if options.build_shared_lib:
 
         if osinfo.soname_pattern_base != None:
@@ -2137,6 +2140,9 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
             variables['libname'] = 'botand'
         else:
             variables['libname'] = 'botan'
+
+        variables['lib_basename'] = variables['libname']
+        variables['cli_exe'] = os.path.join(variables['out_dir'], 'botan-cli' + variables['program_suffix'])
     else:
         variables['botan_pkgconfig'] = os.path.join(build_config.build_dir, PKG_CONFIG_FILENAME)
 
@@ -2144,6 +2150,9 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
         # This can be made consistent over all platforms in the future
         variables['libname'] = 'botan-%d' % (Version.major())
 
+        variables['lib_basename'] = 'lib' + variables['libname']
+        variables['cli_exe'] = os.path.join(variables['out_dir'], 'botan' + variables['program_suffix'])
+
     if options.os == 'llvm':
         # llvm-link doesn't understand -L or -l flags
         variables['link_to_botan'] = '%s/lib%s.a' % (variables['out_dir'], variables['libname'])
@@ -2159,7 +2168,6 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
 
     if variables["makefile_style"] == "gmake":
         templates = [
-            ('gmake_commands.in', True),
             ('gmake_dso.in', options.build_shared_lib),
             ('gmake_coverage.in', options.with_coverage_info),
             ('gmake_fuzzers.in', options.build_fuzzers)
@@ -3160,14 +3168,16 @@ def main_action_configure_build(info_modules, source_paths, options,
     link_headers(build_config.external_headers, 'external',
                  build_config.external_include_dir)
 
-    with open(os.path.join(build_config.build_dir, 'build_config.json'), 'w') as f:
-        json.dump(template_vars, f, sort_keys=True, indent=2)
-
     if options.amalgamation:
         amalgamation_cpp_files = AmalgamationGenerator(build_config, using_mods, options).generate()
         build_config.lib_sources = sorted(amalgamation_cpp_files)
         template_vars.update(MakefileListsGenerator(build_config, options, using_mods, cc, arch, osinfo).generate())
 
+        template_vars['generated_files'] = ' '.join(build_config.lib_sources)
+
+    with open(os.path.join(build_config.build_dir, 'build_config.json'), 'w') as f:
+        json.dump(template_vars, f, sort_keys=True, indent=2)
+
     if options.with_bakefile:
         gen_bakefile(build_config, options, template_vars['link_to'])
 
diff --git a/src/build-data/makefile/gmake.in b/src/build-data/makefile/gmake.in
index c8e7017e5..15e0277e6 100644
--- a/src/build-data/makefile/gmake.in
+++ b/src/build-data/makefile/gmake.in
@@ -1,41 +1,17 @@
 %{header_in}
 
-%{gmake_commands_in}
-
-# Executable targets
-CLI           = %{out_dir}/botan%{program_suffix}
-TEST          = %{out_dir}/botan-test%{program_suffix}
+COPY             = cp
+LN               = ln -fs
 
 # Library targets
-LIB_BASENAME  = %{lib_prefix}%{libname}
+LIB_BASENAME  = %{lib_basename}
 STATIC_LIB    = %{out_dir}/$(LIB_BASENAME).%{static_suffix}
 LIBRARIES     = $(STATIC_LIB)
 
-# File Lists
-INCLUDE_DIR   = %{botan_include_dir}
-
-LIBOBJS       = %{lib_objs}
-
-CLIOBJS       = %{cli_objs}
-
-TESTOBJS      = %{test_objs}
-
-# First make target. Will be used by default
-all: libs cli tests
-
-# Build Commands
-%{lib_build_cmds}
-
-%{cli_build_cmds}
-
-%{test_build_cmds}
-
 # Link Commands
 %{gmake_dso_in}
 
 libs: $(LIBRARIES)
-cli: $(CLI)
-tests: $(TEST)
 
 $(CLI): $(LIBRARIES) $(CLIOBJS)
 	$(CLI_LINK_CMD) $(LDFLAGS) $(CLIOBJS) $(CLI_LINKS_TO) -o $(CLI)
@@ -46,7 +22,6 @@ $(TEST): $(LIBRARIES) $(TESTOBJS)
 	$(TEST_POST_LINK_CMD)
 
 $(STATIC_LIB): $(LIBOBJS)
-	$(RM) $(STATIC_LIB)
 	$(AR) %{ar_options} $(STATIC_LIB) $(LIBOBJS)
 
 # Fake targets
@@ -56,26 +31,5 @@ $(STATIC_LIB): $(LIBOBJS)
 
 %{gmake_fuzzers_in}
 
-SPHINX_CONFIG = %{sphinx_config_dir}
-SPHINX_OPTS = -b html
-
-clean:
-	-$(RM) %{libobj_dir}/*
-	-$(RM) %{testobj_dir}/*
-	-$(RM) %{cliobj_dir}/*
-	-$(RM) $(SONAME_ABI) $(SONAME_BASE)
-	-$(RM) $(LIBRARIES) $(CLI) $(TEST)
-
-distclean: clean
-	$(RM) Makefile
-	$(RM_R) %{build_dir}
-	$(RM) botan_all.cpp botan_all.h
-
 valgrind:
 	valgrind --log-file=botan.%%p.log -v --track-origins=yes --leak-check=full --show-reachable=yes ./botan-test
-
-docs:
-%{build_doc_commands}
-
-install: $(CLI) docs
-	$(PYTHON_EXE) $(SCRIPTS_DIR)/install.py --prefix=%{prefix} --build-dir="%{build_dir}" --bindir=%{bindir} --libdir=%{libdir} --docdir=%{docdir} --includedir=%{includedir}
diff --git a/src/build-data/makefile/gmake_commands.in b/src/build-data/makefile/gmake_commands.in
deleted file mode 100644
index 3d492ae51..000000000
--- a/src/build-data/makefile/gmake_commands.in
+++ /dev/null
@@ -1,5 +0,0 @@
-# Program aliases
-COPY             = cp
-LN               = ln -fs
-RM               = @rm -f
-RM_R             = @rm -rf
diff --git a/src/build-data/makefile/header.in b/src/build-data/makefile/header.in
index de31c8a0b..946960053 100644
--- a/src/build-data/makefile/header.in
+++ b/src/build-data/makefile/header.in
@@ -29,3 +29,43 @@ INSTALLED_LIB_DIR = %{prefix}/%{libdir}
 
 CLI_POST_LINK_CMD  = %{cli_post_link_cmd}
 TEST_POST_LINK_CMD = %{test_post_link_cmd}
+
+# Executable targets
+CLI           = %{cli_exe}
+TEST          = %{test_exe}
+
+# The primary target
+all: libs cli tests
+
+cli: $(CLI)
+tests: $(TEST)
+
+# Object Files
+LIBOBJS       = %{lib_objs}
+
+CLIOBJS       = %{cli_objs}
+
+TESTOBJS      = %{test_objs}
+
+# Build Commands
+%{lib_build_cmds}
+
+%{cli_build_cmds}
+
+%{test_build_cmds}
+
+# Misc targets
+SPHINX_CONFIG = %{sphinx_config_dir}
+SPHINX_OPTS = -b html
+
+docs:
+%{build_doc_commands}
+
+clean:
+	$(PYTHON_EXE) $(SCRIPTS_DIR)/cleanup.py --build-dir="%{build_dir}"
+
+distclean:
+	$(PYTHON_EXE) $(SCRIPTS_DIR)/cleanup.py --build-dir="%{build_dir}" --distclean
+
+install: $(CLI) docs
+	$(PYTHON_EXE) $(SCRIPTS_DIR)/install.py --prefix=%{prefix} --build-dir="%{build_dir}" --bindir=%{bindir} --libdir=%{libdir} --docdir=%{docdir} --includedir=%{includedir}
diff --git a/src/build-data/makefile/nmake.in b/src/build-data/makefile/nmake.in
index eba3a774d..c44fd33aa 100644
--- a/src/build-data/makefile/nmake.in
+++ b/src/build-data/makefile/nmake.in
@@ -3,19 +3,12 @@
 ### Aliases for Common Programs
 
 COPY          = copy
-RM            = @del /Q
-RM_R          = $(RM) /S
-RMDIR         = @rmdir
-
-# Executable targets
-CLI           = %{out_dir}\botan-cli%{program_suffix}
-TEST          = %{out_dir}\botan-test%{program_suffix}
 
 # Library targets
 #
 # LIB_FILENAME is always the .lib file, that is either a static lib or a
 # by-product of the DLL creation used to link the DLL into applications
-LIB_BASENAME  = %{libname}
+LIB_BASENAME  = %{lib_basename}
 LIB_FILENAME  = %{out_dir}\$(LIB_BASENAME).lib
 
 !If "$(SO_OBJ_FLAGS)" == ""
@@ -27,24 +20,6 @@ SO_FILENAME   = %{out_dir}\$(LIB_BASENAME).dll
 LIBRARIES     = $(SO_FILENAME)
 !Endif
 
-
-# File Lists
-LIBOBJS       = %{lib_objs}
-
-CLIOBJS       = %{cli_objs}
-
-TESTOBJS      = %{test_objs}
-
-# First make target. Will be used by default
-all: libs cli tests
-
-# Build Commands
-%{lib_build_cmds}
-
-%{cli_build_cmds}
-
-%{test_build_cmds}
-
 # Link Commands
 $(CLI): $(LIBRARIES) $(CLIOBJS)
 	$(CLI_LINK_CMD) /OUT:$@ $(CLIOBJS) $(LIB_FILENAME) $(CLI_LINKS_TO)
@@ -55,8 +30,6 @@ $(TEST): $(LIBRARIES) $(TESTOBJS)
 	$(TEST_POST_LINK_CMD)
 
 libs: $(LIBRARIES)
-cli: $(CLI)
-tests: $(TEST)
 
 !If "$(SO_OBJ_FLAGS)" == ""
 # static lib
@@ -68,32 +41,3 @@ $(LIB_FILENAME): $(LIBOBJS)
 $(SO_FILENAME): $(LIBOBJS)
 	$(LIB_LINK_CMD) /OUT:$@ $(LIBOBJS) $(LIB_LINKS_TO)
 !Endif
-
-# Fake Targets
-
-SPHINX_CONFIG = %{sphinx_config_dir}
-SPHINX_OPTS = -b html
-
-docs:
-%{build_doc_commands}
-
-clean:
-	-$(RM) %{libobj_dir}\*
-	-$(RM) %{cliobj_dir}\*
-	-$(RM) %{testobj_dir}\*
-	-$(RM) %{out_dir}\*.manifest
-	-$(RM) %{out_dir}\*.exp
-	-$(RM) %{out_dir}\*.dll
-	-$(RM) $(LIBRARIES) $(CLI) $(TEST)
-
-distclean: clean
-	$(RM_R) %{build_dir}
-	$(RMDIR) %{build_dir}\include\botan\internal
-	$(RMDIR) %{build_dir}\include\botan
-	$(RMDIR) %{build_dir}\include
-	$(RMDIR) %{build_dir}\lib %{build_dir}\tests
-	$(RMDIR) %{build_dir}
-	$(RM) Makefile $(LIB_BASENAME).* $(CLI).*
-
-install: $(CLI) docs
-	$(PYTHON_EXE) $(SCRIPTS_DIR)\install.py --prefix=%{prefix} --build-dir="%{build_dir}" --bindir=%{bindir} --libdir=%{libdir} --docdir=%{docdir} --includedir=%{includedir}
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
index e7fd1e80c..b4ae98a0a 100755
--- a/src/scripts/ci_build.py
+++ b/src/scripts/ci_build.py
@@ -362,6 +362,7 @@ def main(args=None):
             'src/python/botan2.py',
             'src/scripts/ci_build.py',
             'src/scripts/install.py',
+            'src/scripts/cleanup.py',
             'src/scripts/website.py',
             'src/scripts/python_unittests.py',
             'src/scripts/python_unittests_unix.py']
@@ -476,6 +477,9 @@ def main(args=None):
                 # Otherwise generate a local HTML report
                 cmds.append(['genhtml', cov_file, '--output-directory', 'lcov-out'])
 
+        cmds.append(make_cmd + ['clean'])
+        cmds.append(make_cmd + ['distclean'])
+
     for cmd in cmds:
         if options.dry_run:
             print('$ ' + ' '.join(cmd))
diff --git a/src/scripts/cleanup.py b/src/scripts/cleanup.py
new file mode 100755
index 000000000..36d35d3ec
--- /dev/null
+++ b/src/scripts/cleanup.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+
+"""
+Implements the "make clean" target
+
+(C) 2017 Jack Lloyd
+
+Botan is released under the Simplified BSD License (see license.txt)
+"""
+
+import os
+import sys
+import re
+import optparse  # pylint: disable=deprecated-module
+import logging
+import json
+import shutil
+
+def remove_dir(d):
+    try:
+        if os.access(d, os.X_OK):
+            logging.debug('Removing directory "%s"', d)
+            shutil.rmtree(d)
+    except Exception as e: # pylint: disable=broad-except
+        logging.error('Failed removing directory "%s": %s', d, e)
+
+def remove_file(f):
+    try:
+        if os.access(f, os.R_OK):
+            logging.debug('Removing file "%s"', f)
+            os.unlink(f)
+    except Exception as e: # pylint: disable=broad-except
+        logging.error('Failed removing file "%s": %s', f, e)
+
+def remove_all_in_dir(d):
+    if os.access(d, os.X_OK):
+        logging.debug('Removing all files in directory "%s"', d)
+
+        for f in os.listdir(d):
+            remove_file(os.path.join(d, f))
+
+def main(args=None):
+    if args is None:
+        args = sys.argv
+
+    parser = optparse.OptionParser()
+    parser.add_option('--build-dir', default='build', metavar='DIR',
+                      help='specify build dir to clean (default %default)')
+
+    parser.add_option('--distclean', action='store_true', default=False,
+                      help='clean everything')
+    parser.add_option('--verbose', action='store_true', default=False,
+                      help='noisy logging')
+
+    (options, args) = parser.parse_args(args)
+
+    logging.basicConfig(stream=sys.stderr,
+                        format='%(levelname) 7s: %(message)s',
+                        level=logging.DEBUG if options.verbose else logging.INFO)
+
+    if len(args) > 1:
+        logging.error("Unknown arguments")
+        return 1
+
+    build_dir = options.build_dir
+
+    if os.access(build_dir, os.X_OK) != True:
+        logging.error("Unable to access build directory")
+        return 1
+
+    build_config_path = os.path.join(build_dir, 'build_config.json')
+    build_config_str = None
+
+    try:
+        build_config_file = open(build_config_path)
+        build_config_str = build_config_file.read()
+        build_config_file.close()
+    except Exception: # pylint: disable=broad-except
+        # Ugh have to do generic catch as different exception type thrown in Python2
+        logging.error("Unable to access build_config.json in build dir")
+        return 1
+
+    build_config = json.loads(build_config_str)
+
+    #print(json.dumps(build_config, sort_keys=True, indent=3))
+
+    if options.distclean:
+        build_dir = build_config['build_dir']
+        remove_dir(build_dir)
+    else:
+        for dir_type in ['libobj_dir', 'cliobj_dir', 'testobj_dir']:
+            dir_path = build_config[dir_type]
+            remove_all_in_dir(dir_path)
+
+    remove_file(build_config['cli_exe'])
+    remove_file(build_config['test_exe'])
+
+    matches_libname = re.compile('^' + build_config['lib_basename'] + '.([a-z]+)')
+
+    known_suffix = ['a', 'so', 'dll', 'manifest', 'exp']
+
+    for f in os.listdir(build_config['out_dir']):
+        match = matches_libname.match(f)
+        if match and match.group(1) in known_suffix:
+            remove_file(f)
+
+    if options.distclean:
+        if 'generated_files' in build_config:
+            for f in build_config['generated_files'].split(' '):
+                remove_file(f)
+
+        remove_file(build_config['makefile_path'])
+
+if __name__ == '__main__':
+    sys.exit(main())