Add a script for generating the documentation

6 files changed, 205 insertions, 73 deletions

diff --git a/configure.py b/configure.py
index ee26d0320..b15ef4a1c 100755
--- a/configure.py
+++ b/configure.py
@@ -261,31 +261,6 @@ class BuildPaths(object): # pylint: disable=too-many-instance-attributes
         else:
             raise InternalError("Unknown src info type '%s'" % (typ))
 
-
-def make_build_doc_commands(source_paths, build_paths, options):
-
-    if options.with_documentation is False:
-        return ""
-
-    def build_manual_command(src_dir, dst_dir):
-        if options.with_sphinx:
-            sphinx = 'sphinx-build -b html -c %s ' % (source_paths.sphinx_config_dir)
-            if options.quiet:
-                sphinx += '-q '
-            sphinx += '%s %s' % (src_dir, dst_dir)
-            return sphinx
-        else:
-            cp_command = 'copy' if options.os == 'windows' else 'cp'
-            return '%s %s%s*.rst %s' % (cp_command, src_dir, os.sep, dst_dir)
-
-    cmds = [
-        build_manual_command(os.path.join(source_paths.doc_dir, 'manual'), build_paths.doc_output_dir_manual)
-    ]
-    if options.with_doxygen:
-        cmds += ['doxygen %s%sbotan.doxy' % (build_paths.build_dir, os.sep)]
-    return '\n'.join(['\t' + cmd for cmd in cmds])
-
-
 def process_command_line(args): # pylint: disable=too-many-locals
     """
     Handle command line options
@@ -2018,6 +1993,8 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
         'base_dir': source_paths.base_dir,
         'src_dir': source_paths.src_dir,
         'doc_dir': source_paths.doc_dir,
+        'scripts_dir': source_paths.scripts_dir,
+        'python_dir': source_paths.python_dir,
 
         'cli_exe': os.path.join(build_dir, osinfo.cli_exe_name + program_suffix),
         'test_exe': os.path.join(build_dir, 'botan-test' + program_suffix),
@@ -2029,8 +2006,6 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
         'command_line': configure_command_line(),
         'local_config': read_textfile(options.local_config),
 
-        'makefile_path': os.path.join(build_config.build_dir, '..', 'Makefile'),
-
         'program_suffix': program_suffix,
 
         'prefix': options.prefix or osinfo.install_root,
@@ -2038,12 +2013,17 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
         'libdir': options.libdir or osinfo.lib_dir,
         'includedir': options.includedir or osinfo.header_dir,
         'docdir': options.docdir or osinfo.doc_dir,
+
         'with_documentation': options.with_documentation,
+        'with_sphinx': options.with_sphinx,
+        'sphinx_config_dir': source_paths.sphinx_config_dir,
+        'with_doxygen': options.with_doxygen,
 
         'out_dir': options.with_build_dir or os.path.curdir,
         'build_dir': build_config.build_dir,
 
-        'scripts_dir': source_paths.scripts_dir,
+        'doc_stamp_file': os.path.join(build_config.build_dir, 'doc.stamp'),
+        'makefile_path': os.path.join(build_config.build_dir, '..', 'Makefile'),
 
         'build_static_lib': options.build_static_lib,
         'build_fuzzers': options.build_fuzzers,
@@ -2058,13 +2038,8 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
         'fuzzobj_dir': build_config.fuzzobj_dir,
 
         'fuzzer_output_dir': build_config.fuzzer_output_dir if build_config.fuzzer_output_dir else '',
-
         'doc_output_dir': build_config.doc_output_dir,
 
-        'build_doc_commands': make_build_doc_commands(source_paths, build_config, options),
-
-        'python_dir': source_paths.python_dir,
-
         'os': options.os,
         'arch': options.arch,
         'submodel': options.cpu,
@@ -2073,6 +2048,9 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
 
         'mp_bits': choose_mp_bits(),
 
+        'python_exe': sys.executable,
+        'python_version': options.python_version,
+
         'cxx': (options.compiler_binary or cc.binary_name),
         'cxx_abi_flags': cc.mach_abi_link_flags(options),
         'linker': cc.linker_name or '$(CXX)',
@@ -2090,6 +2068,10 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
         'exe_link_cmd': cc.binary_link_command_for(osinfo.basename, options) + external_link_cmd(),
         'post_link_cmd': '',
 
+        'ar_command': options.ar_command or cc.ar_command or osinfo.ar_command,
+        'ar_options': cc.ar_options or osinfo.ar_options,
+        'ar_output_to': cc.ar_output_to,
+
         'link_to': ' '.join(
             [cc.add_lib_option + lib for lib in link_to('libs')] +
             [cc.add_framework_option + fw for fw in link_to('frameworks')]
@@ -2110,15 +2092,8 @@ def create_template_vars(source_paths, build_config, options, modules, cc, arch,
         'unsafe_fuzzer_mode_define': '#define BOTAN_UNSAFE_FUZZER_MODE' if options.unsafe_fuzzer_mode else '',
         'fuzzer_type': '#define BOTAN_FUZZER_IS_%s' % (options.build_fuzzers.upper()) if options.build_fuzzers else '',
 
-        'python_exe': sys.executable,
-        'ar_command': options.ar_command or cc.ar_command or osinfo.ar_command,
-        'ar_options': cc.ar_options or osinfo.ar_options,
-        'ar_output_to': cc.ar_output_to,
-
         'mod_list': '\n'.join(sorted([m.basename for m in modules])),
 
-        'python_version': options.python_version,
-        'with_sphinx': options.with_sphinx,
         'house_ecc_curve_defines': make_cpp_macros(HouseEccCurve(options.house_curve).defines()) \
                                    if options.house_curve else '',
         }
diff --git a/doc/contributing.rst b/doc/contributing.rst
index 31e8cb39f..0ce0cd9f6 100644
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@@ -97,12 +97,12 @@ Python
 
 Scripts should be in Python whenever possible.
 
-For configure.py (and install.py) the target is stock (no modules outside the
-standard library) CPython 2.7 plus latest CPython 3.x. Support for CPython 2.6,
-PyPy, etc is great when viable (in the sense of not causing problems for 2.7 or
-3.x, and not requiring huge blocks of version dependent code). As running this
-program succesfully is required for a working build making it as portable as
-possible is considered key.
+For configure.py (and helper scripts install.py, cleanup.py and build_docs.py)
+the target is stock (no modules outside the standard library) CPython 2.7 plus
+latest CPython 3.x. Support for CPython 2.6, PyPy, etc is great when viable (in
+the sense of not causing problems for 2.7 or 3.x, and not requiring huge blocks
+of version dependent code). As running this program succesfully is required for
+a working build, making it as portable as possible is considered key.
 
 The python wrapper botan2.py targets CPython 2.7, 3.x, and latest PyPy. Note that
 a single file is used to avoid dealing with any of Python's various crazy module
diff --git a/src/build-data/makefile.in b/src/build-data/makefile.in
index 2c97ef792..614e5bb55 100644
--- a/src/build-data/makefile.in
+++ b/src/build-data/makefile.in
@@ -28,7 +28,7 @@ INSTALLED_LIB_DIR = %{prefix}/%{libdir}
 POST_LINK_CMD  = %{post_link_cmd}
 
 # The primary target
-all: libs cli tests docs
+all: libs cli docs tests
 
 # Executable targets
 CLI           = %{cli_exe}
@@ -38,11 +38,14 @@ LIBRARIES     = %{library_targets}
 cli: $(CLI)
 tests: $(TEST)
 libs: $(LIBRARIES)
+docs: %{doc_stamp_file}
 
 # Misc targets
 
-docs:
-%{build_doc_commands}
+.PHONY = all cli libs tests docs clean distclean
+
+%{doc_stamp_file}: %{doc_dir}/manual/*.rst
+	$(PYTHON_EXE) $(SCRIPTS_DIR)/build_docs.py --build-dir="%{build_dir}"
 
 clean:
 	$(PYTHON_EXE) $(SCRIPTS_DIR)/cleanup.py --build-dir="%{build_dir}"
@@ -60,14 +63,7 @@ CLIOBJS       = %{cli_objs}
 
 TESTOBJS      = %{test_objs}
 
-# Build Commands
-%{lib_build_cmds}
-
-%{cli_build_cmds}
-
-%{test_build_cmds}
-
-# Library targets
+# Executable targets
 
 $(CLI): $(LIBRARIES) $(CLIOBJS)
 	$(EXE_LINK_CMD) $(LDFLAGS) $(CLIOBJS) $(EXE_LINKS_TO) %{output_to_exe}$@
@@ -77,6 +73,22 @@ $(TEST): $(LIBRARIES) $(TESTOBJS)
 	$(EXE_LINK_CMD) $(LDFLAGS) $(TESTOBJS) $(EXE_LINKS_TO) %{output_to_exe}$@
 	$(POST_LINK_CMD)
 
+%{if build_fuzzers}
+
+FUZZERS=%{fuzzer_bin}
+
+fuzzers: libs $(FUZZERS)
+
+fuzzer_corpus:
+	git clone --depth=1 https://github.com/randombit/crypto-corpus.git fuzzer_corpus
+
+fuzzer_corpus_zip: fuzzer_corpus
+	./src/scripts/create_corpus_zip.py fuzzer_corpus %{fuzzobj_dir}
+
+%{endif}
+
+# Library targets
+
 %{if build_static_lib}
 
 %{out_dir}/%{static_lib_name}: $(LIBOBJS)
@@ -100,18 +112,13 @@ $(TEST): $(LIBRARIES) $(TESTOBJS)
 
 %{endif}
 
-%{if build_fuzzers}
-
-%{fuzzer_build_cmds}
-
-FUZZERS=%{fuzzer_bin}
-
-fuzzers: libs $(FUZZERS)
+# Build Commands
+%{lib_build_cmds}
 
-fuzzer_corpus:
-	git clone --depth=1 https://github.com/randombit/crypto-corpus.git fuzzer_corpus
+%{cli_build_cmds}
 
-fuzzer_corpus_zip: fuzzer_corpus
-	./src/scripts/create_corpus_zip.py fuzzer_corpus %{fuzzobj_dir}
+%{test_build_cmds}
 
+%{if build_fuzzers}
+%{fuzzer_build_cmds}
 %{endif}
diff --git a/src/scripts/build_docs.py b/src/scripts/build_docs.py
new file mode 100755
index 000000000..b30e3d3e1
--- /dev/null
+++ b/src/scripts/build_docs.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+
+"""
+Botan doc generation script
+
+(C) 2014,2015,2017 Jack Lloyd
+
+Botan is released under the Simplified BSD License (see license.txt)
+"""
+
+import sys
+import optparse # pylint: disable=deprecated-module
+import subprocess
+import shutil
+import logging
+import json
+import os
+
+def get_concurrency():
+    """
+    Get default concurrency level of build
+    """
+    def_concurrency = 2
+
+    try:
+        import multiprocessing
+        return max(def_concurrency, multiprocessing.cpu_count())
+    except ImportError:
+        return def_concurrency
+
+def touch(fname):
+    try:
+        os.utime(fname, None)
+    except OSError:
+        open(fname, 'a').close()
+
+def copy_files(src_dir, dest_dir):
+    for f in os.listdir(src_dir):
+        src_file = os.path.join(src_dir, f)
+        dest_file = os.path.join(dest_dir, f)
+        logging.debug("Copying from %s to %s", src_file, dest_file)
+        shutil.copyfile(src_file, dest_file)
+
+def run_and_check(cmd_line):
+
+    logging.debug("Executing %s", ' '.join(cmd_line))
+
+    proc = subprocess.Popen(cmd_line,
+                            close_fds=True)
+
+    (stdout, stderr) = proc.communicate()
+
+    if proc.returncode != 0:
+        logging.error("Error running %s", ' '.join(cmd_line))
+        sys.exit(1)
+
+
+def parse_options(args):
+    parser = optparse.OptionParser()
+
+    parser.add_option('--verbose', action='store_true', default=False,
+                      help='Show debug messages')
+    parser.add_option('--quiet', action='store_true', default=False,
+                      help='Show only warnings and errors')
+
+    parser.add_option('--build-dir', metavar='DIR', default='build',
+                      help='Location of build output (default \'%default\')')
+    parser.add_option('--dry-run', default=False, action='store_true',
+                      help='Just display what would be done')
+
+    (options, args) = parser.parse_args(args)
+
+    if len(args) > 1:
+        logging.error("Unknown arguments")
+        return None
+
+    def log_level():
+        if options.verbose:
+            return logging.DEBUG
+        if options.quiet:
+            return logging.WARNING
+        return logging.INFO
+
+    logging.getLogger().setLevel(log_level())
+
+    return options
+
+
+def main(args=None):
+    if args is None:
+        args = sys.argv
+
+    logging.basicConfig(stream=sys.stdout,
+                        format='%(levelname) 7s: %(message)s')
+
+    options = parse_options(args)
+
+    if options is None:
+        return 1
+
+    with open(os.path.join(options.build_dir, 'build_config.json')) as f:
+        cfg = json.load(f)
+
+    with_docs = bool(cfg['with_documentation'])
+    with_sphinx = bool(cfg['with_sphinx'])
+    with_doxygen = bool(cfg['with_doxygen'])
+
+    doc_stamp_file = cfg['doc_stamp_file']
+
+    manual_src = os.path.join(cfg['doc_dir'], 'manual')
+    manual_output = os.path.join(cfg['doc_output_dir'], 'manual')
+
+    if with_docs is False:
+        logging.debug('Documentation build disabled')
+        return 0
+
+    cmds = []
+
+    if with_doxygen:
+        cmds.append(['doxygen', os.path.join(cfg['build_dir'], 'botan.doxy')])
+
+    if with_sphinx:
+        cmds.append(['sphinx-build', '-q', '-b', 'html', '-c', cfg['sphinx_config_dir'],
+                     '-j', str(get_concurrency()), manual_src, manual_output])
+    else:
+        # otherwise just copy it
+        cmds.append(['cp', manual_src, manual_output])
+
+    cmds.append(['touch', doc_stamp_file])
+
+    for cmd in cmds:
+        if options.dry_run:
+            print(' '.join(cmd))
+        else:
+            if cmd[0] == 'cp':
+                assert len(cmd) == 3
+                copy_files(cmd[1], cmd[2])
+            elif cmd[0] == 'touch':
+                assert len(cmd) == 2
+                touch(cmd[1])
+            else:
+                run_and_check(cmd)
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
index 74c6a7bfb..c81c86795 100755
--- a/src/scripts/ci_build.py
+++ b/src/scripts/ci_build.py
@@ -364,6 +364,7 @@ def main(args=None):
             'src/scripts/ci_build.py',
             'src/scripts/install.py',
             'src/scripts/cleanup.py',
+            'src/scripts/build_docs.py',
             'src/scripts/website.py',
             'src/scripts/python_unittests.py',
             'src/scripts/python_unittests_unix.py']
diff --git a/src/scripts/cleanup.py b/src/scripts/cleanup.py
index d246f766c..71c59609e 100755
--- a/src/scripts/cleanup.py
+++ b/src/scripts/cleanup.py
@@ -69,8 +69,9 @@ def main(args=None):
     build_dir = options.build_dir
 
     if os.access(build_dir, os.X_OK) != True:
-        logging.error("Unable to access build directory")
-        return 1
+        logging.debug('No build directory found')
+        # No build dir: clean enough!
+        return 0
 
     build_config_path = os.path.join(build_dir, 'build_config.json')
     build_config_str = None
@@ -86,8 +87,6 @@ def main(args=None):
 
     build_config = json.loads(build_config_str)
 
-    #print(json.dumps(build_config, sort_keys=True, indent=3))
-
     if options.distclean:
         build_dir = build_config['build_dir']
         remove_file(build_config['makefile_path'])
@@ -97,7 +96,12 @@ def main(args=None):
             dir_path = build_config[dir_type]
             remove_all_in_dir(dir_path)
 
-        shutil.rmtree(build_config['doc_output_dir'])
+        try:
+            shutil.rmtree(build_config['doc_output_dir'])
+        except OSError as e:
+            pass
+
+        #remove_file(build_config['doc_stamp_file'])
 
     remove_file(build_config['cli_exe'])
     remove_file(build_config['test_exe'])