Improvements to distribution script

All timestamps are set based on the release date so we create the
exact same archive each time regardless of when the script is run.
Modulo some timezone issue anyway. This only involved one horrible hack.

Partial port to Python3 but still doesn't work there.

1 files changed, 135 insertions, 89 deletions

diff --git a/src/scripts/dist.py b/src/scripts/dist.py
index 7f666904c..855d93703 100755
--- a/src/scripts/dist.py
+++ b/src/scripts/dist.py
@@ -3,11 +3,12 @@
 """
 Release script for botan (https://botan.randombit.net/)
 
-(C) 2011, 2012, 2013, 2015, 2016 Jack Lloyd
+(C) 2011,2012,2013,2015,2016,2017 Jack Lloyd
 
 Botan is released under the Simplified BSD License (see license.txt)
 """
 
+import time
 import errno
 import logging
 import optparse
@@ -15,11 +16,18 @@ import os
 import shutil
 import subprocess
 import sys
-import tarfile
 import datetime
 import hashlib
 import re
-import StringIO
+import tarfile
+
+# This is horrible, but there is no way to override tarfile's use of time.time
+# in setting the gzip header timestamp, which breaks deterministic archives
+
+def null_time():
+    return 0
+time.time = null_time
+
 
 def check_subprocess_results(subproc, name):
     (stdout, stderr) = subproc.communicate()
@@ -43,7 +51,7 @@ def run_git(args):
     return check_subprocess_results(proc, 'git')
 
 def maybe_gpg(val):
-    # TODO: verify signatures
+    val = val.decode('ascii')
     if 'BEGIN PGP SIGNATURE' in val:
         return val.split('\n')[-2]
     else:
@@ -52,7 +60,7 @@ def maybe_gpg(val):
 def datestamp(tag):
     ts = maybe_gpg(run_git(['show', '--no-patch', '--format=%ai', tag]))
 
-    ts_matcher = re.compile('^(\d{4})-(\d{2})-(\d{2}) \d{2}:\d{2}:\d{2} .*')
+    ts_matcher = re.compile(r'^(\d{4})-(\d{2})-(\d{2}) \d{2}:\d{2}:\d{2} .*')
     match = ts_matcher.match(ts)
 
     if match is None:
@@ -66,10 +74,18 @@ def revision_of(tag):
 
 def extract_revision(revision, to):
     tar_val = run_git(['archive', '--format=tar', '--prefix=%s/' % (to), revision])
-    tar_f = tarfile.open(fileobj=StringIO.StringIO(tar_val))
-    tar_f.extractall()
 
-def gpg_sign(keyid, passphrase_file, files, detached = True):
+    if sys.version_info.major == 3:
+        import io
+        tar_f = tarfile.open(fileobj=io.BytesIO(tar_val))
+        tar_f.extractall()
+    else:
+        import StringIO
+        tar_f = tarfile.open(fileobj=StringIO.StringIO(tar_val))
+        tar_f.extractall()
+
+
+def gpg_sign(keyid, passphrase_file, files, detached=True):
 
     options = ['--armor', '--detach-sign'] if detached else ['--clearsign']
 
@@ -133,7 +149,58 @@ def remove_file_if_exists(fspath):
         if e.errno != errno.ENOENT:
             raise
 
-def main(args = None):
+def rewrite_version_file(version_file, target_version, rev_id, rel_date):
+
+    version_file_name = os.path.basename(version_file)
+
+    contents = open(version_file).readlines()
+
+    version_re = re.compile('release_(major|minor|patch) = ([0-9]+)')
+
+    def content_rewriter():
+        for line in contents:
+
+            if target_version != 'HEAD':
+                match = version_re.match(line)
+                if match:
+                    name_to_idx = {
+                        'major': 0,
+                        'minor': 1,
+                        'patch': 2
+                    }
+                    version_parts = target_version.split('.')
+                    assert len(version_parts) == 3
+                    in_tag = int(version_parts[name_to_idx[match.group(1)]])
+                    in_file = int(match.group(2))
+
+                    if in_tag != in_file:
+                        raise Exception('Version number part "%s" in %s does not match tag %s' %
+                                        (match.group(1), version_file_name, target_version))
+
+            if line == 'release_vc_rev = None\n':
+                yield 'release_vc_rev = \'git:%s\'\n' % (rev_id)
+            elif line == 'release_datestamp = 0\n':
+                yield 'release_datestamp = %d\n' % (rel_date)
+            elif line == "release_type = \'unreleased\'\n":
+                if target_version == 'HEAD':
+                    yield "release_type = 'snapshot'\n"
+                else:
+                    yield "release_type = 'release'\n"
+            else:
+                yield line
+
+    open(version_file, 'w').write(''.join(list(content_rewriter())))
+
+def rel_date_to_epoch(rel_date):
+    rel_str = str(rel_date)
+    year = int(rel_str[0:4])
+    month = int(rel_str[5:6])
+    day = int(rel_str[7:8])
+
+    dt = datetime.datetime(year, month, day, 6, 0, 0)
+    return (dt - datetime.datetime(1970, 1, 1)).total_seconds()
+
+def main(args=None):
     if args is None:
         args = sys.argv[1:]
 
@@ -146,11 +213,11 @@ def main(args = None):
             return logging.ERROR
         return logging.INFO
 
-    logging.basicConfig(stream = sys.stderr,
-                        format = '%(levelname) 7s: %(message)s',
-                        level = log_level())
+    logging.basicConfig(stream=sys.stderr,
+                        format='%(levelname) 7s: %(message)s',
+                        level=log_level())
 
-    if len(args) == 0 or len(args) > 2:
+    if len(args) != 1 and len(args) != 2:
         logging.error('Usage error, try --help')
         return 1
 
@@ -168,11 +235,11 @@ def main(args = None):
         try:
             logging.info('Creating release for version %s' % (args[0]))
 
-            (major,minor,patch) = map(int, args[0].split('.'))
+            (major, minor, patch) = map(int, args[0].split('.'))
 
-            assert args[0] == '%d.%d.%d' % (major,minor,patch)
+            assert args[0] == '%d.%d.%d' % (major, minor, patch)
             target_version = args[0]
-        except:
+        except ValueError as e:
             logging.error('Invalid version number %s' % (args[0]))
             return 1
     else:
@@ -181,7 +248,7 @@ def main(args = None):
 
     def output_name(args):
         if is_snapshot:
-            datestamp = datetime.date.today().isoformat().replace('-', '')
+            today = datetime.date.today().isoformat().replace('-', '')
 
             def snapshot_name(branch):
                 if branch == 'master':
@@ -189,7 +256,7 @@ def main(args = None):
                 else:
                     return branch
 
-            return 'botan-%s-snapshot-%s' % (snapshot_name(args[1]), datestamp)
+            return 'botan-%s-snapshot-%s' % (snapshot_name(args[1]), today)
         else:
             return 'Botan-' + args[0]
 
@@ -216,52 +283,19 @@ def main(args = None):
 
     extract_revision(rev_id, output_basename)
 
-    version_file = os.path.join(output_basename, 'botan_version.py')
+    version_file = None
+
+    for possible_version_file in ['version.txt', 'botan_version.py']:
+        full_path = os.path.join(output_basename, possible_version_file)
+        if os.access(full_path, os.R_OK):
+            version_file = full_path
+            break
 
-    if os.access(version_file, os.R_OK) == False:
+    if not os.access(version_file, os.R_OK):
         logging.error('Cannot read %s' % (version_file))
         return 2
 
-    # rewrite botan_version.py
-
-    contents = open(version_file).readlines()
-
-    version_re = re.compile('release_(major|minor|patch) = ([0-9]+)')
-    version_parts = target_version.split('.')
-    assert len(version_parts) == 3
-
-    def content_rewriter():
-        for line in contents:
-
-            if target_version != 'HEAD':
-                match = version_re.match(line)
-                if match:
-                    name_to_idx = {
-                        'major': 0,
-                        'minor': 1,
-                        'patch': 2
-                    }
-                    in_tag = int(version_parts[name_to_idx[match.group(1)]])
-                    in_file = int(match.group(2))
-
-                    if in_tag != in_file:
-                        logging.error('Version number part "%s" in botan_version.py does not match tag %s' %
-                                      (match.group(1), target_version))
-                        raise Exception('Bad botan_version.py')
-
-            if line == 'release_vc_rev = None\n':
-                yield 'release_vc_rev = \'git:%s\'\n' % (rev_id)
-            elif line == 'release_datestamp = 0\n':
-                yield 'release_datestamp = %d\n' % (rel_date)
-            elif line == "release_type = \'unreleased\'\n":
-                if args[0] == 'snapshot':
-                    yield "release_type = 'snapshot'\n"
-                else:
-                    yield "release_type = 'released'\n"
-            else:
-                yield line
-
-    open(version_file, 'w').write(''.join(list(content_rewriter())))
+    rewrite_version_file(version_file, target_version, rev_id, rel_date)
 
     try:
         os.makedirs(options.output_dir)
@@ -278,39 +312,51 @@ def main(args = None):
     if options.write_hash_file != None:
         hash_file = open(options.write_hash_file, 'w')
 
-    for archive in archives:
-        logging.debug('Writing archive type "%s"' % (archive))
-
-        output_archive = output_basename + '.' + archive
-
-        remove_file_if_exists(output_archive)
-        remove_file_if_exists(output_archive + '.asc')
-
-        if archive in ['tgz', 'tbz']:
+    rel_epoch = rel_date_to_epoch(rel_date)
 
-            def write_mode():
-                if archive == 'tgz':
-                    return 'w:gz'
-                elif archive == 'tbz':
-                    return 'w:bz2'
+    for archive_type in archives:
+        if archive_type not in ['tar', 'tgz', 'tbz']:
+            raise Exception('Unknown archive type "%s"' % (archive_type))
 
-            archive = tarfile.open(output_archive, write_mode())
+        output_archive = output_basename + '.' + archive_type
 
-            all_files = []
-            for (curdir,_,files) in os.walk(output_basename):
-                all_files += [os.path.join(curdir, f) for f in files]
-            all_files.sort()
+        logging.info('Writing archive "%s"' % (output_archive))
 
-            for f in all_files:
-                archive.add(f)
-            archive.close()
+        remove_file_if_exists(output_archive)
+        remove_file_if_exists(output_archive + '.asc')
 
-            if hash_file != None:
-                sha256 = hashlib.new('sha256')
-                sha256.update(open(output_archive).read())
-                hash_file.write("%s  %s\n" % (sha256.hexdigest(), output_archive))
-        else:
-            raise Exception('Unknown archive type "%s"' % (archive))
+        all_files = []
+        for (curdir, _, files) in os.walk(output_basename):
+            all_files += [os.path.join(curdir, f) for f in files]
+        all_files.sort()
+
+        def write_mode(archive_type):
+            if archive_type == 'tgz':
+                return 'w:gz'
+            elif archive_type == 'tbz':
+                return 'w:bz2'
+            elif archive_type == 'tar':
+                return 'w'
+
+        archive = tarfile.open(output_archive, write_mode(archive_type))
+
+        for f in all_files:
+            tarinfo = archive.gettarinfo(f)
+            tarinfo.uid = 500
+            tarinfo.gid = 500
+            tarinfo.uname = "botan"
+            tarinfo.gname = "botan"
+            tarinfo.mtime = rel_epoch
+            archive.addfile(tarinfo, open(f))
+        archive.close()
+
+        sha256 = hashlib.new('sha256')
+        sha256.update(open(output_archive).read())
+        archive_hash = sha256.hexdigest().upper()
+
+        logging.info('SHA-256(%s) = %s' % (output_archive, archive_hash))
+        if hash_file != None:
+            hash_file.write("%s  %s\n" % (archive_hash, output_archive))
 
         output_files.append(output_archive)
 
@@ -344,5 +390,5 @@ if __name__ == '__main__':
     except Exception as e:
         logging.error(e)
         import traceback
-        logging.debug(traceback.format_exc())
+        logging.error(traceback.format_exc())
         sys.exit(1)