Build: split fetch into df-fetch and df-verify

- moved common python code to lib/hb_distfile.py
- beautified tmpfile creation
- added stack-style resource management to df-fetch
- fixed contrib assumptions about single URL

7 files changed, 450 insertions, 308 deletions

diff --git a/make/configure.py b/make/configure.py
index 01274dc3b..e27b477fa 100644
--- a/make/configure.py
+++ b/make/configure.py
@@ -14,7 +14,9 @@ import json
 import optparse
 import os
 import platform
+import random
 import re
+import string
 import subprocess
 import sys
 import time
@@ -128,7 +130,7 @@ class Configure( object ):
         dir = os.path.dirname( args[0] )
         if len(args) > 1 and args[1].find('w') != -1:
             self.mkdirs( dir )
-        m = re.match( '^(.*)\.tmp$', args[0] )
+        m = re.match( '^(.*)\.tmp\..{8}$', args[0] )
         if m:
             self.infof( 'write: %s\n', m.group(1) )
         else:
@@ -210,6 +212,10 @@ class Configure( object ):
         if os.path.abspath( self.src_dir ) == os.path.abspath( self.build_dir ):
             self.build_dir = os.path.join( self.build_dir, 'build' )
 
+    ## generate a temporary filename - not worried about race conditions
+    def mktmpname( self, filename ):
+        return filename + '.tmp.' + ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
+
 ###############################################################################
 ##
 ## abstract action
@@ -1153,7 +1159,7 @@ class ConfigDocument:
         else:
             raise ValueError, 'unknown file type: ' + type
 
-        ftmp  = fname + '.tmp'
+        ftmp = cfg.mktmpname(fname)
         try:
             try:
                 file = cfg.open( ftmp, 'w' )
@@ -1177,16 +1183,16 @@ class ConfigDocument:
 
 ###############################################################################
 
-def encodeFetchConfig():
-    fname = 'fetch.cfg'
-    ftmp = fname + '.tmp'
-    data = [
-        options.verbose_fetch,
-        options.disable_fetch,
-        options.disable_fetch_md5,
-        options.accept_fetch_url,
-        options.deny_fetch_url,
-    ]
+def encodeDistfileConfig():
+    fname = 'distfile.cfg'
+    ftmp = cfg.mktmpname(fname)
+    data = {
+        'disable-fetch':  options.disable_df_fetch,
+        'disable-verify': options.disable_df_verify,
+        'verbosity':      options.df_verbosity,
+        'accept-url':     options.df_accept_url,
+        'deny-url':       options.df_deny_url,
+    }
     try:
         try:
             file = cfg.open( ftmp, 'w' )
@@ -1252,13 +1258,13 @@ def createCLI():
     cli.add_option( '--force', default=False, action='store_true', help='overwrite existing build config' )
     cli.add_option( '--verbose', default=False, action='store_true', help='increase verbosity' )
 
-    ## add fetch options
-    grp = OptionGroup( cli, 'Fetch Options' )
-    grp.add_option( '--verbose-fetch', default=False, action='store_true', help='increase fetch verbosity' )
-    grp.add_option( '--disable-fetch', default=False, action='store_true', help='disable automatic downloads of 3rd-party distributions' )
-    grp.add_option( '--disable-fetch-md5', default=False, action='store_true', help='disable MD5 data error detection' )
-    grp.add_option( '--accept-fetch-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern' )
-    grp.add_option( '--deny-fetch-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern' )
+    ## add distfile options
+    grp = OptionGroup( cli, 'Distfile Options' )
+    grp.add_option( '--disable-df-fetch', default=False, action='store_true', help='disable distfile downloads' )
+    grp.add_option( '--disable-df-verify', default=False, action='store_true', help='disable distfile data verification' )
+    grp.add_option( '--df-verbose', default=1, action='count', dest='df_verbosity', help='increase distfile tools verbosity' )
+    grp.add_option( '--df-accept-url', default=[], action='append', metavar='SPEC', help='accept URLs matching regex pattern' )
+    grp.add_option( '--df-deny-url', default=[], action='append', metavar='SPEC', help='deny URLs matching regex pattern' )
     cli.add_option_group( grp )
 
     ## add install options
@@ -1929,7 +1935,7 @@ int main ()
     ## perform
     doc.write( 'make' )
     doc.write( 'm4' )
-    encodeFetchConfig()
+    encodeDistfileConfig()
 
     if options.launch:
         Launcher( targets )
diff --git a/make/df-fetch.py b/make/df-fetch.py
new file mode 100644
index 000000000..eeb520c7a
--- /dev/null
+++ b/make/df-fetch.py
@@ -0,0 +1,203 @@
+###############################################################################
+##
+## Coded for minimum version of Python 2.7 .
+##
+## Python3 is incompatible.
+##
+## Authors: konablend
+##
+###############################################################################
+
+import hashlib
+import re
+import os
+import sys
+import urllib2
+
+sys.dont_write_bytecode = True
+sys.path.insert(0, os.path.join(sys.path[0], 'lib'))
+import hb_distfile
+
+###############################################################################
+
+## simple structure object
+class Struct(object):
+    pass
+
+## track resources and ensure cleanup
+##
+## - items are lambdas accepting no args
+## - item order of insertion is important
+## - cleanup will run in reverse order of insertion
+## - item update does not effect order
+##
+class Ensure(object):
+    def __init__(self):
+        super(Ensure, self).__setattr__('_items', [])
+
+    def __delattr__(self, key):
+        if key in self.__dict__:
+            self._items.remove(self.__dict__[key])
+        super(Ensure, self).__delattr__(key)
+
+    def __setattr__(self, key, value):
+        if not key in self.__dict__:
+            self._items.insert(0, value)
+        super(Ensure, self).__setattr__(key, value)
+
+    def run(self):
+        for item in self._items:
+            try:
+                item()
+            except Exception:
+                pass
+
+###############################################################################
+
+class Tool(hb_distfile.Tool):
+    def __init__(self):
+        super(Tool, self).__init__()
+        self.parser.prog = self.name
+        self.parser.usage = '%prog [OPTIONS] URL...'
+        self.parser.description = 'Fetch and verify distfile data integrity.'
+        self.parser.add_option('--disable', default=False, action='store_true', help='do nothing and exit without error')
+        self.parser.add_option('--md5', default=None, action='store', metavar='HASH', help='verify MD5 HASH against data')
+        self.parser.add_option('--accept-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern')
+        self.parser.add_option('--deny-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern')
+        self.parser.add_option('--output', default=None, action='store', metavar='FILE', help='write to FILE')
+        self._parse()
+
+    def _load_config2(self, parser, data):
+        parser.values.disable    = data['disable-fetch']
+        parser.values.accept_url = data['accept-url']
+        parser.values.deny_url   = data['deny-url']
+
+    def _run(self, error):
+        if self.options.disable:
+            self.infof('%s disabled; stop.\n' % self.name)
+            sys.exit(0)
+        if len(self.args) < 1:
+            self.parser.print_usage()
+            sys.exit(1)
+        ## create URL objects and keep active
+        urls = []
+        i = 0
+        for arg in self.args:
+            url = URL(arg, i)
+            if url.active:
+                urls.append(url)
+            i += 1
+        ## try each URL until first success
+        error.op = 'download'
+        while urls:
+            url = urls.pop(0)
+            try:
+                url.download(error)
+                break
+            except Exception, x:
+                ## propagate exception if no remaining urls
+                if not urls:
+                    raise
+                self.errln('%s failure; %s' % (error.op,x))
+
+    def run(self):
+        error = hb_distfile.ToolError('run')
+        try:
+            self._run(error)
+        except Exception, x:
+            self.debug_exception()
+            self.errln('%s failure; %s' % (error.op,x), exit=1)
+
+###############################################################################
+
+class URL(object):
+    def __init__(self, url, index):
+        self.index = index
+        self.url = url
+        self.active = True
+        self.rule = 'none'
+        self._accept()
+        self._deny()
+        tool.verbosef('URL[%d]: %s\n' % (self.index,self.url))
+        tool.verbosef('  active: %s\n' % ('yes' if self.active else 'no'))
+        tool.verbosef('  rule:   %s\n' % (self.rule))
+
+    def _accept(self):
+        if not tool.options.accept_url:
+            return
+        index = 0
+        for spec in tool.options.accept_url:
+            if re.search(spec, self.url):
+                self.rule = 'via accept rule %d: %s' % (index,spec)
+                return
+            index += 1
+        self.active = False
+        self.rule = 'no matching accept rule'
+
+    def _deny(self):
+        index = 0
+        for spec in tool.options.deny_url:
+            if re.search(spec, self.url):
+                self.active = False
+                self.rule = 'via deny rule %d: %s' % (index,spec)
+                return
+            index += 1
+
+    def _download(self, error, ensure):
+        filename = tool.options.output
+        hasher = hashlib.md5()
+        if filename:
+            tool.infof('downloading %s to %s\n' % (self.url,filename))
+            ftmp = tool.mktmpname(filename)
+            hout = open(ftmp, 'w')
+            ensure.unlink_ftmp = lambda: os.unlink(ftmp)
+            ensure.close_hout = lambda: hout.close()
+        else:
+            tool.infof('downloading %s\n' % (self.url))
+        hin = urllib2.urlopen(self.url, None, 30)
+        ensure.close_hin = lambda: hin.close()
+        info = hin.info()
+        try:
+            content_length = int(info.getheader('Content-Length'))
+        except:
+            content_length = None
+        data_total = 0
+        while True:
+            data = hin.read(65536)
+            if not data:
+                break
+            if filename:
+                hout.write(data)
+            hasher.update(data)
+            data_total += len(data)
+        if content_length and content_length != data_total:
+            raise error('expected %d bytes, got %d bytes' % (content_length,data_total))
+        s = 'downloaded %d bytes' % data_total
+        if filename:
+            s += '; MD5 (%s) = %s' % (filename,hasher.hexdigest())
+        else:
+            s += '; MD5 = %s' % (hasher.hexdigest())
+        if tool.options.md5:
+            md5_pass = tool.options.md5 == hasher.hexdigest()
+            s += ' (%s)' % ('pass' if md5_pass else 'fail; expecting %s' % tool.options.md5)
+        tool.infof('%s\n' % s)
+        if filename and tool.options.md5:
+            if md5_pass:
+                if os.access(filename, os.F_OK) and not os.access(filename, os.W_OK):
+                    raise error("permission denied: '%s'" % filename)
+            else:
+                raise error("expected MD5 hash '%s', got '%s'" % (tool.options.md5, hasher.hexdigest()))
+            os.rename(ftmp,filename)
+            del ensure.unlink_ftmp
+
+    def download(self, error):
+        ensure = Ensure()
+        try:
+            self._download(error, ensure)
+        finally:
+            ensure.run()
+
+###############################################################################
+
+tool = Tool()
+tool.run()
diff --git a/make/df-verify.py b/make/df-verify.py
new file mode 100644
index 000000000..2d56afaa5
--- /dev/null
+++ b/make/df-verify.py
@@ -0,0 +1,91 @@
+###############################################################################
+##
+## Coded for minimum version of Python 2.7 .
+##
+## Python3 is incompatible.
+##
+## Authors: konablend
+##
+###############################################################################
+
+import hashlib
+import os
+import sys
+
+sys.dont_write_bytecode = True
+sys.path.insert(0, os.path.join(sys.path[0], 'lib'))
+import hb_distfile
+
+###############################################################################
+
+## simple structure object
+class Struct(object):
+    pass
+
+###############################################################################
+
+class Tool(hb_distfile.Tool):
+    def __init__(self):
+        super(Tool, self).__init__()
+        self.parser.prog = self.name
+        self.parser.usage = '%prog [OPTIONS] FILE'
+        self.parser.description = 'Verify distfile data integrity.'
+        self.parser.add_option('--disable', default=False, action='store_true', help='do nothing and exit without error')
+        self.parser.add_option('--md5', default=None, action='store', metavar='HASH', help='verify MD5 HASH against data')
+        self._parse()
+
+    def _load_config2(self, parser, data):
+        parser.values.disable = data['disable-verify']
+
+    def _scan(self, filename):
+        self.verbosef('scanning %s\n' % filename)
+        hasher = hashlib.md5()
+        with open(filename, 'r') as o:
+            data_total = 0
+            while True:
+                data = o.read(65536)
+                if not data:
+                    break
+                hasher.update(data)
+                data_total += len(data)
+        self.verbosef('scanned %d bytes\n' % data_total)
+        r = Struct()
+        r.md5 = hasher.hexdigest()
+        r.size = data_total
+        return r
+
+    def _verify(self, filename):
+        r = Struct()
+        r.scan = self._scan(filename)
+        r.status = self.options.md5 == r.scan.md5
+        return r
+
+    def _run(self, error):
+        if self.options.disable:
+            self.infof('%s disabled; stop.\n' % self.name)
+            sys.exit(0)
+        if len(self.args) != 1:
+            self.parser.print_usage()
+            sys.exit(1)
+        filename = self.args[0]
+        if self.options.md5:
+            error.op = 'verify'
+            r = self._verify(filename)
+            self.infof('MD5 (%s) = %s (%s)\n', filename, r.scan.md5, 'pass' if r.status else 'fail; expecting %s' % self.options.md5)
+        else:
+            error.op = 'scan'
+            r = self._scan(filename)
+            self.infof('MD5 (%s) = %s (%d bytes)\n', filename, r.md5, r.size)
+
+    def run(self):
+        error = hb_distfile.ToolError('run')
+        try:
+            self._run(error)
+        except Exception, x:
+            self.debug_exception()
+            self.errln('%s failure; %s' % (error.op,x), exit=1)
+
+###############################################################################
+
+tool = Tool()
+tool.run()
diff --git a/make/fetch.py b/make/fetch.py
deleted file mode 100644
index 33bc277f9..000000000
--- a/make/fetch.py
+++ /dev/null
@@ -1,268 +0,0 @@
-###############################################################################
-##
-## This script is coded for minimum version of Python 2.7 .
-##
-## Python3 is incompatible.
-##
-## Authors: konablend
-##
-###############################################################################
-
-import errno
-import hashlib
-import json
-import os
-import re
-import tempfile
-import traceback
-import urllib2
-
-from optparse import OptionGroup
-from optparse import OptionParser
-from sys import stderr
-from sys import stdout
-from urlparse import urlparse
-
-###############################################################################
-
-class Fetch(object):
-    def __init__(self, options, urls):
-        if options.disable:
-            self.errln('fetching files from the network is disabled.')
-        self.options = options
-        self.urls = urls
-        if len(self.urls) > 1:
-            self.infof('\n')
-        self.verbosef('OPTIONS:\n')
-        self.verbosef('  disable:        %s\n' % self.options.disable)
-        self.verbosef('  disable_md5:    %s\n' % self.options.disable_md5)
-        self.verbosef('  config:         %s\n' % self.options.config)
-        self.verbosef('  md5:            %s\n' % self.options.md5)
-        self.verbosef('  output_dir:     %s\n' % self.options.output_dir)
-        index = 0
-        for spec in self.options.accept_url:
-            self.verbosef('  accept_url[%d]:  %s\n' % (index,spec))
-            index += 1
-        if not self.options.accept_url:
-            self.verbosef('  accept_url:     %s\n' % None)
-        index = 0
-        for spec in self.options.deny_url:
-            self.verbosef('  deny_url[%d]:    %s\n' % (index,spec))
-            index += 1
-        if not self.options.deny_url:
-            self.verbosef('  deny_url:       %s\n' % None)
-
-    def run(self):
-        if not self.urls:
-            self.errln('no URLs specified')
-        files = []
-        for url in self.urls:
-            files.append(self._process_url(url))
-        index = 0
-        for file in files:
-            file.dump(index)
-            index += 1
-        canon = files[0].filename
-        for file in files:
-            if file.filename != canon:
-                self.errln('URL basename is not consistent')
-        scan = os.access(canon, os.F_OK)
-        for file in files:
-            if file.run(scan):
-                return
-        self.errln('%s failed.' % ('scan' if scan else 'download'))
-
-    def errln(self, format, *args):
-        s = (format % args)
-        if re.match( '^.*[!?:;.]$', s ):
-            stderr.write('ERROR: %s fetch stop.\n' % (s))
-        else:
-            stderr.write('ERROR: %s; fetch stop.\n' % (s))
-        exit(1)
-
-    def warnln(self, format, *args):
-        s = (format % args)
-        if re.match( '^.*[!?:;.]$', s ):
-            stderr.write('WARNING: %s fetch continuing.\n' % (s))
-        else:
-            stderr.write('WARNING: %s; fetch continuing.\n' % (s))
-
-    def infof(self, format, *args):
-        stdout.write(format % args)
-
-    def verbosef(self, format, *args):
-        if self.options.verbose:
-            stdout.write(format % args)
-
-    def _process_url(self, url):
-        props = {}
-        index = 0
-        while True:
-            ## optional per-URL properties
-            ## [key=value][...]URL
-            m = re.match('(\[(\w+)=([^]]+)\])?(.*)', url[index:])
-            if not m.group(1):
-                break
-            props[m.group(2)] = m.group(3)
-            index += len(m.group(1))
-        return File(url[index:], **props)
-
-###############################################################################
-
-class File(object):
-    def __init__(self, url, **kwargs):
-        self.url = url
-        self.props = kwargs # not currently used
-        self.filename = os.path.join(fetch.options.output_dir,os.path.basename(urlparse(self.url).path))
-        self.active = True
-        self.active_descr = 'default'
-        self._accept()
-        self._deny()
-
-    def dump(self, index):
-        fetch.verbosef('URL[%d]: %s\n' % (index,self.url))
-        fetch.verbosef('  filename: %s\n' % self.filename)
-        fetch.verbosef('  active: %s (%s)\n' % ('yes' if self.active else 'no',self.active_descr))
-
-    def run(self, scan):
-        if not self.active:
-            return False
-        try:
-            if (self._scan() if scan else self._download()):
-                return True
-        except Exception, x:
-            if fetch.options.verbose:
-                traceback.print_exc()
-            fetch.warnln('%s' % x)
-        return False
-
-    def _accept(self):
-        if not fetch.options.accept_url:
-            return
-        index = 0
-        for spec in fetch.options.accept_url:
-            if re.match(spec, self.url):
-                self.active_descr = 'via accept rule %d: %s' % (index,spec)
-                return
-            index += 1
-        self.active = False
-        self.active_descr = 'no matching spec'
-
-    def _deny(self):
-        index = 0
-        for spec in fetch.options.deny_url:
-            if re.match(spec, self.url):
-                self.active = False
-                self.active_descr = 'via deny rule %d: %s' % (index,spec)
-                return
-            index += 1
-
-    def _download(self):
-        fetch.infof('downloading %s to %s\n' % (self.url,self.filename))
-        hasher = hashlib.md5()
-        ftmp = self.filename + '.' + os.path.basename(tempfile.mktemp())
-        r = urllib2.urlopen(self.url, None, 30)
-        try:
-            o = open(ftmp, 'w')
-            info = r.info()
-            try:
-                content_length = int(info.getheader('Content-Length'))
-            except:
-                content_length = None
-            data_total = 0
-            while True:
-                data = r.read(65536)
-                if not data:
-                    break
-                o.write(data)
-                hasher.update(data)
-                data_total += len(data)
-        except:
-            os.unlink(ftmp)
-        finally:
-            for closeable in [r,o]:
-                try:
-                    if not closeable:
-                        continue
-                    closeable.close()
-                except:
-                    pass
-
-        if content_length and content_length != data_total:
-            fetch.warnln('expected %d bytes, got %d bytes' % (content_length,data_total))
-            os.unlink(ftmp)
-            return False
-
-        if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 == hasher.hexdigest():
-            s = ' (verified)'
-        else:
-            s = ''
-
-        fetch.infof("downloaded '%s' - %d bytes - %s%s\n" % (self.filename,data_total,hasher.hexdigest(),s))
-        if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 != hasher.hexdigest():
-            os.unlink(ftmp)
-            raise RuntimeError("expected MD5 hash '%s', got '%s'" % (fetch.options.md5, hasher.hexdigest()))
-
-        if os.access(self.filename, os.F_OK) and not os.access(self.filename, os.W_OK):
-            os.unlink(ftmp)
-            raise IOError(errno.EACCES, "Permission denied: '%s'" % self.filename)
-
-        try:
-            os.rename(ftmp,self.filename)
-        except:
-            os.unlink(ftmp)
-        return True
-
-    def _scan(self):
-        fetch.infof('scanning %s\n' % self.filename)
-        hasher = hashlib.md5()
-        try:
-            o = open(self.filename, 'r')
-            data_total = 0
-            while True:
-                data = o.read(65536)
-                if not data:
-                    break
-                hasher.update(data)
-                data_total += len(data)
-        finally:
-            o.close()
-
-        if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 == hasher.hexdigest():
-            s = ' (verified)'
-        else:
-            s = ''
-
-        fetch.infof("scanned '%s' - %d bytes - %s%s\n" % (self.filename,data_total,hasher.hexdigest(),s))
-        if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 != hasher.hexdigest():
-            raise RuntimeError("expected MD5 hash '%s', got '%s'" % (fetch.options.md5, hasher.hexdigest()))
-        return True
-
-###############################################################################
-
-def load_config(option, opt, value, parser):
-    with open(value, 'r') as file:
-        data = json.load(file)
-        parser.values.verbose     = data[0]
-        parser.values.disable     = data[1]
-        parser.values.disable_md5 = data[2]
-        parser.values.accept_url  = data[3]
-        parser.values.deny_url    = data[4]
-
-###############################################################################
-
-parser = OptionParser('usage: %prog [OPTIONS...] [URL...]')
-
-parser.description = 'Fetch files from the network.'
-
-parser.add_option('--verbose', default=False, action='store_true', help='increase verbosity')
-parser.add_option('--config', default=None, action='callback', metavar='FILE', type='str', callback=load_config, help='specify configuration file')
-parser.add_option('--disable', default=False, action='store_true', help='print disabled message and exit with error')
-parser.add_option('--disable-md5', default=False, action='store_true', help='disable MD5 data error detection')
-parser.add_option('--md5', default=None, action='store', metavar='HASH', help='set default MD5 hash value')
-parser.add_option('--accept-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern')
-parser.add_option('--deny-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern')
-parser.add_option('--output-dir', default='', action='store', help='specify output directory')
-
-fetch = Fetch(*parser.parse_args())
-fetch.run()
diff --git a/make/include/contrib.defs b/make/include/contrib.defs
index f48b57074..c161dc78c 100644
--- a/make/include/contrib.defs
+++ b/make/include/contrib.defs
@@ -26,11 +26,12 @@ define import.CONTRIB.defs
     ##
     ## target: fetch
     ##
-    $(1).FETCH.tar    = $$(CONTRIB.download/)$$(notdir $$(firstword $$($(1).FETCH.url)))
-    $(1).FETCH.url    = FETCH_IS_UNDEFINED
-    $(1).FETCH.target = $$($(1).FETCH.tar)
+    $(1).FETCH.url      = FETCH_URL_IS_UNDEFINED
+    $(1).FETCH.basename = $$(notdir $$(firstword $$($(1).FETCH.url)))
+    $(1).FETCH.distfile = $$(CONTRIB.download/)$$($(1).FETCH.basename)
+    $(1).FETCH.target   = $$($(1).FETCH.distfile)
     define $(1).FETCH
-        $$(FETCH.exe) --config $(BUILD/)fetch.cfg --output-dir $$(dir $$@) $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) $$($(1).FETCH.url)
+        $$(DF.FETCH.exe) --config $(BUILD/)distfile.cfg $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) --output $$@ $$($(1).FETCH.url)
     endef
 
     ##
@@ -38,19 +39,19 @@ define import.CONTRIB.defs
     ##
     $(1).VERIFY.target = $$($(1).build/).stamp.verify
     define $(1).VERIFY
-        $$(FETCH.exe) --config $(BUILD/)fetch.cfg --output-dir $$(dir $$($(1).FETCH.tar)) $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) $$($(1).FETCH.url)
+        $$(DF.VERIFY.exe) --config $(BUILD/)distfile.cfg $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) $$($(1).FETCH.distfile)
         $$(TOUCH.exe) $$@
     endef
 
     ##
     ## target: extract
     ##
-    $(1).EXTRACT.tarbase = $$(strip $$(foreach x,tar.bz2 tar.gz,$$(patsubst %.$$(x),%,$$(filter %.$$(x),$$(notdir $$($(1).FETCH.url))))))
+    $(1).EXTRACT.tarbase = $$(strip $$(foreach x,tar.bz2 tar.gz,$$(patsubst %.$$(x),%,$$(filter %.$$(x),$$($(1).FETCH.basename)))))
     $(1).EXTRACT.dir/    = $$($(1).build/)$$($(1).EXTRACT.tarbase)/
     $(1).EXTRACT.target  = $$($(1).build/).stamp.extract
     define $(1).EXTRACT
         $$(RM.exe) -fr $$($(1).EXTRACT.dir/)
-        $$(TAR.exe) xfC $$($(1).FETCH.tar) $$($(1).build/)
+        $$(TAR.exe) xfC $$($(1).FETCH.distfile) $$($(1).build/)
         $$(TOUCH.exe) $$@
     endef
 
@@ -345,6 +346,8 @@ $($(1).name): $($(1).name).build
 ##
 contrib.fetch: $($(1).name).fetch
 contrib.verify: $($(1).name).verify
+contrib.verify.touch: $($(1).name).verify.touch
+contrib.verify.untouch: $($(1).name).verify.untouch
 contrib.extract: $($(1).name).extract
 contrib.patch: $($(1).name).patch
 contrib.configure: $($(1).name).configure
diff --git a/make/include/tool.defs b/make/include/tool.defs
index 579a6dd98..cfd3b3ef4 100644
--- a/make/include/tool.defs
+++ b/make/include/tool.defs
@@ -1,13 +1,14 @@
-AR.exe    = ar
-CP.exe    = cp
-FETCH.exe = $(SRC/)make/python_launcher $(SRC/)make/fetch.py
-M4.exe    = m4
-MKDIR.exe = mkdir
-PATCH.exe = patch
-RM.exe    = rm
-TAR.exe   = tar
-TOUCH.exe = touch
-MV.exe    = mv
-ZIP.exe   = zip
-LN.exe    = ln
-GIT.exe   = git
+AR.exe        = ar
+CP.exe        = cp
+DF.FETCH.exe  = $(SRC/)make/python_launcher $(SRC/)make/df-fetch.py
+DF.VERIFY.exe = $(SRC/)make/python_launcher $(SRC/)make/df-verify.py
+M4.exe        = m4
+MKDIR.exe     = mkdir
+PATCH.exe     = patch
+RM.exe        = rm
+TAR.exe       = tar
+TOUCH.exe     = touch
+MV.exe        = mv
+ZIP.exe       = zip
+LN.exe        = ln
+GIT.exe       = git
diff --git a/make/lib/hb_distfile.py b/make/lib/hb_distfile.py
new file mode 100644
index 000000000..72cda054b
--- /dev/null
+++ b/make/lib/hb_distfile.py
@@ -0,0 +1,106 @@
+###############################################################################
+##
+## Coded for minimum version of Python 2.7 .
+##
+## Python3 is incompatible.
+##
+## Authors: konablend
+##
+###############################################################################
+
+import json
+import os
+import random
+import re
+import string
+import sys
+import traceback
+
+from optparse import OptionParser
+
+###############################################################################
+
+class Tool(object):
+    LOG_QUIET   = 0
+    LOG_INFO    = 1
+    LOG_VERBOSE = 2
+    LOG_DEBUG   = 3
+
+    def __init__(self):
+        self.name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
+        self.parser = OptionParser()
+        self.parser.add_option('-v', '--verbose', default=Tool.LOG_INFO, action='count', dest='verbosity', help='increase verbosity')
+        self.parser.add_option('--config', default=None, action='callback', metavar='FILE', type='str', callback=self._load_config, help='specify configuration file')
+
+    def _parse(self):
+        (self.options,self.args) = self.parser.parse_args()
+
+    ## be sure not to use any methods referencing self.options as we are still parsing args
+    def _load_config(self, option, opt, value, parser):
+        with open(value, 'r') as file:
+            data = json.load(file)
+        parser.values.verbosity = data['verbosity']
+        extend = getattr(self, '_load_config2', None)
+        if extend:
+            extend(parser, data)
+
+    ## newline not required
+    def errln(self, format, *args, **kwargs):
+        s = (format % args)
+        if re.match('^.*[!?:;.]$', s):
+            if kwargs.get('exit', None) != None:
+                sys.stderr.write('ERROR: %s stop.\n' % (s))
+                sys.exit(1)
+            sys.stderr.write('ERROR: %s continuing\n' % (s))
+        else:
+            if kwargs.get('exit', None) != None:
+                sys.stderr.write('ERROR: %s; stop.\n' % (s))
+                sys.exit(1)
+            sys.stderr.write('ERROR: %s; continuing.\n' % (s))
+
+    ## newline not required
+    def warnln(self, format, *args):
+        s = (format % args)
+        if re.match( '^.*[!?:;.]$', s ):
+            sys.stdout.write('WARNING: %s continuing.\n' % (s))
+        else:
+            sys.stdout.write('WARNING: %s; continuing.\n' % (s))
+
+    ## newline required
+    def infof(self, format, *args):
+        if self.options.verbosity >= Tool.LOG_INFO:
+            sys.stdout.write(format % args)
+
+    ## newline required
+    def verbosef(self, format, *args):
+        if self.options.verbosity >= Tool.LOG_VERBOSE:
+            sys.stdout.write(format % args)
+
+    ## newline required
+    def debugf(self, format, *args):
+        if self.options.verbosity >= Tool.LOG_DEBUG:
+            sys.stdout.write(format % args)
+
+    def debug_exception(self, xinfo=None):
+        if self.options.verbosity >= Tool.LOG_DEBUG:
+            if not xinfo:
+                xinfo = sys.exc_info()
+            traceback.print_exception(*xinfo)
+
+    ## generate a temporary filename - not worried about race conditions
+    def mktmpname(self, filename):
+        return filename + '.tmp.' + ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
+
+###############################################################################
+
+class ToolError(Exception):
+    def __init__(self, op='unknown', text=None):
+        self.op = op
+        self.text = text
+
+    def __call__(self, text):
+        self.text = text
+        return self
+
+    def __str__(self):
+        return self.text