Build: refactor fetch for contrib tarballs

Fetch is now python-based and runs on the same version as does
configure. The source script is make/fetch.py. New features:

MD5 hash tracking for tarballs. Data values for all contribs added.
Upon download, the file will be verified, and only then will it be moved
into place inside downloads/ . Files that exist before the build system
does a fetch will not be md5-checked.

Multiple URLs for tarballs. Each module may specify one or more URLs and
by convention the official HandBrake should be first when possible. Each
URL is tried in sequence, and if it fails for any reason, the next URL
is tried. If no URL succeeds, a hard-error is reported.

Network fetching may be disabled via configure options. --disable-fetch
will hard-error if a fetch is attempted. --accept-fetch-url=SPEC and
--deny-fetch-url=SPEC offer an ACL-style mechanism using regex to match
against URLs. For example, --accept-fecth-url='.*/download.handbrake.fr/.*'
would skip any non-matching URLs.

Build dependencies have been lightened. wget and curl are no longer
required. TODO: GTK packaging should also be able to remove those deps.

7 files changed, 321 insertions, 33 deletions

diff --git a/make/configure.py b/make/configure.py
index 7ef88977e..b5aa8efb2 100644
--- a/make/configure.py
+++ b/make/configure.py
@@ -1,7 +1,8 @@
 ###############################################################################
 ##
-## This script is coded for minimum version of Python 2.4 .
-## Pyhthon3 is incompatible.
+## This script is coded for minimum version of Python 2.7 .
+##
+## Python3 is incompatible.
 ##
 ## Authors: konablend
 ##
@@ -9,6 +10,7 @@
 
 import fnmatch
 import glob
+import json
 import optparse
 import os
 import platform
@@ -19,7 +21,6 @@ import time
 from datetime import datetime, timedelta
 
 from optparse import OptionGroup
-from optparse import OptionGroup
 from optparse import OptionParser
 from sys import stderr
 from sys import stdout
@@ -81,13 +82,13 @@ class Configure( object ):
     def infof( self, format, *args ):
         line = format % args
         self._log_verbose.append( line )
-        if cfg.verbose >= Configure.OUT_INFO:
+        if self.verbose >= Configure.OUT_INFO:
             self._log_info.append( line )
             stdout.write( line )
     def verbosef( self, format, *args ):
         line = format % args
         self._log_verbose.append( line )
-        if cfg.verbose >= Configure.OUT_VERBOSE:
+        if self.verbose >= Configure.OUT_VERBOSE:
             stdout.write( line )
 
     ## doc is ready to be populated
@@ -97,14 +98,14 @@ class Configure( object ):
         self.src_final    = self._final_dir( self.build_dir, self.src_dir )
         self.prefix_final = self._final_dir( self.build_dir, self.prefix_dir )
 
-        cfg.infof( 'compute: makevar SRC/    = %s\n', self.src_final )
-        cfg.infof( 'compute: makevar BUILD/  = %s\n', self.build_final )
-        cfg.infof( 'compute: makevar PREFIX/ = %s\n', self.prefix_final )
+        self.infof( 'compute: makevar SRC/    = %s\n', self.src_final )
+        self.infof( 'compute: makevar BUILD/  = %s\n', self.build_final )
+        self.infof( 'compute: makevar PREFIX/ = %s\n', self.prefix_final )
 
     ## perform chdir and enable log recording
     def chdir( self ):
         if os.path.abspath( self.build_dir ) == os.path.abspath( self.src_dir ):
-            cfg.errln( 'build (scratch) directory must not be the same as top-level source root!' )
+            self.errln( 'build (scratch) directory must not be the same as top-level source root!' )
 
         if self.build_dir != os.curdir:
             if os.path.exists( self.build_dir ):
@@ -136,18 +137,18 @@ class Configure( object ):
         try:
             return open( *args )
         except Exception, x:
-            cfg.errln( 'open failure: %s', x )
+            self.errln( 'open failure: %s', x )
 
     def record_log( self ):
         if not self._record:
             return
         self._record = False
         self.verbose = Configure.OUT_QUIET
-        file = cfg.open( 'log/config.info.txt', 'w' )
+        file = self.open( 'log/config.info.txt', 'w' )
         for line in self._log_info:
             file.write( line )
         file.close()
-        file = cfg.open( 'log/config.verbose.txt', 'w' )
+        file = self.open( 'log/config.verbose.txt', 'w' )
         for line in self._log_verbose:
             file.write( line )
         file.close()
@@ -1175,6 +1176,39 @@ class ConfigDocument:
             cfg.errln( 'failed writing to %s\n%s', fname, x )
 
 ###############################################################################
+
+def encodeFetchConfig():
+    fname = 'fetch.cfg'
+    ftmp = fname + '.tmp'
+    data = [
+        options.disable_fetch,
+        options.disable_fetch_md5,
+        options.accept_fetch_url,
+        options.deny_fetch_url,
+    ]
+    try:
+        try:
+            file = cfg.open( ftmp, 'w' )
+            json.dump(data, file)
+            file.write('\n')
+        finally:
+            try:
+                file.close()
+            except:
+                pass
+    except Exception, x:
+        try:
+            os.remove( ftmp )
+        except Exception, x:
+            pass
+        cfg.errln( 'failed writing to %s\n%s', ftmp, x )
+
+    try:
+        os.rename( ftmp, fname )
+    except Exception, x:
+        cfg.errln( 'failed writing to %s\n%s', fname, x )
+
+###############################################################################
 ##
 ## create cli parser
 ##
@@ -1212,9 +1246,19 @@ def createCLI():
 
     ## add hidden options
     cli.add_option( '--xcode-driver', default='bootstrap', action='store', help=optparse.SUPPRESS_HELP )
+
+    ## add general options
     cli.add_option( '--force', default=False, action='store_true', help='overwrite existing build config' )
     cli.add_option( '--verbose', default=False, action='store_true', help='increase verbosity' )
 
+    ## add fetch options
+    grp = OptionGroup( cli, 'Fetch Options' )
+    grp.add_option( '--disable-fetch', default=False, action='store_true', help='disable automatic downloads of 3rd-party distributions' )
+    grp.add_option( '--disable-fetch-md5', default=False, action='store_true', help='disable MD5 data error detection' )
+    grp.add_option( '--accept-fetch-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern' )
+    grp.add_option( '--deny-fetch-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern' )
+    cli.add_option_group( grp )
+
     ## add install options
     grp = OptionGroup( cli, 'Directory Locations' )
     h = IfHost( 'specify sysroot of SDK for Xcode builds', '*-*-darwin*', none=optparse.SUPPRESS_HELP ).value
@@ -1448,7 +1492,6 @@ try:
     class Tools:
         ar    = ToolProbe( 'AR.exe',    'ar' )
         cp    = ToolProbe( 'CP.exe',    'cp' )
-        curl  = ToolProbe( 'CURL.exe',  'curl', abort=False )
         gcc   = ToolProbe( 'GCC.gcc',   'gcc', IfHost( 'gcc-4', '*-*-cygwin*' ))
 
         if host.match( '*-*-darwin*' ):
@@ -1463,7 +1506,6 @@ try:
         ranlib   = ToolProbe( 'RANLIB.exe',   'ranlib' )
         strip    = ToolProbe( 'STRIP.exe',    'strip' )
         tar      = ToolProbe( 'TAR.exe',      'gtar', 'tar' )
-        wget     = ToolProbe( 'WGET.exe',     'wget', abort=False )
         yasm     = ToolProbe( 'YASM.exe',     'yasm', abort=False, minversion=[1,2,0] )
         autoconf = ToolProbe( 'AUTOCONF.exe', 'autoconf', abort=False )
         automake = ToolProbe( 'AUTOMAKE.exe', 'automake', abort=False )
@@ -1474,8 +1516,6 @@ try:
         xcodebuild = ToolProbe( 'XCODEBUILD.exe', 'xcodebuild', abort=False )
         lipo       = ToolProbe( 'LIPO.exe',       'lipo', abort=False )
 
-        fetch = SelectTool( 'FETCH.select', 'fetch', ['wget',wget], ['curl',curl] )
-
     ## run tool probes
     for tool in ToolProbe.tools:
         tool.run()
@@ -1887,6 +1927,8 @@ int main ()
     ## perform
     doc.write( 'make' )
     doc.write( 'm4' )
+    encodeFetchConfig()
+
     if options.launch:
         Launcher( targets )
 
diff --git a/make/fetch.py b/make/fetch.py
new file mode 100644
index 000000000..6b781faf9
--- /dev/null
+++ b/make/fetch.py
@@ -0,0 +1,235 @@
+###############################################################################
+##
+## This script is coded for minimum version of Python 2.7 .
+##
+## Python3 is incompatible.
+##
+## Authors: konablend
+##
+###############################################################################
+
+import errno
+import hashlib
+import json
+import os
+import re
+import tempfile
+import traceback
+import urllib2
+
+from optparse import OptionGroup
+from optparse import OptionParser
+from sys import stderr
+from sys import stdout
+from urlparse import urlparse
+
+###############################################################################
+
+class Fetch(object):
+    def __init__(self, options, urls):
+        if options.disable:
+            self.errln('fetching files from the network is disabled.')
+        self.options = options
+        self.urls = urls
+        if len(self.urls) > 1:
+            self.infof('\n')
+        self.verbosef('OPTIONS:\n')
+        self.verbosef('  disable:        %s\n' % self.options.disable)
+        self.verbosef('  disable_md5:    %s\n' % self.options.disable_md5)
+        self.verbosef('  config:         %s\n' % self.options.config)
+        self.verbosef('  md5:            %s\n' % self.options.md5)
+        self.verbosef('  output_dir:     %s\n' % self.options.output_dir)
+        index = 0
+        for spec in self.options.accept_url:
+            self.verbosef('  accept_url[%d]:  %s\n' % (index,spec))
+            index += 1
+        if not self.options.accept_url:
+            self.verbosef('  accept_url:     %s\n' % None)
+        index = 0
+        for spec in self.options.deny_url:
+            self.verbosef('  deny_url[%d]:    %s\n' % (index,spec))
+            index += 1
+        if not self.options.deny_url:
+            self.verbosef('  deny_url:       %s\n' % None)
+
+    def run(self):
+        files = []
+        for url in self.urls:
+            files.append(self._process_url(url))
+        index = 0
+        for file in files:
+            file.dump(index)
+            index += 1
+        for file in files:
+            if file.run():
+                return
+        self.errln('download failed.')
+
+    def errln(self, format, *args):
+        s = (format % args)
+        if re.match( '^.*[!?:;.]$', s ):
+            stderr.write('ERROR: %s fetch stop.\n' % (s))
+        else:
+            stderr.write('ERROR: %s; fetch stop.\n' % (s))
+        exit(1)
+
+    def warnln(self, format, *args):
+        s = (format % args)
+        if re.match( '^.*[!?:;.]$', s ):
+            stderr.write('WARNING: %s fetch continuing.\n' % (s))
+        else:
+            stderr.write('WARNING: %s; fetch continuing.\n' % (s))
+
+    def infof(self, format, *args):
+        stdout.write(format % args)
+
+    def verbosef(self, format, *args):
+        if self.options.verbose:
+            stdout.write(format % args)
+
+    def _process_url(self, url):
+        props = {}
+        index = 0
+        while True:
+            m = re.match('(\[(\w+)=([^]]+)\])?(.*)', url[index:])
+            if not m.group(1):
+                break
+            props[m.group(2)] = m.group(3)
+            index += len(m.group(1))
+        return File(url[index:], **props)
+
+###############################################################################
+
+class File(object):
+    def __init__(self, url, **kwargs):
+        self.url = url
+        self.md5 = kwargs.get('md5', fetch.options.md5)
+        self.filename = os.path.join(fetch.options.output_dir,os.path.basename(urlparse(self.url).path))
+        self.active = True
+        self.active_descr = 'default'
+        self._accept()
+        self._deny()
+
+    def dump(self, index):
+        fetch.verbosef('URL[%d]: %s\n' % (index,self.url))
+        fetch.verbosef('  MD5: %s\n' % self.md5)
+        fetch.verbosef('  filename: %s\n' % self.filename)
+        fetch.verbosef('  active: %s (%s)\n' % ('yes' if self.active else 'no',self.active_descr))
+
+    def run(self):
+        if not self.active:
+            return False
+        try:
+            if self._download():
+                return True
+        except Exception, x:
+            if fetch.options.verbose:
+                traceback.print_exc()
+            fetch.warnln('%s' % x)
+        return False
+
+    def _accept(self):
+        if not fetch.options.accept_url:
+            return
+        index = 0
+        for spec in fetch.options.accept_url:
+            if re.match(spec, self.url):
+                self.active_descr = 'via accept rule %d: %s' % (index,spec)
+                return
+            index += 1
+        self.active = False
+        self.active_descr = 'no matching spec'
+
+    def _deny(self):
+        index = 0
+        for spec in fetch.options.deny_url:
+            if re.match(spec, self.url):
+                self.active = False
+                self.active_descr = 'via deny rule %d: %s' % (index,spec)
+                return
+            index += 1
+
+    def _download(self):
+        fetch.infof('downloading %s to %s\n' % (self.url,self.filename))
+        hasher = hashlib.md5()
+        ftmp = self.filename + '.' + os.path.basename(tempfile.mktemp())
+        r = urllib2.urlopen(self.url, None, 30)
+        try:
+            o = open(ftmp, 'w')
+            info = r.info()
+            try:
+                content_length = int(info.getheader('Content-Length'))
+            except:
+                content_length = None
+            data_total = 0
+            while True:
+                data = r.read(65536)
+                if not data:
+                    break
+                o.write(data)
+                hasher.update(data)
+                data_total += len(data)
+        except:
+            os.unlink(ftmp)
+        finally:
+            for closeable in [r,o]:
+                try:
+                    if not closeable:
+                        continue
+                    closeable.close()
+                except:
+                    pass
+
+        if content_length and content_length != data_total:
+            fetch.warnln('expected %d bytes, got %d bytes' % (content_length,data_total))
+            os.unlink(ftmp)
+            return False
+
+        if not fetch.options.disable_md5 and self.md5 and self.md5 == hasher.hexdigest():
+            s = ' (verified)'
+        else:
+            s = ''
+
+        fetch.infof("downloaded '%s' - %d bytes - %s%s\n" % (self.filename,data_total,hasher.hexdigest(),s))
+        if not fetch.options.disable_md5 and self.md5 and self.md5 != hasher.hexdigest():
+            os.unlink(ftmp)
+            raise RuntimeError("expected MD5 hash '%s', got '%s'" % (self.md5, hasher.hexdigest()))
+
+        if os.access(self.filename, os.F_OK) and not os.access(self.filename, os.W_OK):
+            os.unlink(ftmp)
+            raise IOError(errno.EACCES, "Permission denied: '%s'" % self.filename)
+
+        try:
+            os.rename(ftmp,self.filename)
+        except:
+            os.unlink(ftmp)
+
+        return True
+
+###############################################################################
+
+def load_config(option, opt, value, parser):
+    with open(value, 'r') as file:
+        data = json.load(file)
+        parser.values.disable     = data[0]
+        parser.values.disable_md5 = data[1]
+        parser.values.accept_url  = data[2]
+        parser.values.deny_url    = data[3]
+
+###############################################################################
+
+parser = OptionParser('usage: %prog [OPTIONS...] [URL...]')
+
+parser.description = 'Fetch files from the network.'
+
+parser.add_option('--verbose', default=False, action='store_true', help='increase verbosity')
+parser.add_option('--config', default=None, action='callback', metavar='FILE', type='str', callback=load_config, help='specify configuration file')
+parser.add_option('--disable', default=False, action='store_true', help='print disabled message and exit with error')
+parser.add_option('--disable-md5', default=False, action='store_true', help='disable MD5 data error detection')
+parser.add_option('--md5', default=None, action='store', metavar='HASH', help='set default MD5 hash value')
+parser.add_option('--accept-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern')
+parser.add_option('--deny-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern')
+parser.add_option('--output-dir', default='', action='store', help='specify output directory')
+
+fetch = Fetch(*parser.parse_args())
+fetch.run()
diff --git a/make/include/contrib.defs b/make/include/contrib.defs
index 55f2992c6..040a416e0 100644
--- a/make/include/contrib.defs
+++ b/make/include/contrib.defs
@@ -26,11 +26,11 @@ define import.CONTRIB.defs
     ##
     ## target: fetch
     ##
-    $(1).FETCH.tar    = $$(CONTRIB.download/)$$(notdir $$($(1).FETCH.url))
+    $(1).FETCH.tar    = $$(CONTRIB.download/)$$(notdir $$(firstword $$($(1).FETCH.url)))
     $(1).FETCH.url    = FETCH_IS_UNDEFINED
     $(1).FETCH.target = $$($(1).FETCH.tar)
     define $(1).FETCH
-        $$(call FETCH,$$@,$$($(1).FETCH.url))
+        $$(FETCH.exe) --config $(BUILD/)fetch.cfg --output-dir $$(dir $$@) $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) $$($(1).FETCH.url)
     endef
 
     ##
diff --git a/make/include/main.defs b/make/include/main.defs
index 1781fb41e..1aa36c485 100644
--- a/make/include/main.defs
+++ b/make/include/main.defs
@@ -2,7 +2,6 @@ include $(SRC/)make/include/base.defs
 include $(SRC/)make/include/contrib.defs
 include $(SRC/)make/include/function.defs
 include $(SRC/)make/include/gcc.defs
-include $(SRC/)make/include/select.defs
 include $(SRC/)make/include/target.defs
 include $(SRC/)make/include/tool.defs
 
diff --git a/make/include/select.defs b/make/include/select.defs
deleted file mode 100644
index 32a652523..000000000
--- a/make/include/select.defs
+++ /dev/null
@@ -1,12 +0,0 @@
-##
-## fetch a file from the web via well-known anonymous protocols such as HTTP.
-##
-## $(1) = output filename
-## $(2) = URL
-##
-FETCH = $(FETCH.$(FETCH.select))
-
-FETCH.select  = MISSING
-FETCH.MISSING = $(error one of the following tools is required: wget, curl)
-FETCH.curl    = $(CURL.exe) -q -L -o $(1) $(2)
-FETCH.wget    = $(WGET.exe) -O $(1) $(2)
diff --git a/make/include/tool.defs b/make/include/tool.defs
index 3efe5d83f..579a6dd98 100644
--- a/make/include/tool.defs
+++ b/make/include/tool.defs
@@ -1,13 +1,12 @@
 AR.exe    = ar
 CP.exe    = cp
-CURL.exe  = curl
+FETCH.exe = $(SRC/)make/python_launcher $(SRC/)make/fetch.py
 M4.exe    = m4
 MKDIR.exe = mkdir
 PATCH.exe = patch
 RM.exe    = rm
 TAR.exe   = tar
 TOUCH.exe = touch
-WGET.exe  = wget
 MV.exe    = mv
 ZIP.exe   = zip
 LN.exe    = ln
diff --git a/make/python_launcher b/make/python_launcher
new file mode 100755
index 000000000..c2d9d8da0
--- /dev/null
+++ b/make/python_launcher
@@ -0,0 +1,25 @@
+#! /bin/sh
+#
+
+inpath()
+{
+    IFS=:
+    for d in $PATH
+    do
+        if [ -x $d/$1 ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+for p in python2.7 python2.6 python2.5 python2.4 python2 python
+do
+    if ( inpath $p ); then
+        exec $p "$@"
+        exit 0
+    fi
+done
+
+echo "ERROR: no suitable version of python found."
+exit 1