Build: refactor fetch for contrib tarballs

Fetch is now python-based and runs on the same version as does
configure. The source script is make/fetch.py. New features:

MD5 hash tracking for tarballs. Data values for all contribs added.
Upon download, the file will be verified, and only then will it be moved
into place inside downloads/ . Files that exist before the build system
does a fetch will not be md5-checked.

Multiple URLs for tarballs. Each module may specify one or more URLs and
by convention the official HandBrake should be first when possible. Each
URL is tried in sequence, and if it fails for any reason, the next URL
is tried. If no URL succeeds, a hard-error is reported.

Network fetching may be disabled via configure options. --disable-fetch
will hard-error if a fetch is attempted. --accept-fetch-url=SPEC and
--deny-fetch-url=SPEC offer an ACL-style mechanism using regex to match
against URLs. For example, --accept-fecth-url='.*/download.handbrake.fr/.*'
would skip any non-matching URLs.

Build dependencies have been lightened. wget and curl are no longer
required. TODO: GTK packaging should also be able to remove those deps.

1 files changed, 58 insertions, 16 deletions

diff --git a/make/configure.py b/make/configure.py
index 7ef88977e..b5aa8efb2 100644
--- a/make/configure.py
+++ b/make/configure.py
@@ -1,7 +1,8 @@
 ###############################################################################
 ##
-## This script is coded for minimum version of Python 2.4 .
-## Pyhthon3 is incompatible.
+## This script is coded for minimum version of Python 2.7 .
+##
+## Python3 is incompatible.
 ##
 ## Authors: konablend
 ##
@@ -9,6 +10,7 @@
 
 import fnmatch
 import glob
+import json
 import optparse
 import os
 import platform
@@ -19,7 +21,6 @@ import time
 from datetime import datetime, timedelta
 
 from optparse import OptionGroup
-from optparse import OptionGroup
 from optparse import OptionParser
 from sys import stderr
 from sys import stdout
@@ -81,13 +82,13 @@ class Configure( object ):
     def infof( self, format, *args ):
         line = format % args
         self._log_verbose.append( line )
-        if cfg.verbose >= Configure.OUT_INFO:
+        if self.verbose >= Configure.OUT_INFO:
             self._log_info.append( line )
             stdout.write( line )
     def verbosef( self, format, *args ):
         line = format % args
         self._log_verbose.append( line )
-        if cfg.verbose >= Configure.OUT_VERBOSE:
+        if self.verbose >= Configure.OUT_VERBOSE:
             stdout.write( line )
 
     ## doc is ready to be populated
@@ -97,14 +98,14 @@ class Configure( object ):
         self.src_final    = self._final_dir( self.build_dir, self.src_dir )
         self.prefix_final = self._final_dir( self.build_dir, self.prefix_dir )
 
-        cfg.infof( 'compute: makevar SRC/    = %s\n', self.src_final )
-        cfg.infof( 'compute: makevar BUILD/  = %s\n', self.build_final )
-        cfg.infof( 'compute: makevar PREFIX/ = %s\n', self.prefix_final )
+        self.infof( 'compute: makevar SRC/    = %s\n', self.src_final )
+        self.infof( 'compute: makevar BUILD/  = %s\n', self.build_final )
+        self.infof( 'compute: makevar PREFIX/ = %s\n', self.prefix_final )
 
     ## perform chdir and enable log recording
     def chdir( self ):
         if os.path.abspath( self.build_dir ) == os.path.abspath( self.src_dir ):
-            cfg.errln( 'build (scratch) directory must not be the same as top-level source root!' )
+            self.errln( 'build (scratch) directory must not be the same as top-level source root!' )
 
         if self.build_dir != os.curdir:
             if os.path.exists( self.build_dir ):
@@ -136,18 +137,18 @@ class Configure( object ):
         try:
             return open( *args )
         except Exception, x:
-            cfg.errln( 'open failure: %s', x )
+            self.errln( 'open failure: %s', x )
 
     def record_log( self ):
         if not self._record:
             return
         self._record = False
         self.verbose = Configure.OUT_QUIET
-        file = cfg.open( 'log/config.info.txt', 'w' )
+        file = self.open( 'log/config.info.txt', 'w' )
         for line in self._log_info:
             file.write( line )
         file.close()
-        file = cfg.open( 'log/config.verbose.txt', 'w' )
+        file = self.open( 'log/config.verbose.txt', 'w' )
         for line in self._log_verbose:
             file.write( line )
         file.close()
@@ -1175,6 +1176,39 @@ class ConfigDocument:
             cfg.errln( 'failed writing to %s\n%s', fname, x )
 
 ###############################################################################
+
+def encodeFetchConfig():
+    fname = 'fetch.cfg'
+    ftmp = fname + '.tmp'
+    data = [
+        options.disable_fetch,
+        options.disable_fetch_md5,
+        options.accept_fetch_url,
+        options.deny_fetch_url,
+    ]
+    try:
+        try:
+            file = cfg.open( ftmp, 'w' )
+            json.dump(data, file)
+            file.write('\n')
+        finally:
+            try:
+                file.close()
+            except:
+                pass
+    except Exception, x:
+        try:
+            os.remove( ftmp )
+        except Exception, x:
+            pass
+        cfg.errln( 'failed writing to %s\n%s', ftmp, x )
+
+    try:
+        os.rename( ftmp, fname )
+    except Exception, x:
+        cfg.errln( 'failed writing to %s\n%s', fname, x )
+
+###############################################################################
 ##
 ## create cli parser
 ##
@@ -1212,9 +1246,19 @@ def createCLI():
 
     ## add hidden options
     cli.add_option( '--xcode-driver', default='bootstrap', action='store', help=optparse.SUPPRESS_HELP )
+
+    ## add general options
     cli.add_option( '--force', default=False, action='store_true', help='overwrite existing build config' )
     cli.add_option( '--verbose', default=False, action='store_true', help='increase verbosity' )
 
+    ## add fetch options
+    grp = OptionGroup( cli, 'Fetch Options' )
+    grp.add_option( '--disable-fetch', default=False, action='store_true', help='disable automatic downloads of 3rd-party distributions' )
+    grp.add_option( '--disable-fetch-md5', default=False, action='store_true', help='disable MD5 data error detection' )
+    grp.add_option( '--accept-fetch-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern' )
+    grp.add_option( '--deny-fetch-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern' )
+    cli.add_option_group( grp )
+
     ## add install options
     grp = OptionGroup( cli, 'Directory Locations' )
     h = IfHost( 'specify sysroot of SDK for Xcode builds', '*-*-darwin*', none=optparse.SUPPRESS_HELP ).value
@@ -1448,7 +1492,6 @@ try:
     class Tools:
         ar    = ToolProbe( 'AR.exe',    'ar' )
         cp    = ToolProbe( 'CP.exe',    'cp' )
-        curl  = ToolProbe( 'CURL.exe',  'curl', abort=False )
         gcc   = ToolProbe( 'GCC.gcc',   'gcc', IfHost( 'gcc-4', '*-*-cygwin*' ))
 
         if host.match( '*-*-darwin*' ):
@@ -1463,7 +1506,6 @@ try:
         ranlib   = ToolProbe( 'RANLIB.exe',   'ranlib' )
         strip    = ToolProbe( 'STRIP.exe',    'strip' )
         tar      = ToolProbe( 'TAR.exe',      'gtar', 'tar' )
-        wget     = ToolProbe( 'WGET.exe',     'wget', abort=False )
         yasm     = ToolProbe( 'YASM.exe',     'yasm', abort=False, minversion=[1,2,0] )
         autoconf = ToolProbe( 'AUTOCONF.exe', 'autoconf', abort=False )
         automake = ToolProbe( 'AUTOMAKE.exe', 'automake', abort=False )
@@ -1474,8 +1516,6 @@ try:
         xcodebuild = ToolProbe( 'XCODEBUILD.exe', 'xcodebuild', abort=False )
         lipo       = ToolProbe( 'LIPO.exe',       'lipo', abort=False )
 
-        fetch = SelectTool( 'FETCH.select', 'fetch', ['wget',wget], ['curl',curl] )
-
     ## run tool probes
     for tool in ToolProbe.tools:
         tool.run()
@@ -1887,6 +1927,8 @@ int main ()
     ## perform
     doc.write( 'make' )
     doc.write( 'm4' )
+    encodeFetchConfig()
+
     if options.launch:
         Launcher( targets )