Build: refactor fetch for contrib tarballs

Fetch is now python-based and runs on the same version as does
configure. The source script is make/fetch.py. New features:

MD5 hash tracking for tarballs. Data values for all contribs added.
Upon download, the file will be verified, and only then will it be moved
into place inside downloads/ . Files that exist before the build system
does a fetch will not be md5-checked.

Multiple URLs for tarballs. Each module may specify one or more URLs and
by convention the official HandBrake should be first when possible. Each
URL is tried in sequence, and if it fails for any reason, the next URL
is tried. If no URL succeeds, a hard-error is reported.

Network fetching may be disabled via configure options. --disable-fetch
will hard-error if a fetch is attempted. --accept-fetch-url=SPEC and
--deny-fetch-url=SPEC offer an ACL-style mechanism using regex to match
against URLs. For example, --accept-fecth-url='.*/download.handbrake.fr/.*'
would skip any non-matching URLs.

Build dependencies have been lightened. wget and curl are no longer
required. TODO: GTK packaging should also be able to remove those deps.

40 files changed, 353 insertions, 35 deletions

diff --git a/contrib/autoconf/module.defs b/contrib/autoconf/module.defs
index c1c4140d9..b23480332 100644
--- a/contrib/autoconf/module.defs
+++ b/contrib/autoconf/module.defs
@@ -2,3 +2,4 @@ $(eval $(call import.MODULE.defs,AUTOCONF,autoconf,M4))
 $(eval $(call import.CONTRIB.defs,AUTOCONF))
 
 AUTOCONF.FETCH.url = http://download.handbrake.fr/handbrake/contrib/autoconf-2.69.tar.bz2
+AUTOCONF.FETCH.md5 = 58e0cca25061e1b3e2bc30a5be13815b
diff --git a/contrib/automake/module.defs b/contrib/automake/module.defs
index 72d39e6b0..d42542aad 100644
--- a/contrib/automake/module.defs
+++ b/contrib/automake/module.defs
@@ -2,3 +2,4 @@ $(eval $(call import.MODULE.defs,AUTOMAKE,automake,AUTOCONF))
 $(eval $(call import.CONTRIB.defs,AUTOMAKE))
 
 AUTOMAKE.FETCH.url = http://download.handbrake.fr/handbrake/contrib/automake-1.13.1.tar.bz2
+AUTOMAKE.FETCH.md5 = c66595aafe79e4e5455b4b3491fe6b05
diff --git a/contrib/bzip2/module.defs b/contrib/bzip2/module.defs
index 0a7463e20..b477d8b51 100644
--- a/contrib/bzip2/module.defs
+++ b/contrib/bzip2/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,BZIP2,bzip2))
 $(eval $(call import.CONTRIB.defs,BZIP2))
 
 BZIP2.FETCH.url = http://download.handbrake.fr/handbrake/contrib/bzip2-1.0.6.tar.gz
+BZIP2.FETCH.md5 = 6ec09fe81e3380184ed11e21ea06c215
 BZIP2.EXTRACT.tarbase = bzip2
 BZIP2.CONFIGURE = $(TOUCH.exe) $@
 
diff --git a/contrib/cmake/module.defs b/contrib/cmake/module.defs
index 0cd986bd6..56f6fea59 100644
--- a/contrib/cmake/module.defs
+++ b/contrib/cmake/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,CMAKE,cmake))
 $(eval $(call import.CONTRIB.defs,CMAKE))
 
 CMAKE.FETCH.url = http://download.handbrake.fr/handbrake/contrib/cmake-3.3.2.tar.gz
+CMAKE.FETCH.md5 = 5febbd11bcaac854a27eebaf4a124be2
 
 CMAKE.CONFIGURE.deps =
 CMAKE.CONFIGURE.shared =
diff --git a/contrib/fdk-aac/module.defs b/contrib/fdk-aac/module.defs
index 157c73132..dc8bafd74 100644
--- a/contrib/fdk-aac/module.defs
+++ b/contrib/fdk-aac/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,FDKAAC,fdkaac))
 $(eval $(call import.CONTRIB.defs,FDKAAC))
 
 FDKAAC.FETCH.url = http://download.handbrake.fr/handbrake/contrib/fdk-aac-0.1.4.tar.gz
+FDKAAC.FETCH.md5 = e274a7d7f6cd92c71ec5c78e4dc9f8b7
 
 # fdk-aac configure script fails to add compiler optimizations if the
 # CFLAGS env variable is set during configure.  Since we set it, we
diff --git a/contrib/ffmpeg/module.defs b/contrib/ffmpeg/module.defs
index 682ecb575..03ddb3d0b 100644
--- a/contrib/ffmpeg/module.defs
+++ b/contrib/ffmpeg/module.defs
@@ -7,6 +7,7 @@ $(eval $(call import.MODULE.defs,FFMPEG,ffmpeg,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,FFMPEG))
 
 FFMPEG.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libav-v11.3-0-g00abc00.tar.gz
+FFMPEG.FETCH.md5 = 23648740623345f122cfd0e015063c7e
 
 FFMPEG.CONFIGURE.deps  =
 FFMPEG.CONFIGURE.env   =
diff --git a/contrib/fontconfig/module.defs b/contrib/fontconfig/module.defs
index 8fe0785eb..ccb61f021 100644
--- a/contrib/fontconfig/module.defs
+++ b/contrib/fontconfig/module.defs
@@ -3,6 +3,7 @@ $(eval $(call import.MODULE.defs,FONTCONFIG,fontconfig,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,FONTCONFIG))
 
 FONTCONFIG.FETCH.url = http://download.handbrake.fr/handbrake/contrib/fontconfig-2.11.94.tar.bz2
+FONTCONFIG.FETCH.md5 = c988ea12f4117330246e041109152b4a
 FONTCONFIG.EXTRACT.tarbase = fontconfig-2.11.94
 
 #
diff --git a/contrib/freetype/module.defs b/contrib/freetype/module.defs
index 4646ce295..be83c7855 100644
--- a/contrib/freetype/module.defs
+++ b/contrib/freetype/module.defs
@@ -3,5 +3,6 @@ $(eval $(call import.MODULE.defs,FREETYPE,freetype,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,FREETYPE))
 
 FREETYPE.FETCH.url = http://download.handbrake.fr/handbrake/contrib/freetype-2.6.tar.bz2
+FREETYPE.FETCH.md5 = 5682890cb0267f6671dd3de6eabd3e69
 
 FREETYPE.CONFIGURE.extra = --with-harfbuzz=no --with-png=no
diff --git a/contrib/fribidi/module.defs b/contrib/fribidi/module.defs
index 646a566ab..36d50f639 100644
--- a/contrib/fribidi/module.defs
+++ b/contrib/fribidi/module.defs
@@ -6,6 +6,7 @@ $(eval $(call import.MODULE.defs,FRIBIDI,fribidi))
 $(eval $(call import.CONTRIB.defs,FRIBIDI))
 
 FRIBIDI.FETCH.url = http://download.handbrake.fr/handbrake/contrib/fribidi-0.19.7.tar.bz2
+FRIBIDI.FETCH.md5 = 6c7e7cfdd39c908f7ac619351c1c5c23
 
 ifeq (1-mingw,$(BUILD.cross)-$(BUILD.system))
     FRIBIDI.CONFIGURE.extra = --with-glib=no
diff --git a/contrib/jansson/module.defs b/contrib/jansson/module.defs
index 494dc845e..7a9bd0ae3 100644
--- a/contrib/jansson/module.defs
+++ b/contrib/jansson/module.defs
@@ -2,5 +2,6 @@ $(eval $(call import.MODULE.defs,JANSSON,jansson))
 $(eval $(call import.CONTRIB.defs,JANSSON))
 
 JANSSON.FETCH.url = http://download.handbrake.fr/handbrake/contrib/jansson-2.6.tar.bz2
+JANSSON.FETCH.md5 = c70a52488db623a26f7213c7c6b7c878
 
 JANSSON.CONFIGURE.bootstrap = rm -fr aclocal.m4 autom4te.cache; mkdir m4; autoreconf -fiv;
diff --git a/contrib/lame/module.defs b/contrib/lame/module.defs
index fbe842ad9..33da72785 100644
--- a/contrib/lame/module.defs
+++ b/contrib/lame/module.defs
@@ -2,4 +2,5 @@ $(eval $(call import.MODULE.defs,LAME,lame))
 $(eval $(call import.CONTRIB.defs,LAME))
 
 LAME.FETCH.url = http://download.handbrake.fr/handbrake/contrib/lame-3.98.tar.gz
+LAME.FETCH.md5 = 7036b52e792538fd665595d56b9e49a0
 LAME.EXTRACT.tarbase = lame
diff --git a/contrib/libass/module.defs b/contrib/libass/module.defs
index 6f32d0ec1..03e37bd27 100644
--- a/contrib/libass/module.defs
+++ b/contrib/libass/module.defs
@@ -3,6 +3,7 @@ $(eval $(call import.MODULE.defs,LIBASS,libass,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,LIBASS))
 
 LIBASS.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libass-0.12.3.tar.gz
+LIBASS.FETCH.md5 = 648ee785f966c69d4b5d50948e509d93
 
 # Disable as many external dependencies as I can get away with
 # and tell configure where to find our version of freetype
diff --git a/contrib/libbluray/module.defs b/contrib/libbluray/module.defs
index e336bfda0..7154bc706 100644
--- a/contrib/libbluray/module.defs
+++ b/contrib/libbluray/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBBLURAY,libbluray,PKGCONFIG LIBXML2 FREETYPE)
 $(eval $(call import.CONTRIB.defs,LIBBLURAY))
 
 LIBBLURAY.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libbluray-0.8.1.tar.bz2
+LIBBLURAY.FETCH.md5 = c7a89966ad091fa6075a58d31a698e70
 
 ifneq (max,$(LIBBLURAY.GCC.g))
 	LIBBLURAY.CONFIGURE.extra += --disable-debug
diff --git a/contrib/libdvdnav/module.defs b/contrib/libdvdnav/module.defs
index 27f171be8..2626d1697 100644
--- a/contrib/libdvdnav/module.defs
+++ b/contrib/libdvdnav/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBDVDNAV,libdvdnav,PKGCONFIG LIBDVDREAD))
 $(eval $(call import.CONTRIB.defs,LIBDVDNAV))
 
 LIBDVDNAV.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libdvdnav-5.0.1-0-gaa3659d.tar.gz
+LIBDVDNAV.FETCH.md5 = 1fb10188c7a33f480011d3ddf32553c7
 LIBDVDNAV.CONFIGURE.bootstrap = rm -fr aclocal.m4 autom4te.cache configure; autoreconf -I m4 -fiv;
 
 ifneq (max,$(LIBDVDNAV.GCC.g))
diff --git a/contrib/libdvdread/module.defs b/contrib/libdvdread/module.defs
index bf05121d3..25e2eb0be 100644
--- a/contrib/libdvdread/module.defs
+++ b/contrib/libdvdread/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBDVDREAD,libdvdread,PKGCONFIG))
 $(eval $(call import.CONTRIB.defs,LIBDVDREAD))
 
 LIBDVDREAD.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libdvdread-5.0.0-6-gcb1ae87.tar.gz
+LIBDVDREAD.FETCH.md5 = 607a5dd41b0dd2f35433d6deac79b99e
 LIBDVDREAD.CONFIGURE.bootstrap = rm -fr aclocal.m4 autom4te.cache configure; autoreconf -I m4 -fiv;
 
 ifeq (1-mingw,$(BUILD.cross)-$(BUILD.system))
diff --git a/contrib/libgnurx/module.defs b/contrib/libgnurx/module.defs
index e8e2ebe0c..a4c0f2504 100644
--- a/contrib/libgnurx/module.defs
+++ b/contrib/libgnurx/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBGNURX,libgnurx))
 $(eval $(call import.CONTRIB.defs,LIBGNURX))
 
 LIBGNURX.FETCH.url = http://download.handbrake.fr/handbrake/contrib/mingw-libgnurx-2.5.1-src.tar.gz
+LIBGNURX.FETCH.md5 = 35c8fed3101ca1f253e9b6b1966661f6
 LIBGNURX.EXTRACT.tarbase = mingw-libgnurx-2.5.1
 
 LIBGNURX.CONFIGURE.env += AR="$(AR.exe)"
diff --git a/contrib/libiconv/module.defs b/contrib/libiconv/module.defs
index 391361f26..19e4a1240 100644
--- a/contrib/libiconv/module.defs
+++ b/contrib/libiconv/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBICONV,libiconv))
 $(eval $(call import.CONTRIB.defs,LIBICONV))
 
 LIBICONV.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libiconv-1.13.tar.bz2
+LIBICONV.FETCH.md5 = 9c8b41b77f508edf72e26b406e873a2f
 
 # this contrib will not build under MinGW with -std=gnu99
 ifeq (1-mingw,$(BUILD.cross)-$(BUILD.system))
diff --git a/contrib/libmfx/module.defs b/contrib/libmfx/module.defs
index 3c6407921..719b4d840 100644
--- a/contrib/libmfx/module.defs
+++ b/contrib/libmfx/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBMFX,libmfx))
 $(eval $(call import.CONTRIB.defs,LIBMFX))
 
 LIBMFX.FETCH.url = http://download.handbrake.fr/contrib/mfx_dispatch-9f4a84d7.tar.gz
+LIBMFX.FETCH.md5 = 694058b83b43b39b7e5b5fc38dbe2b88
 
 LIBMFX.CONFIGURE.bootstrap = rm -fr aclocal.m4 autom4te.cache; autoreconf -fiv;
 
diff --git a/contrib/libogg/module.defs b/contrib/libogg/module.defs
index 4b1c8c9ac..e1b6e5fdc 100644
--- a/contrib/libogg/module.defs
+++ b/contrib/libogg/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBOGG,libogg))
 $(eval $(call import.CONTRIB.defs,LIBOGG))
 
 LIBOGG.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libogg-1.3.0.tar.gz
+LIBOGG.FETCH.md5 = 0a7eb40b86ac050db3a789ab65fe21c2
 LIBOGG.EXTRACT.tarbase = libogg-1.3.0
 
 LIBOGG.CONFIGURE.extra = --disable-sdl
diff --git a/contrib/libsamplerate/module.defs b/contrib/libsamplerate/module.defs
index 468aa4739..410e6a134 100644
--- a/contrib/libsamplerate/module.defs
+++ b/contrib/libsamplerate/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBSAMPLERATE,libsamplerate))
 $(eval $(call import.CONTRIB.defs,LIBSAMPLERATE))
 
 LIBSAMPLERATE.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libsamplerate-0.1.4.tar.gz
+LIBSAMPLERATE.FETCH.md5 = 69ec6c05f487458f688dda8f3e722e5d
 LIBSAMPLERATE.EXTRACT.tarbase = libsamplerate
 
 # Disable to avoid Carbon.h dependency on OSX
diff --git a/contrib/libtheora/module.defs b/contrib/libtheora/module.defs
index 7aef8e2dc..54435da6f 100644
--- a/contrib/libtheora/module.defs
+++ b/contrib/libtheora/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBTHEORA,libtheora,LIBOGG LIBVORBIS))
 $(eval $(call import.CONTRIB.defs,LIBTHEORA))
 
 LIBTHEORA.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libtheora-1.1.0.tar.bz2
+LIBTHEORA.FETCH.md5 = d0f83cf7f13e2b3bd068a858ca1398ad
 
 LIBTHEORA.CONFIGURE.extra = \
     --disable-examples \
diff --git a/contrib/libtool/module.defs b/contrib/libtool/module.defs
index 803fb726a..26739730c 100644
--- a/contrib/libtool/module.defs
+++ b/contrib/libtool/module.defs
@@ -2,3 +2,4 @@ $(eval $(call import.MODULE.defs,LIBTOOL,libtool,AUTOCONF AUTOMAKE PKGCONFIG))
 $(eval $(call import.CONTRIB.defs,LIBTOOL))
 
 LIBTOOL.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libtool-2.4.2.tar.bz2
+LIBTOOL.FETCH.md5 = f1a2109b9cd9c03a8df1f47e672c3aeb
diff --git a/contrib/libvorbis/module.defs b/contrib/libvorbis/module.defs
index 84d018fc8..d63c92b75 100644
--- a/contrib/libvorbis/module.defs
+++ b/contrib/libvorbis/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBVORBIS,libvorbis,LIBOGG))
 $(eval $(call import.CONTRIB.defs,LIBVORBIS))
 
 LIBVORBIS.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libvorbis-aotuv_b6.03.tar.bz2
+LIBVORBIS.FETCH.md5 = 586d2ac0fa13f32cba78be5db4a16330
 LIBVORBIS.EXTRACT.tarbase = aotuv-b6.03_20110424
 
 LIBVORBIS.CONFIGURE.extra = --with-ogg=$(call fn.ABSOLUTE,$(CONTRIB.build/)) HAVE_PKG_CONFIG="no"
diff --git a/contrib/libvpx/module.defs b/contrib/libvpx/module.defs
index e257d3493..77ab27cb1 100644
--- a/contrib/libvpx/module.defs
+++ b/contrib/libvpx/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBVPX,libvpx,YASM))
 $(eval $(call import.CONTRIB.defs,LIBVPX))
 
 LIBVPX.FETCH.url = http://download.handbrake.fr/contrib/libvpx-v1.3.0.tar.bz2
+LIBVPX.FETCH.md5 = 14783a148872f2d08629ff7c694eb31f
 
 LIBVPX.CONFIGURE.args.host =
 LIBVPX.CONFIGURE.deps  =
diff --git a/contrib/libxml2/module.defs b/contrib/libxml2/module.defs
index 4c1d14bba..aa37d0521 100644
--- a/contrib/libxml2/module.defs
+++ b/contrib/libxml2/module.defs
@@ -3,6 +3,7 @@ $(eval $(call import.MODULE.defs,LIBXML2,libxml2,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,LIBXML2))
 
 LIBXML2.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libxml2-2.7.7.tar.gz
+LIBXML2.FETCH.md5 = 9abc9959823ca9ff904f1fbcf21df066
 LIBXML2.EXTRACT.tarbase = libxml2-2.7.7
 
 # The Python components do not build on MinGW due to the lack of a select() call
diff --git a/contrib/m4/module.defs b/contrib/m4/module.defs
index 1fd5b0aa4..b73674bed 100644
--- a/contrib/m4/module.defs
+++ b/contrib/m4/module.defs
@@ -2,3 +2,4 @@ $(eval $(call import.MODULE.defs,M4,m4))
 $(eval $(call import.CONTRIB.defs,M4))
 
 M4.FETCH.url = http://download.handbrake.fr/handbrake/contrib/m4-1.4.16.tar.bz2
+M4.FETCH.md5 = 8a7cef47fecab6272eb86a6be6363b2f
diff --git a/contrib/pkgconfig/module.defs b/contrib/pkgconfig/module.defs
index f87a343f3..24a006852 100644
--- a/contrib/pkgconfig/module.defs
+++ b/contrib/pkgconfig/module.defs
@@ -2,5 +2,6 @@ $(eval $(call import.MODULE.defs,PKGCONFIG,pkgconfig))
 $(eval $(call import.CONTRIB.defs,PKGCONFIG))
 
 PKGCONFIG.FETCH.url = http://download.handbrake.fr/handbrake/contrib/pkg-config-0.28.tar.bz2
+PKGCONFIG.FETCH.md5 = 60c2d5353d501c5619d535c6d4d80f1c
 
 PKGCONFIG.CONFIGURE.extra = --with-internal-glib --disable-host-tool
diff --git a/contrib/pthreadw32/module.defs b/contrib/pthreadw32/module.defs
index 9499e855c..c2bfc0a5e 100644
--- a/contrib/pthreadw32/module.defs
+++ b/contrib/pthreadw32/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,PTHREADW32,pthreadw32))
 $(eval $(call import.CONTRIB.defs,PTHREADW32))
 
 PTHREADW32.FETCH.url = http://download.handbrake.fr/handbrake/contrib/pthreads-w32-cvs20100909.tar.bz2
+PTHREADW32.FETCH.md5 = d1dea735b53176567e9841ca77388633
 
 PTHREADW32.CONFIGURE = $(TOUCH.exe) $@
 
diff --git a/contrib/x264/module.defs b/contrib/x264/module.defs
index ff4936a81..7497b9d6d 100644
--- a/contrib/x264/module.defs
+++ b/contrib/x264/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,X264,x264,YASM PTHREADW32))
 $(eval $(call import.CONTRIB.defs,X264))
 
 X264.FETCH.url = http://download.handbrake.fr/handbrake/contrib/x264-r2665-a01e339.tar.gz
+X264.FETCH.md5 = 4ffeac9157c5a7119b5b6ff36a5d96b7
 X264.EXTRACT.tarbase = x264
 
 X264.GCC.args.c_std =
diff --git a/contrib/x265/module.defs b/contrib/x265/module.defs
index 9a6c3dec4..e760a0836 100644
--- a/contrib/x265/module.defs
+++ b/contrib/x265/module.defs
@@ -3,6 +3,7 @@ $(eval $(call import.MODULE.defs,X265,x265,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,X265))
 
 X265.FETCH.url = http://download.handbrake.fr/contrib/x265_1.9.tar.gz
+X265.FETCH.md5 = f34a1c4c660ff07511365cb0983cf164
 
 X265.CONFIGURE.exe         = cmake
 X265.CONFIGURE.args.prefix = -DCMAKE_INSTALL_PREFIX="$(X265.CONFIGURE.prefix)"
diff --git a/contrib/yasm/module.defs b/contrib/yasm/module.defs
index b5f3ee788..ae1e31e7f 100644
--- a/contrib/yasm/module.defs
+++ b/contrib/yasm/module.defs
@@ -2,4 +2,5 @@ $(eval $(call import.MODULE.defs,YASM,yasm))
 $(eval $(call import.CONTRIB.defs,YASM))
 
 YASM.FETCH.url = http://download.handbrake.fr/handbrake/contrib/yasm-1.2.0.tar.gz
+YASM.FETCH.md5 = 4cfc0686cf5350dd1305c4d905eb55a6
 
diff --git a/contrib/zlib/module.defs b/contrib/zlib/module.defs
index 9ddab5746..541ced6dc 100644
--- a/contrib/zlib/module.defs
+++ b/contrib/zlib/module.defs
@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,ZLIB,zlib))
 $(eval $(call import.CONTRIB.defs,ZLIB))
 
 ZLIB.FETCH.url = http://download.handbrake.fr/handbrake/contrib/zlib-1.2.3.tar.gz
+ZLIB.FETCH.md5 = af3358a811ad3469a2e54db49f77d52a
 ZLIB.EXTRACT.tarbase = zlib
 ZLIB.CONFIGURE.args = !sete @dir !env !exe @prefix !extra
 
diff --git a/doc/texi/building/prerequisites.common.texi b/doc/texi/building/prerequisites.common.texi
index 30b1621f1..4c2088cd1 100644
--- a/doc/texi/building/prerequisites.common.texi
+++ b/doc/texi/building/prerequisites.common.texi
@@ -3,10 +3,8 @@ The following general tools are used on various platforms and it is recommended
 @itemize @bullet
 @item git - 2.4.3
 @item python - Python 2.7.1
-@item curl - curl 7.21.4 (or wget)
 @item m4 - GNU M4 1.4.6
 @item make - GNU Make 3.81
 @item patch - Patch 2.5.8
 @item tar - GNU tar 1.26
-@item wget - GNU Wget 1.13.4 (or curl)
 @end itemize
diff --git a/make/configure.py b/make/configure.py
index 7ef88977e..b5aa8efb2 100644
--- a/make/configure.py
+++ b/make/configure.py
@@ -1,7 +1,8 @@
 ###############################################################################
 ##
-## This script is coded for minimum version of Python 2.4 .
-## Pyhthon3 is incompatible.
+## This script is coded for minimum version of Python 2.7 .
+##
+## Python3 is incompatible.
 ##
 ## Authors: konablend
 ##
@@ -9,6 +10,7 @@
 
 import fnmatch
 import glob
+import json
 import optparse
 import os
 import platform
@@ -19,7 +21,6 @@ import time
 from datetime import datetime, timedelta
 
 from optparse import OptionGroup
-from optparse import OptionGroup
 from optparse import OptionParser
 from sys import stderr
 from sys import stdout
@@ -81,13 +82,13 @@ class Configure( object ):
     def infof( self, format, *args ):
         line = format % args
         self._log_verbose.append( line )
-        if cfg.verbose >= Configure.OUT_INFO:
+        if self.verbose >= Configure.OUT_INFO:
             self._log_info.append( line )
             stdout.write( line )
     def verbosef( self, format, *args ):
         line = format % args
         self._log_verbose.append( line )
-        if cfg.verbose >= Configure.OUT_VERBOSE:
+        if self.verbose >= Configure.OUT_VERBOSE:
             stdout.write( line )
 
     ## doc is ready to be populated
@@ -97,14 +98,14 @@ class Configure( object ):
         self.src_final    = self._final_dir( self.build_dir, self.src_dir )
         self.prefix_final = self._final_dir( self.build_dir, self.prefix_dir )
 
-        cfg.infof( 'compute: makevar SRC/    = %s\n', self.src_final )
-        cfg.infof( 'compute: makevar BUILD/  = %s\n', self.build_final )
-        cfg.infof( 'compute: makevar PREFIX/ = %s\n', self.prefix_final )
+        self.infof( 'compute: makevar SRC/    = %s\n', self.src_final )
+        self.infof( 'compute: makevar BUILD/  = %s\n', self.build_final )
+        self.infof( 'compute: makevar PREFIX/ = %s\n', self.prefix_final )
 
     ## perform chdir and enable log recording
     def chdir( self ):
         if os.path.abspath( self.build_dir ) == os.path.abspath( self.src_dir ):
-            cfg.errln( 'build (scratch) directory must not be the same as top-level source root!' )
+            self.errln( 'build (scratch) directory must not be the same as top-level source root!' )
 
         if self.build_dir != os.curdir:
             if os.path.exists( self.build_dir ):
@@ -136,18 +137,18 @@ class Configure( object ):
         try:
             return open( *args )
         except Exception, x:
-            cfg.errln( 'open failure: %s', x )
+            self.errln( 'open failure: %s', x )
 
     def record_log( self ):
         if not self._record:
             return
         self._record = False
         self.verbose = Configure.OUT_QUIET
-        file = cfg.open( 'log/config.info.txt', 'w' )
+        file = self.open( 'log/config.info.txt', 'w' )
         for line in self._log_info:
             file.write( line )
         file.close()
-        file = cfg.open( 'log/config.verbose.txt', 'w' )
+        file = self.open( 'log/config.verbose.txt', 'w' )
         for line in self._log_verbose:
             file.write( line )
         file.close()
@@ -1175,6 +1176,39 @@ class ConfigDocument:
             cfg.errln( 'failed writing to %s\n%s', fname, x )
 
 ###############################################################################
+
+def encodeFetchConfig():
+    fname = 'fetch.cfg'
+    ftmp = fname + '.tmp'
+    data = [
+        options.disable_fetch,
+        options.disable_fetch_md5,
+        options.accept_fetch_url,
+        options.deny_fetch_url,
+    ]
+    try:
+        try:
+            file = cfg.open( ftmp, 'w' )
+            json.dump(data, file)
+            file.write('\n')
+        finally:
+            try:
+                file.close()
+            except:
+                pass
+    except Exception, x:
+        try:
+            os.remove( ftmp )
+        except Exception, x:
+            pass
+        cfg.errln( 'failed writing to %s\n%s', ftmp, x )
+
+    try:
+        os.rename( ftmp, fname )
+    except Exception, x:
+        cfg.errln( 'failed writing to %s\n%s', fname, x )
+
+###############################################################################
 ##
 ## create cli parser
 ##
@@ -1212,9 +1246,19 @@ def createCLI():
 
     ## add hidden options
     cli.add_option( '--xcode-driver', default='bootstrap', action='store', help=optparse.SUPPRESS_HELP )
+
+    ## add general options
     cli.add_option( '--force', default=False, action='store_true', help='overwrite existing build config' )
     cli.add_option( '--verbose', default=False, action='store_true', help='increase verbosity' )
 
+    ## add fetch options
+    grp = OptionGroup( cli, 'Fetch Options' )
+    grp.add_option( '--disable-fetch', default=False, action='store_true', help='disable automatic downloads of 3rd-party distributions' )
+    grp.add_option( '--disable-fetch-md5', default=False, action='store_true', help='disable MD5 data error detection' )
+    grp.add_option( '--accept-fetch-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern' )
+    grp.add_option( '--deny-fetch-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern' )
+    cli.add_option_group( grp )
+
     ## add install options
     grp = OptionGroup( cli, 'Directory Locations' )
     h = IfHost( 'specify sysroot of SDK for Xcode builds', '*-*-darwin*', none=optparse.SUPPRESS_HELP ).value
@@ -1448,7 +1492,6 @@ try:
     class Tools:
         ar    = ToolProbe( 'AR.exe',    'ar' )
         cp    = ToolProbe( 'CP.exe',    'cp' )
-        curl  = ToolProbe( 'CURL.exe',  'curl', abort=False )
         gcc   = ToolProbe( 'GCC.gcc',   'gcc', IfHost( 'gcc-4', '*-*-cygwin*' ))
 
         if host.match( '*-*-darwin*' ):
@@ -1463,7 +1506,6 @@ try:
         ranlib   = ToolProbe( 'RANLIB.exe',   'ranlib' )
         strip    = ToolProbe( 'STRIP.exe',    'strip' )
         tar      = ToolProbe( 'TAR.exe',      'gtar', 'tar' )
-        wget     = ToolProbe( 'WGET.exe',     'wget', abort=False )
         yasm     = ToolProbe( 'YASM.exe',     'yasm', abort=False, minversion=[1,2,0] )
         autoconf = ToolProbe( 'AUTOCONF.exe', 'autoconf', abort=False )
         automake = ToolProbe( 'AUTOMAKE.exe', 'automake', abort=False )
@@ -1474,8 +1516,6 @@ try:
         xcodebuild = ToolProbe( 'XCODEBUILD.exe', 'xcodebuild', abort=False )
         lipo       = ToolProbe( 'LIPO.exe',       'lipo', abort=False )
 
-        fetch = SelectTool( 'FETCH.select', 'fetch', ['wget',wget], ['curl',curl] )
-
     ## run tool probes
     for tool in ToolProbe.tools:
         tool.run()
@@ -1887,6 +1927,8 @@ int main ()
     ## perform
     doc.write( 'make' )
     doc.write( 'm4' )
+    encodeFetchConfig()
+
     if options.launch:
         Launcher( targets )
 
diff --git a/make/fetch.py b/make/fetch.py
new file mode 100644
index 000000000..6b781faf9
--- /dev/null
+++ b/make/fetch.py
@@ -0,0 +1,235 @@
+###############################################################################
+##
+## This script is coded for minimum version of Python 2.7 .
+##
+## Python3 is incompatible.
+##
+## Authors: konablend
+##
+###############################################################################
+
+import errno
+import hashlib
+import json
+import os
+import re
+import tempfile
+import traceback
+import urllib2
+
+from optparse import OptionGroup
+from optparse import OptionParser
+from sys import stderr
+from sys import stdout
+from urlparse import urlparse
+
+###############################################################################
+
+class Fetch(object):
+    def __init__(self, options, urls):
+        if options.disable:
+            self.errln('fetching files from the network is disabled.')
+        self.options = options
+        self.urls = urls
+        if len(self.urls) > 1:
+            self.infof('\n')
+        self.verbosef('OPTIONS:\n')
+        self.verbosef('  disable:        %s\n' % self.options.disable)
+        self.verbosef('  disable_md5:    %s\n' % self.options.disable_md5)
+        self.verbosef('  config:         %s\n' % self.options.config)
+        self.verbosef('  md5:            %s\n' % self.options.md5)
+        self.verbosef('  output_dir:     %s\n' % self.options.output_dir)
+        index = 0
+        for spec in self.options.accept_url:
+            self.verbosef('  accept_url[%d]:  %s\n' % (index,spec))
+            index += 1
+        if not self.options.accept_url:
+            self.verbosef('  accept_url:     %s\n' % None)
+        index = 0
+        for spec in self.options.deny_url:
+            self.verbosef('  deny_url[%d]:    %s\n' % (index,spec))
+            index += 1
+        if not self.options.deny_url:
+            self.verbosef('  deny_url:       %s\n' % None)
+
+    def run(self):
+        files = []
+        for url in self.urls:
+            files.append(self._process_url(url))
+        index = 0
+        for file in files:
+            file.dump(index)
+            index += 1
+        for file in files:
+            if file.run():
+                return
+        self.errln('download failed.')
+
+    def errln(self, format, *args):
+        s = (format % args)
+        if re.match( '^.*[!?:;.]$', s ):
+            stderr.write('ERROR: %s fetch stop.\n' % (s))
+        else:
+            stderr.write('ERROR: %s; fetch stop.\n' % (s))
+        exit(1)
+
+    def warnln(self, format, *args):
+        s = (format % args)
+        if re.match( '^.*[!?:;.]$', s ):
+            stderr.write('WARNING: %s fetch continuing.\n' % (s))
+        else:
+            stderr.write('WARNING: %s; fetch continuing.\n' % (s))
+
+    def infof(self, format, *args):
+        stdout.write(format % args)
+
+    def verbosef(self, format, *args):
+        if self.options.verbose:
+            stdout.write(format % args)
+
+    def _process_url(self, url):
+        props = {}
+        index = 0
+        while True:
+            m = re.match('(\[(\w+)=([^]]+)\])?(.*)', url[index:])
+            if not m.group(1):
+                break
+            props[m.group(2)] = m.group(3)
+            index += len(m.group(1))
+        return File(url[index:], **props)
+
+###############################################################################
+
+class File(object):
+    def __init__(self, url, **kwargs):
+        self.url = url
+        self.md5 = kwargs.get('md5', fetch.options.md5)
+        self.filename = os.path.join(fetch.options.output_dir,os.path.basename(urlparse(self.url).path))
+        self.active = True
+        self.active_descr = 'default'
+        self._accept()
+        self._deny()
+
+    def dump(self, index):
+        fetch.verbosef('URL[%d]: %s\n' % (index,self.url))
+        fetch.verbosef('  MD5: %s\n' % self.md5)
+        fetch.verbosef('  filename: %s\n' % self.filename)
+        fetch.verbosef('  active: %s (%s)\n' % ('yes' if self.active else 'no',self.active_descr))
+
+    def run(self):
+        if not self.active:
+            return False
+        try:
+            if self._download():
+                return True
+        except Exception, x:
+            if fetch.options.verbose:
+                traceback.print_exc()
+            fetch.warnln('%s' % x)
+        return False
+
+    def _accept(self):
+        if not fetch.options.accept_url:
+            return
+        index = 0
+        for spec in fetch.options.accept_url:
+            if re.match(spec, self.url):
+                self.active_descr = 'via accept rule %d: %s' % (index,spec)
+                return
+            index += 1
+        self.active = False
+        self.active_descr = 'no matching spec'
+
+    def _deny(self):
+        index = 0
+        for spec in fetch.options.deny_url:
+            if re.match(spec, self.url):
+                self.active = False
+                self.active_descr = 'via deny rule %d: %s' % (index,spec)
+                return
+            index += 1
+
+    def _download(self):
+        fetch.infof('downloading %s to %s\n' % (self.url,self.filename))
+        hasher = hashlib.md5()
+        ftmp = self.filename + '.' + os.path.basename(tempfile.mktemp())
+        r = urllib2.urlopen(self.url, None, 30)
+        try:
+            o = open(ftmp, 'w')
+            info = r.info()
+            try:
+                content_length = int(info.getheader('Content-Length'))
+            except:
+                content_length = None
+            data_total = 0
+            while True:
+                data = r.read(65536)
+                if not data:
+                    break
+                o.write(data)
+                hasher.update(data)
+                data_total += len(data)
+        except:
+            os.unlink(ftmp)
+        finally:
+            for closeable in [r,o]:
+                try:
+                    if not closeable:
+                        continue
+                    closeable.close()
+                except:
+                    pass
+
+        if content_length and content_length != data_total:
+            fetch.warnln('expected %d bytes, got %d bytes' % (content_length,data_total))
+            os.unlink(ftmp)
+            return False
+
+        if not fetch.options.disable_md5 and self.md5 and self.md5 == hasher.hexdigest():
+            s = ' (verified)'
+        else:
+            s = ''
+
+        fetch.infof("downloaded '%s' - %d bytes - %s%s\n" % (self.filename,data_total,hasher.hexdigest(),s))
+        if not fetch.options.disable_md5 and self.md5 and self.md5 != hasher.hexdigest():
+            os.unlink(ftmp)
+            raise RuntimeError("expected MD5 hash '%s', got '%s'" % (self.md5, hasher.hexdigest()))
+
+        if os.access(self.filename, os.F_OK) and not os.access(self.filename, os.W_OK):
+            os.unlink(ftmp)
+            raise IOError(errno.EACCES, "Permission denied: '%s'" % self.filename)
+
+        try:
+            os.rename(ftmp,self.filename)
+        except:
+            os.unlink(ftmp)
+
+        return True
+
+###############################################################################
+
+def load_config(option, opt, value, parser):
+    with open(value, 'r') as file:
+        data = json.load(file)
+        parser.values.disable     = data[0]
+        parser.values.disable_md5 = data[1]
+        parser.values.accept_url  = data[2]
+        parser.values.deny_url    = data[3]
+
+###############################################################################
+
+parser = OptionParser('usage: %prog [OPTIONS...] [URL...]')
+
+parser.description = 'Fetch files from the network.'
+
+parser.add_option('--verbose', default=False, action='store_true', help='increase verbosity')
+parser.add_option('--config', default=None, action='callback', metavar='FILE', type='str', callback=load_config, help='specify configuration file')
+parser.add_option('--disable', default=False, action='store_true', help='print disabled message and exit with error')
+parser.add_option('--disable-md5', default=False, action='store_true', help='disable MD5 data error detection')
+parser.add_option('--md5', default=None, action='store', metavar='HASH', help='set default MD5 hash value')
+parser.add_option('--accept-url', default=[], action='append', metavar='SPEC', help='accept URL regex pattern')
+parser.add_option('--deny-url', default=[], action='append', metavar='SPEC', help='deny URL regex pattern')
+parser.add_option('--output-dir', default='', action='store', help='specify output directory')
+
+fetch = Fetch(*parser.parse_args())
+fetch.run()
diff --git a/make/include/contrib.defs b/make/include/contrib.defs
index 55f2992c6..040a416e0 100644
--- a/make/include/contrib.defs
+++ b/make/include/contrib.defs
@@ -26,11 +26,11 @@ define import.CONTRIB.defs
     ##
     ## target: fetch
     ##
-    $(1).FETCH.tar    = $$(CONTRIB.download/)$$(notdir $$($(1).FETCH.url))
+    $(1).FETCH.tar    = $$(CONTRIB.download/)$$(notdir $$(firstword $$($(1).FETCH.url)))
     $(1).FETCH.url    = FETCH_IS_UNDEFINED
     $(1).FETCH.target = $$($(1).FETCH.tar)
     define $(1).FETCH
-        $$(call FETCH,$$@,$$($(1).FETCH.url))
+        $$(FETCH.exe) --config $(BUILD/)fetch.cfg --output-dir $$(dir $$@) $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) $$($(1).FETCH.url)
     endef
 
     ##
diff --git a/make/include/main.defs b/make/include/main.defs
index 1781fb41e..1aa36c485 100644
--- a/make/include/main.defs
+++ b/make/include/main.defs
@@ -2,7 +2,6 @@ include $(SRC/)make/include/base.defs
 include $(SRC/)make/include/contrib.defs
 include $(SRC/)make/include/function.defs
 include $(SRC/)make/include/gcc.defs
-include $(SRC/)make/include/select.defs
 include $(SRC/)make/include/target.defs
 include $(SRC/)make/include/tool.defs
 
diff --git a/make/include/select.defs b/make/include/select.defs
deleted file mode 100644
index 32a652523..000000000
--- a/make/include/select.defs
+++ /dev/null
@@ -1,12 +0,0 @@
-##
-## fetch a file from the web via well-known anonymous protocols such as HTTP.
-##
-## $(1) = output filename
-## $(2) = URL
-##
-FETCH = $(FETCH.$(FETCH.select))
-
-FETCH.select  = MISSING
-FETCH.MISSING = $(error one of the following tools is required: wget, curl)
-FETCH.curl    = $(CURL.exe) -q -L -o $(1) $(2)
-FETCH.wget    = $(WGET.exe) -O $(1) $(2)
diff --git a/make/include/tool.defs b/make/include/tool.defs
index 3efe5d83f..579a6dd98 100644
--- a/make/include/tool.defs
+++ b/make/include/tool.defs
@@ -1,13 +1,12 @@
 AR.exe    = ar
 CP.exe    = cp
-CURL.exe  = curl
+FETCH.exe = $(SRC/)make/python_launcher $(SRC/)make/fetch.py
 M4.exe    = m4
 MKDIR.exe = mkdir
 PATCH.exe = patch
 RM.exe    = rm
 TAR.exe   = tar
 TOUCH.exe = touch
-WGET.exe  = wget
 MV.exe    = mv
 ZIP.exe   = zip
 LN.exe    = ln
diff --git a/make/python_launcher b/make/python_launcher
new file mode 100755
index 000000000..c2d9d8da0
--- /dev/null
+++ b/make/python_launcher
@@ -0,0 +1,25 @@
+#! /bin/sh
+#
+
+inpath()
+{
+    IFS=:
+    for d in $PATH
+    do
+        if [ -x $d/$1 ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+for p in python2.7 python2.6 python2.5 python2.4 python2 python
+do
+    if ( inpath $p ); then
+        exec $p "$@"
+        exit 0
+    fi
+done
+
+echo "ERROR: no suitable version of python found."
+exit 1