Build: add target: contrib.verify

New target sits between extract and fetch. Thus every build ensures that
exach tarball is not corrupt before extract.

2 files changed, 68 insertions, 11 deletions

diff --git a/make/fetch.py b/make/fetch.py
index 6b781faf9..1a78e5a57 100644
--- a/make/fetch.py
+++ b/make/fetch.py
@@ -53,6 +53,8 @@ class Fetch(object):
             self.verbosef('  deny_url:       %s\n' % None)
 
     def run(self):
+        if not self.urls:
+            self.errln('no URLs specified')
         files = []
         for url in self.urls:
             files.append(self._process_url(url))
@@ -60,10 +62,15 @@ class Fetch(object):
         for file in files:
             file.dump(index)
             index += 1
+        canon = files[0].filename
         for file in files:
-            if file.run():
+            if file.filename != canon:
+                self.errln('URL basename is not consistent')
+        scan = os.access(canon, os.F_OK)
+        for file in files:
+            if file.run(scan):
                 return
-        self.errln('download failed.')
+        self.errln('%s failed.' % ('scan' if scan else 'download'))
 
     def errln(self, format, *args):
         s = (format % args)
@@ -91,6 +98,8 @@ class Fetch(object):
         props = {}
         index = 0
         while True:
+            ## optional per-URL properties
+            ## [key=value][...]URL
             m = re.match('(\[(\w+)=([^]]+)\])?(.*)', url[index:])
             if not m.group(1):
                 break
@@ -103,7 +112,7 @@ class Fetch(object):
 class File(object):
     def __init__(self, url, **kwargs):
         self.url = url
-        self.md5 = kwargs.get('md5', fetch.options.md5)
+        self.props = kwargs # not currently used
         self.filename = os.path.join(fetch.options.output_dir,os.path.basename(urlparse(self.url).path))
         self.active = True
         self.active_descr = 'default'
@@ -112,15 +121,14 @@ class File(object):
 
     def dump(self, index):
         fetch.verbosef('URL[%d]: %s\n' % (index,self.url))
-        fetch.verbosef('  MD5: %s\n' % self.md5)
         fetch.verbosef('  filename: %s\n' % self.filename)
         fetch.verbosef('  active: %s (%s)\n' % ('yes' if self.active else 'no',self.active_descr))
 
-    def run(self):
+    def run(self, scan):
         if not self.active:
             return False
         try:
-            if self._download():
+            if (self._scan() if scan else self._download()):
                 return True
         except Exception, x:
             if fetch.options.verbose:
@@ -185,15 +193,15 @@ class File(object):
             os.unlink(ftmp)
             return False
 
-        if not fetch.options.disable_md5 and self.md5 and self.md5 == hasher.hexdigest():
+        if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 == hasher.hexdigest():
             s = ' (verified)'
         else:
             s = ''
 
         fetch.infof("downloaded '%s' - %d bytes - %s%s\n" % (self.filename,data_total,hasher.hexdigest(),s))
-        if not fetch.options.disable_md5 and self.md5 and self.md5 != hasher.hexdigest():
+        if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 != hasher.hexdigest():
             os.unlink(ftmp)
-            raise RuntimeError("expected MD5 hash '%s', got '%s'" % (self.md5, hasher.hexdigest()))
+            raise RuntimeError("expected MD5 hash '%s', got '%s'" % (fetch.options.md5, hasher.hexdigest()))
 
         if os.access(self.filename, os.F_OK) and not os.access(self.filename, os.W_OK):
             os.unlink(ftmp)
@@ -203,7 +211,31 @@ class File(object):
             os.rename(ftmp,self.filename)
         except:
             os.unlink(ftmp)
+        return True
+
+    def _scan(self):
+        fetch.infof('scanning %s\n' % self.filename)
+        hasher = hashlib.md5()
+        try:
+            o = open(self.filename, 'r')
+            data_total = 0
+            while True:
+                data = o.read(65536)
+                if not data:
+                    break
+                hasher.update(data)
+                data_total += len(data)
+        finally:
+            o.close()
+
+        if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 == hasher.hexdigest():
+            s = ' (verified)'
+        else:
+            s = ''
 
+        fetch.infof("scanned '%s' - %d bytes - %s%s\n" % (self.filename,data_total,hasher.hexdigest(),s))
+        if not fetch.options.disable_md5 and fetch.options.md5 and fetch.options.md5 != hasher.hexdigest():
+            raise RuntimeError("expected MD5 hash '%s', got '%s'" % (fetch.options.md5, hasher.hexdigest()))
         return True
 
 ###############################################################################
diff --git a/make/include/contrib.defs b/make/include/contrib.defs
index 040a416e0..f48b57074 100644
--- a/make/include/contrib.defs
+++ b/make/include/contrib.defs
@@ -34,6 +34,15 @@ define import.CONTRIB.defs
     endef
 
     ##
+    ## target: verify
+    ##
+    $(1).VERIFY.target = $$($(1).build/).stamp.verify
+    define $(1).VERIFY
+        $$(FETCH.exe) --config $(BUILD/)fetch.cfg --output-dir $$(dir $$($(1).FETCH.tar)) $$(if $$($(1).FETCH.md5),--md5 $$($(1).FETCH.md5)) $$($(1).FETCH.url)
+        $$(TOUCH.exe) $$@
+    endef
+
+    ##
     ## target: extract
     ##
     $(1).EXTRACT.tarbase = $$(strip $$(foreach x,tar.bz2 tar.gz,$$(patsubst %.$$(x),%,$$(filter %.$$(x),$$(notdir $$($(1).FETCH.url))))))
@@ -213,7 +222,22 @@ define import.CONTRIB.rules
 $($(1).name).fetch: $$($(1).FETCH.target)
 
 $$($(1).FETCH.target): | $$(dir $$($(1).FETCH.target))
-	$$($(1).FETCH)
+	@$$($(1).FETCH)
+
+##
+## target: verify
+##
+$($(1).name).verify: $$($(1).VERIFY.target)
+
+$$($(1).VERIFY.target): | $$(dir $$($(1).VERIFY.target))
+$$($(1).VERIFY.target): $$($(1).FETCH.target)
+	@$$($(1).VERIFY)
+
+$($(1).name).verify.touch:
+	$$(TOUCH.exe) $$($(1).VERIFY.target)
+
+$($(1).name).verify.untouch:
+	$$(RM.exe) -f $$($(1).VERIFY.target)
 
 ##
 ## target: extract
@@ -222,7 +246,7 @@ $$($(1).FETCH.target): | $$(dir $$($(1).FETCH.target))
 $($(1).name).extract: $$($(1).EXTRACT.target)
 
 $$($(1).EXTRACT.target): | $$(dir $$($(1).EXTRACT.target))
-$$($(1).EXTRACT.target): $$($(1).FETCH.target)
+$$($(1).EXTRACT.target): $$($(1).VERIFY.target)
 	$$($(1).EXTRACT)
 
 $($(1).name).extract.touch:
@@ -320,6 +344,7 @@ $($(1).name): $($(1).name).build
 ## participate with global convenience targets
 ##
 contrib.fetch: $($(1).name).fetch
+contrib.verify: $($(1).name).verify
 contrib.extract: $($(1).name).extract
 contrib.patch: $($(1).name).patch
 contrib.configure: $($(1).name).configure