Bug 1380118 - aom: Support vendoring from github. r?froydnj draft
authorRalph Giles <giles@mozilla.com>
Mon, 10 Jul 2017 16:17:49 -0700
changeset 643638 e3dc669514dbf76eab12d58d065631c98c425727
parent 643637 331eeff69bf1bcf2d86ddb6272dbedd4d434797f
child 643639 f20c3c964d7bafcbefa3cbe30fdf4bef07a387bd
push id73163
push userbmo:giles@thaumas.net
push dateThu, 10 Aug 2017 00:19:36 +0000
Bug 1380118 - aom: Support vendoring from github. r?froydnj Add a --repo switch to `mach vendor aom` to allow specifying an alternate repository url. Update our vendor script to support commit query and snapshot download from github as well as upstream's gitiles instance. This lets us work with experimental branches for testing. Also cleans up some naming and checks the passed url for one of the two supported sites. We could fall back to doing a complete clone and query the local repository, but this covers most use cases. MozReview-Commit-ID: B1SAvnSDuxA
--- a/python/mozbuild/mozbuild/vendor_aom.py
+++ b/python/mozbuild/mozbuild/vendor_aom.py
@@ -7,35 +7,60 @@ from __future__ import absolute_import, 
 from distutils.version import LooseVersion
 import logging
 from mozbuild.base import (
 import mozfile
 import mozpack.path as mozpath
+import os
 import requests
 import re
 import sys
 import tarfile
+from urlparse import urlparse
 class VendorAOM(MozbuildObject):
-    base_url = 'https://aomedia.googlesource.com/aom/'
-    def upstream_url(self, revision):
+    def upstream_snapshot(self, revision):
         '''Construct a url for a tarball snapshot of the given revision.'''
-        return mozpath.join(self.base_url, '+archive', revision + '.tar.gz')
+        if 'googlesource' in self.repo_url:
+            return mozpath.join(self.repo_url, '+archive', revision + '.tar.gz')
+        elif 'github' in self.repo_url:
+            return mozpath.join(self.repo_url, 'archive', revision + '.tar.gz')
+        else:
+            raise ValueError('Unknown git host, no snapshot lookup method')
     def upstream_commit(self, revision):
         '''Convert a revision to a git commit and timestamp.
         Ask the upstream repo to convert the requested revision to
         a git commit id and timestamp, so we can be precise in
         what we're vendoring.'''
-        url = mozpath.join(self.base_url, '+', revision + '?format=JSON')
+        if 'googlesource' in self.repo_url:
+            return self.upstream_googlesource_commit(revision)
+        elif 'github' in self.repo_url:
+            return self.upstream_github_commit(revision)
+        else:
+            raise ValueError('Unknown git host, no commit lookup method')
+    def upstream_validate(self, url):
+        '''Validate repository urls to make sure we can handle them.'''
+        host = urlparse(url).netloc
+        valid_domains = ('googlesource.com', 'github.com')
+        if not any(filter(lambda domain: domain in host, valid_domains)):
+            self.log(logging.ERROR, 'upstream_url', {},
+                     '''Unsupported git host %s; cannot fetch snapshots.
+Please set a repository url with --repo on either googlesource or github.''' % host)
+            sys.exit(1)
+    def upstream_googlesource_commit(self, revision):
+        '''Query gitiles for a git commit and timestamp.'''
+        url = mozpath.join(self.repo_url, '+', revision + '?format=JSON')
         self.log(logging.INFO, 'fetch', {'url': url},
                  'Fetching commit id from {url}')
         req = requests.get(url)
             info = req.json()
         except ValueError as e:
             if 'No JSON object' in e.message:
@@ -43,35 +68,53 @@ class VendorAOM(MozbuildObject):
                 # at the beginning of the json response. Work around this.
                 # https://bugs.chromium.org/p/chromium/issues/detail?id=718550
                 import json
                 info = json.loads(req.text[4:])
         return (info['commit'], info['committer']['time'])
+    def upstream_github_commit(self, revision):
+        '''Query the github api for a git commit id and timestamp.'''
+        repo = urlparse(self.repo_url).path
+        url = mozpath.join('https://api.github.com/repos/', repo, 'commits', revision)
+        self.log(logging.INFO, 'fetch', {'url': url},
+                'Fetching commit id from {url}')
+        req = requests.get(url)
+        req.raise_for_status()
+        info = req.json()
+        return (info['sha'], info['commit']['committer']['date'])
     def fetch_and_unpack(self, revision, target):
         '''Fetch and unpack upstream source'''
-        url = self.upstream_url(revision)
+        url = self.upstream_snapshot(revision)
         self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}')
-        filename = 'libaom-' + revision + '.tar.gz'
+        prefix = 'aom-' + revision
+        filename = prefix + '.tar.gz'
         with open(filename, 'wb') as f:
             req = requests.get(url, stream=True)
             for data in req.iter_content(4096):
         tar = tarfile.open(filename)
         bad_paths = filter(lambda name: name.startswith('/') or '..' in name,
         if any(bad_paths):
             raise Exception("Tar archive contains non-local paths,"
                             "e.g. '%s'" % bad_paths[0])
         self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target)
         self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.')
+        # Github puts everything properly down a directory; move it up.
+        if all(map(lambda name: name.startswith(prefix), tar.getnames())):
+            tardir = mozpath.join(target, prefix)
+            os.system('mv %s/* %s/.* %s' % (tardir, tardir, target))
+            os.rmdir(tardir)
+        # Remove the tarball.
     def update_readme(self, revision, timestamp, target):
         filename = mozpath.join(target, 'README_MOZILLA')
         with open(filename) as f:
             readme = f.read()
         prefix = 'The git commit ID used was'
@@ -157,24 +200,30 @@ Please check manually and update the ven
                      '''You have uncommitted changes to the following files:
 Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`.
-    def vendor(self, revision, ignore_modified=False):
+    def vendor(self, revision, repo, ignore_modified=False):
         if not ignore_modified:
         if not revision:
             revision = 'master'
+        if repo:
+            self.repo_url = repo
+        else:
+            self.repo_url = 'https://aomedia.googlesource.com/aom/'
+        self.upstream_validate(self.repo_url)
         commit, timestamp = self.upstream_commit(revision)
         vendor_dir = mozpath.join(self.topsrcdir, 'third_party/aom')
         self.fetch_and_unpack(commit, vendor_dir)
         self.log(logging.INFO, 'clean_upstream', {},
                  '''Removing unnecessary files.''')
         glue_dir = mozpath.join(self.topsrcdir, 'media/libaom')