Bug 1314147 - Add 'mach vendor aom' for maintaining av1 codec support. r?froydnj draft
authorRalph Giles <giles@mozilla.com>
Wed, 15 Mar 2017 14:54:10 -0700
changeset 572905 9d785699e09b9a7ef7e348f666735671e6dcd4ab
parent 572904 35c922cb9b98d65448330d74886e1e8eca13f56d
child 572906 4c7cc9060012a10fe5ff1df399b810362bf8ccc6
push id57221
push userbmo:giles@thaumas.net
push dateThu, 04 May 2017 22:41:26 +0000
reviewersfroydnj
bugs1314147
milestone55.0a1
Bug 1314147 - Add 'mach vendor aom' for maintaining av1 codec support. r?froydnj We've traditionally had per-directory 'update' scripts for third-party media codec implementations. Instead, leverage the new 'mach vendor' command to centralize the import and build file generation, placing the upstream source in third_party/aom. Note this includes another copy of gtest and other dependencies which we don't use, but they're required by the upstream build process we use to generate our own build description, so I've left them in for now. MozReview-Commit-ID: CnWcSwvQZEh
python/mozbuild/mozbuild/mach_commands.py
python/mozbuild/mozbuild/vendor_aom.py
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1831,16 +1831,29 @@ class Vendor(MachCommandBase):
     @CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true',
         help='Permit overly-large files to be added to the repository',
         default=False)
     def vendor_rust(self, **kwargs):
         from mozbuild.vendor_rust import VendorRust
         vendor_command = self._spawn(VendorRust)
         vendor_command.vendor(**kwargs)
 
+    @SubCommand('vendor', 'aom',
+                description='Vendor av1 video codec reference implementation into the source repository.')
+    @CommandArgument('-r', '--revision',
+        help='Repository tag or commit to update to.')
+    @CommandArgument('--ignore-modified', action='store_true',
+        help='Ignore modified files in current checkout',
+        default=False)
+    def vendor_aom(self, **kwargs):
+        from mozbuild.vendor_aom import VendorAOM
+        vendor_command = self._spawn(VendorAOM)
+        vendor_command.vendor(**kwargs)
+
+
 @CommandProvider
 class WebRTCGTestCommands(GTestCommands):
     @Command('webrtc-gtest', category='testing',
         description='Run WebRTC.org GTest unit tests.')
     @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter',
         help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns),"
              "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).")
     @CommandArgumentGroup('debugging')
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/vendor_aom.py
@@ -0,0 +1,157 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from distutils.version import LooseVersion
+import logging
+from mozbuild.base import (
+    BuildEnvironmentNotFoundException,
+    MozbuildObject,
+)
+import mozfile
+import mozpack.path as mozpath
+import requests
+import re
+import sys
+import tarfile
+
+class VendorAOM(MozbuildObject):
+    base_url = 'https://aomedia.googlesource.com/aom/'
+
+    def upstream_url(self, revision):
+        '''Construct a url for a tarball snapshot of the given revision.'''
+        return mozpath.join(self.base_url, '+archive', revision + '.tar.gz')
+
+    def upstream_commit(self, revision):
+        '''Convert a revision to a git commit and timestamp.
+
+        Ask the upstream repo to convert the requested revision to
+        a git commit id and timestamp, so we can be precise in
+        what we're vendoring.'''
+        url = mozpath.join(self.base_url, '+', revision + '?format=JSON')
+        self.log(logging.INFO, 'fetch', {'url': url},
+                 'Fetching commit id from {url}')
+        req = requests.get(url)
+        req.raise_for_status()
+        try:
+            info = req.json()
+        except ValueError as e:
+            if 'No JSON object' in e.message:
+                # As of 2017 May, googlesource sends 4 garbage characters
+                # at the beginning of the json response. Work around this.
+                # https://bugs.chromium.org/p/chromium/issues/detail?id=718550
+                import json
+                info = json.loads(req.text[4:])
+            else:
+                raise
+        return (info['commit'], info['committer']['time'])
+
+    def fetch_and_unpack(self, revision, target):
+        '''Fetch and unpack upstream source'''
+        url = self.upstream_url(revision)
+        self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}')
+        filename = 'libaom-' + revision + '.tar.gz'
+        with open(filename, 'wb') as f:
+            req = requests.get(url, stream=True)
+            for data in req.iter_content(4096):
+                f.write(data)
+        tar = tarfile.open(filename)
+        bad_paths = filter(lambda name: name.startswith('/') or '..' in name,
+                           tar.getnames())
+        if any(bad_paths):
+            raise Exception("Tar archive contains non-local paths,"
+                            "e.g. '%s'" % bad_paths[0])
+        self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target)
+        mozfile.remove(target)
+        self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.')
+        tar.extractall(target)
+        mozfile.remove(filename)
+
+    def update_readme(self, revision, timestamp, target):
+        filename = mozpath.join(target, 'README_MOZILLA')
+        with open(filename) as f:
+            readme = f.read()
+
+        prefix = 'The git commit ID used was'
+        if prefix in readme:
+            new_readme = re.sub(prefix + ' [v\.a-f0-9]+.*$',
+                                prefix + ' %s (%s).' % (revision, timestamp),
+                                readme)
+        else:
+            new_readme = '%s\n\n%s %s.' % (readme, prefix, revision)
+
+        if readme != new_readme:
+            with open(filename, 'w') as f:
+                f.write(new_readme)
+
+    def clean_upstream(self, target):
+        '''Remove files we don't want to import.'''
+        mozfile.remove(mozpath.join(target, '.gitattributes'))
+        mozfile.remove(mozpath.join(target, '.gitignore'))
+        mozfile.remove(mozpath.join(target, 'build', '.gitattributes'))
+        mozfile.remove(mozpath.join(target, 'build' ,'.gitignore'))
+
+    def generate_sources(self, target):
+        '''
+        Run the library's native build system to update ours.
+
+        Invoke configure for each supported platform to generate
+        appropriate config and header files, then invoke the
+        makefile to obtain a list of source files, writing
+        these out in the appropriate format for our build
+        system to use.
+        '''
+        config_dir = mozpath.join(target, 'config')
+        self.log(logging.INFO, 'rm_confg_dir', {}, 'rm -rf %s' % config_dir)
+        mozfile.remove(config_dir)
+        self.run_process(args=['./generate_sources_mozbuild.sh'],
+                         cwd=target)
+
+    def check_modified_files(self):
+        '''
+        Ensure that there aren't any uncommitted changes to files
+        in the working copy, since we're going to change some state
+        on the user.
+        '''
+        modified = self.repository.get_modified_files()
+        if modified:
+            self.log(logging.ERROR, 'modified_files', {},
+                     '''You have uncommitted changes to the following files:
+
+{files}
+
+Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`.
+'''.format(files='\n'.join(sorted(modified))))
+            sys.exit(1)
+
+    def vendor(self, revision, ignore_modified=False):
+        self.populate_logger()
+        self.log_manager.enable_unstructured()
+
+        if not ignore_modified:
+            self.check_modified_files()
+        if not revision:
+            revision = 'master'
+        commit, timestamp = self.upstream_commit(revision)
+
+        vendor_dir = mozpath.join(self.topsrcdir, 'third_party/aom')
+        self.fetch_and_unpack(commit, vendor_dir)
+        self.log(logging.INFO, 'clean_upstream', {},
+                 '''Removing unnecessary files.''')
+        self.clean_upstream(vendor_dir)
+        glue_dir = mozpath.join(self.topsrcdir, 'media/libaom')
+        self.log(logging.INFO, 'generate_sources', {},
+                 '''Generating build files...''')
+        self.generate_sources(glue_dir)
+        self.log(logging.INFO, 'update_readme', {},
+                 '''Updating README_MOZILLA.''')
+        self.update_readme(commit, timestamp, glue_dir)
+        self.repository.add_remove_files(vendor_dir)
+        self.log(logging.INFO, 'add_remove_files', {},
+                 '''Registering changes with version control.''')
+        self.repository.add_remove_files(vendor_dir)
+        self.repository.add_remove_files(glue_dir)
+        self.log(logging.INFO, 'done', {'revision': revision},
+                 '''Update to aom version '{revision}' ready to commit.''')