author Jeff Walden <>
Tue, 19 Nov 2019 04:55:39 +0000
changeset 502538 b5c5ba07d3dbd0d07b66fa42a103f4df2c27d3a2
parent 500642 4257591c09c6586fc06f3837e66ed5be81bcee96
permissions -rw-r--r--
Bug 1596544 - intl_ValidateAndCanonicalizeUnicodeExtensionType should ignore the second |option| argument until it's needed to report an error. r=anba Differential Revision:

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, # You can obtain one at

from __future__ import absolute_import, print_function, unicode_literals

import logging
from mozbuild.base import (
import mozfile
import mozpack.path as mozpath
import os
import requests
import re
import sys
import tarfile
from urllib.parse import urlparse

class VendorAOM(MozbuildObject):
    def upstream_snapshot(self, revision):
        '''Construct a url for a tarball snapshot of the given revision.'''
        if 'googlesource' in self.repo_url:
            return mozpath.join(self.repo_url, '+archive', revision + '.tar.gz')
        elif 'github' in self.repo_url:
            return mozpath.join(self.repo_url, 'archive', revision + '.tar.gz')
            raise ValueError('Unknown git host, no snapshot lookup method')

    def upstream_commit(self, revision):
        '''Convert a revision to a git commit and timestamp.

        Ask the upstream repo to convert the requested revision to
        a git commit id and timestamp, so we can be precise in
        what we're vendoring.'''
        if 'googlesource' in self.repo_url:
            return self.upstream_googlesource_commit(revision)
        elif 'github' in self.repo_url:
            return self.upstream_github_commit(revision)
            raise ValueError('Unknown git host, no commit lookup method')

    def upstream_validate(self, url):
        '''Validate repository urls to make sure we can handle them.'''
        host = urlparse(url).netloc
        valid_domains = ('', '')
        if not any(filter(lambda domain: domain in host, valid_domains)):
            self.log(logging.ERROR, 'upstream_url', {},
                     '''Unsupported git host %s; cannot fetch snapshots.

Please set a repository url with --repo on either googlesource or github.''' % host)

    def upstream_googlesource_commit(self, revision):
        '''Query gitiles for a git commit and timestamp.'''
        url = mozpath.join(self.repo_url, '+', revision + '?format=JSON')
        self.log(logging.INFO, 'fetch', {'url': url},
                 'Fetching commit id from {url}')
        req = requests.get(url)
            info = req.json()
        except ValueError:
            # As of 2017 May, googlesource sends 4 garbage characters
            # at the beginning of the json response. Work around this.
            import json
            info = json.loads(req.text[4:])
        return (info['commit'], info['committer']['time'])

    def upstream_github_commit(self, revision):
        '''Query the github api for a git commit id and timestamp.'''
        github_api = ''
        repo = urlparse(self.repo_url).path[1:]
        url = mozpath.join(github_api, 'repos', repo, 'commits', revision)
        self.log(logging.INFO, 'fetch', {'url': url},
                 'Fetching commit id from {url}')
        req = requests.get(url)
        info = req.json()
        return (info['sha'], info['commit']['committer']['date'])

    def fetch_and_unpack(self, revision, target):
        '''Fetch and unpack upstream source'''
        url = self.upstream_snapshot(revision)
        self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}')
        prefix = 'aom-' + revision
        filename = prefix + '.tar.gz'
        with open(filename, 'wb') as f:
            req = requests.get(url, stream=True)
            for data in req.iter_content(4096):
        tar =
        bad_paths = filter(lambda name: name.startswith('/') or '..' in name,
        if any(bad_paths):
            raise Exception("Tar archive contains non-local paths,"
                            "e.g. '%s'" % bad_paths[0])
        self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target)
        self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.')
        # Github puts everything properly down a directory; move it up.
        if all(map(lambda name: name.startswith(prefix), tar.getnames())):
            tardir = mozpath.join(target, prefix)
            os.system('mv %s/* %s/.* %s' % (tardir, tardir, target))
        # Remove the tarball.

    def update_readme(self, revision, timestamp, target):
        filename = mozpath.join(target, 'README_MOZILLA')
        with open(filename) as f:
            readme =

        prefix = 'The git commit ID used was'
        if prefix in readme:
            new_readme = re.sub(prefix + ' [v\.a-f0-9]+.*$',
                                prefix + ' %s (%s).' % (revision, timestamp),
            new_readme = '%s\n\n%s %s.' % (readme, prefix, revision)

        prefix = 'The last update was pulled from'
        new_readme = re.sub(prefix + ' https*://.*',
                            prefix + ' %s' % self.repo_url,

        if readme != new_readme:
            with open(filename, 'w') as f:

    def clean_upstream(self, target):
        '''Remove files we don't want to import.'''
        mozfile.remove(mozpath.join(target, '.gitattributes'))
        mozfile.remove(mozpath.join(target, '.gitignore'))
        mozfile.remove(mozpath.join(target, 'build', '.gitattributes'))
        mozfile.remove(mozpath.join(target, 'build', '.gitignore'))

    def generate_sources(self, target):
        Run the library's native build system to update ours.

        Invoke configure for each supported platform to generate
        appropriate config and header files, then invoke the
        makefile to obtain a list of source files, writing
        these out in the appropriate format for our build
        system to use.
        config_dir = mozpath.join(target, 'config')
        self.log(logging.INFO, 'rm_confg_dir', {}, 'rm -rf %s' % config_dir)
                         cwd=target, log_name='generate_sources')

    def check_modified_files(self):
        Ensure that there aren't any uncommitted changes to files
        in the working copy, since we're going to change some state
        on the user.
        modified = self.repository.get_changed_files('M')
        if modified:
            self.log(logging.ERROR, 'modified_files', {},
                     '''You have uncommitted changes to the following files:


Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`.

    def vendor(self, revision, repo, ignore_modified=False):

        if not ignore_modified:
        if not revision:
            revision = 'master'
        if repo:
            self.repo_url = repo
            self.repo_url = ''

        commit, timestamp = self.upstream_commit(revision)

        vendor_dir = mozpath.join(self.topsrcdir, 'third_party/aom')
        self.fetch_and_unpack(commit, vendor_dir)
        self.log(logging.INFO, 'clean_upstream', {},
                 '''Removing unnecessary files.''')
        glue_dir = mozpath.join(self.topsrcdir, 'media/libaom')
        self.log(logging.INFO, 'generate_sources', {},
                 '''Generating build files...''')
        self.log(logging.INFO, 'update_readme', {},
                 '''Updating README_MOZILLA.''')
        self.update_readme(commit, timestamp, glue_dir)
        self.log(logging.INFO, 'add_remove_files', {},
                 '''Registering changes with version control.''')
        self.log(logging.INFO, 'done', {'revision': revision},
                 '''Update to aom version '{revision}' ready to commit.''')