bug 1339706 - l10n-bumper should build platforms from in-tree files. r=rail a=release
authorAki Sasaki <asasaki@mozilla.com>
Thu, 23 Feb 2017 16:25:59 -0800
changeset 378685 5fb1aafa8fd310af8d868076b42dbff28ef09c8c
parent 378684 166b520f6e4cd78b8cdbbb49febc740bca491b20
child 378686 ddb1f78c74c155aabba863c176d9bd367aeba791
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrail, release
bugs1339706
milestone53.0a2
bug 1339706 - l10n-bumper should build platforms from in-tree files. r=rail a=release
testing/mozharness/configs/l10n_bumper/jamun.py
testing/mozharness/configs/l10n_bumper/mozilla-aurora.py
testing/mozharness/configs/l10n_bumper/mozilla-beta.py
testing/mozharness/configs/l10n_bumper/mozilla-central.py
testing/mozharness/scripts/l10n_bumper.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/jamun.py
@@ -0,0 +1,34 @@
+MULTI_REPO = "projects/jamun"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-aurora.py
@@ -0,0 +1,33 @@
+MULTI_REPO = "releases/mozilla-aurora"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-beta.py
@@ -0,0 +1,34 @@
+MULTI_REPO = "releases/mozilla-beta"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-central.py
@@ -0,0 +1,33 @@
+MULTI_REPO = "mozilla-central"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
new file mode 100755
--- /dev/null
+++ b/testing/mozharness/scripts/l10n_bumper.py
@@ -0,0 +1,307 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" l10n_bumper.py
+
+    Updates a gecko repo with up to date changesets from l10n.mozilla.org.
+
+    Specifically, it updates l10n-changesets.json which is used by mobile releases.
+
+    This is to allow for `mach taskgraph` to reference specific l10n revisions
+    without having to resort to task.extra or commandline base64 json hacks.
+"""
+import codecs
+import os
+import pprint
+import sys
+import time
+from urlparse import urlparse
+try:
+    import simplejson as json
+    assert json
+except ImportError:
+    import json
+
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.errors import HgErrorList
+from mozharness.base.vcs.vcsbase import VCSScript
+from mozharness.base.log import ERROR, FATAL
+
+
+class L10nBumper(VCSScript):
+    def __init__(self, require_config_file=True):
+        super(L10nBumper, self).__init__(
+            all_actions=[
+                'clobber',
+                'check-treestatus',
+                'checkout-gecko',
+                'bump-changesets',
+                'push',
+                'push-loop',
+            ],
+            default_actions=[
+                'push-loop',
+            ],
+            require_config_file=require_config_file,
+            # Default config options
+            config={
+                'treestatus_base_url': 'https://treestatus.mozilla-releng.net',
+                'log_max_rotate': 99,
+            }
+        )
+
+    # Helper methods {{{1
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+
+        abs_dirs = super(L10nBumper, self).query_abs_dirs()
+
+        abs_dirs.update({
+            'gecko_local_dir':
+            os.path.join(
+                abs_dirs['abs_work_dir'],
+                self.config.get('gecko_local_dir', os.path.basename(self.config['gecko_pull_url']))
+            ),
+        })
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def hg_commit(self, path, repo_path, message):
+        """
+        Commits changes in repo_path, with specified user and commit message
+        """
+        user = self.config['hg_user']
+        hg = self.query_exe('hg', return_type='list')
+        env = self.query_env(partial_env={'LANG': 'en_US.UTF-8'})
+        cmd = hg + ['add', path]
+        self.run_command(cmd, cwd=repo_path, env=env)
+        cmd = hg + ['commit', '-u', user, '-m', message]
+        status = self.run_command(cmd, cwd=repo_path, env=env)
+
+    def hg_push(self, repo_path):
+        hg = self.query_exe('hg', return_type='list')
+        command = hg + ["push", "-e",
+                        "ssh -oIdentityFile=%s -l %s" % (
+                            self.config["ssh_key"], self.config["ssh_user"],
+                        ),
+                        self.config["gecko_push_url"]]
+        status = self.run_command(command, cwd=repo_path,
+                                  error_list=HgErrorList)
+        if status != 0:
+            # We failed; get back to a known state so we can either retry
+            # or fail out and continue later.
+            self.run_command(hg + ["--config", "extensions.mq=",
+                                   "strip", "--no-backup", "outgoing()"],
+                             cwd=repo_path)
+            self.run_command(hg + ["up", "-C"],
+                             cwd=repo_path)
+            self.run_command(hg + ["--config", "extensions.purge=",
+                                   "purge", "--all"],
+                             cwd=repo_path)
+            return False
+        return True
+
+    def _read_json(self, path):
+        contents = self.read_from_file(path)
+        try:
+            json_contents = json.loads(contents)
+            return json_contents
+        except ValueError:
+            self.error("%s is invalid json!" % path)
+
+    def _read_version(self, path):
+        contents = self.read_from_file(path).split('\n')[0]
+        return contents.split('.')
+
+    def _build_locale_map(self, old_contents, new_contents):
+        locale_map = {}
+        for key in old_contents:
+            if key not in new_contents:
+                locale_map[key] = "removed"
+        for k,v in new_contents.items():
+            if old_contents.get(k, {}).get('revision') != v['revision']:
+                locale_map[k] = v['revision']
+            elif old_contents.get(k, {}).get('platforms') != v['platforms']:
+                locale_map[k] = v['platforms']
+        return locale_map
+
+    def _build_platform_dict(self, bump_config):
+        dirs = self.query_abs_dirs()
+        repo_path = dirs['gecko_local_dir']
+        platform_dict = {}
+        for platform_config in bump_config['platform_configs']:
+            path = os.path.join(repo_path, platform_config['path'])
+            self.info("Reading %s for %s locales..." % (path, platform_config['platforms']))
+            contents = self.read_from_file(path)
+            for locale in contents.splitlines():
+                platforms = platform_dict.get(locale, {}).get('platforms', [])
+                platforms = sorted(list(platform_config['platforms']) + platforms)
+                platform_dict[locale] = {'platforms': platforms}
+        self.info("Built platform_dict:\n%s" % pprint.pformat(platform_dict))
+        return platform_dict
+
+    def _build_revision_dict(self, bump_config, version_list):
+        self.info("Building revision dict...")
+        platform_dict = self._build_platform_dict(bump_config)
+        revision_dict = {}
+        if bump_config.get('revision_url'):
+            repl_dict = {
+                'MAJOR_VERSION': version_list[0],
+            }
+
+            url = bump_config['revision_url'] % repl_dict
+            path = self.download_file(url, error_level=FATAL)
+            revision_info = self.read_from_file(path)
+            self.info("Got %s" % revision_info)
+            for line in revision_info.splitlines():
+                locale, revision = line.split(' ')
+                if locale in platform_dict:
+                    revision_dict[locale] = platform_dict[locale]
+                    revision_dict[locale]['revision'] = revision
+        else:
+            for k, v in platform_dict.items():
+                v['revision'] = 'default'
+                revision_dict[k] = v
+        self.info("revision_dict:\n%s" % pprint.pformat(revision_dict))
+        return revision_dict
+
+    def build_commit_message(self, name, locale_map):
+        revisions = []
+        comments = ''
+        for locale, revision in sorted(locale_map.items()):
+            comments += "%s -> %s\n" % (locale, revision)
+        message = 'Bumping %s a=l10n-bump\n\n' % (
+            name,
+        )
+        message += comments
+        message = message.encode("utf-8")
+        return message
+
+    def query_treestatus(self):
+        "Return True if we can land based on treestatus"
+        c = self.config
+        dirs = self.query_abs_dirs()
+        tree = c.get('treestatus_tree', os.path.basename(c['gecko_pull_url'].rstrip("/")))
+        treestatus_url = "%s/trees/%s" % (c['treestatus_base_url'], tree)
+        treestatus_json = os.path.join(dirs['abs_work_dir'], 'treestatus.json')
+        if not os.path.exists(dirs['abs_work_dir']):
+            self.mkdir_p(dirs['abs_work_dir'])
+        self.rmtree(treestatus_json)
+
+        self.run_command(["curl", "--retry", "4", "-o", treestatus_json, treestatus_url], throw_exception=True)
+
+        treestatus = self._read_json(treestatus_json)
+        if treestatus['result']['status'] != 'closed':
+            self.info("treestatus is %s - assuming we can land" % repr(treestatus['result']['status']))
+            return True
+
+        return False
+
+    # Actions {{{1
+    def check_treestatus(self):
+        if not self.query_treestatus():
+            self.info("breaking early since treestatus is closed")
+            sys.exit(0)
+
+    def checkout_gecko(self):
+        c = self.config
+        dirs = self.query_abs_dirs()
+        dest = dirs['gecko_local_dir']
+        repos = [{
+            'repo': c['gecko_pull_url'],
+            'tag': c.get('gecko_tag', 'default'),
+            'dest': dest,
+            'vcs': 'hg',
+        }]
+        self.vcs_checkout_repos(repos)
+
+    def bump_changesets(self):
+        dirs = self.query_abs_dirs()
+        repo_path = dirs['gecko_local_dir']
+        version_path = os.path.join(repo_path, self.config['version_path'])
+        changes = False
+        version_list = self._read_version(version_path)
+        for bump_config in self.config['bump_configs']:
+            path = os.path.join(repo_path,
+                                bump_config['path'])
+            # For now, assume format == 'json'.  When we add desktop support,
+            # we may need to add flatfile support
+            if os.path.exists(path):
+                old_contents = self._read_json(path)
+            else:
+                old_contents = {}
+
+            new_contents = self._build_revision_dict(bump_config, version_list)
+
+            if new_contents == old_contents:
+                continue
+            # super basic sanity check
+            if not isinstance(new_contents, dict) or len(new_contents) < 5:
+                self.error("Cowardly refusing to land a broken-seeming changesets file!")
+                continue
+
+            # Write to disk
+            content_string = json.dumps(new_contents, sort_keys=True, indent=4)
+            fh = codecs.open(path, encoding='utf-8', mode='w+')
+            fh.write(content_string + "\n")
+            fh.close()
+
+            locale_map = self._build_locale_map(old_contents, new_contents)
+
+            # Commit
+            message = self.build_commit_message(bump_config['name'],
+                                                locale_map)
+            self.hg_commit(path, repo_path, message)
+            changes = True
+        return changes
+
+    def push(self):
+        dirs = self.query_abs_dirs()
+        repo_path = dirs['gecko_local_dir']
+        return self.hg_push(repo_path)
+
+    def push_loop(self):
+        max_retries = 5
+        for _ in range(max_retries):
+            changed = False
+            if not self.query_treestatus():
+                # Tree is closed; exit early to avoid a bunch of wasted time
+                self.info("breaking early since treestatus is closed")
+                break
+
+            self.checkout_gecko()
+            if self.bump_changesets():
+                changed = True
+
+            if not changed:
+                # Nothing changed, we're all done
+                self.info("No changes - all done")
+                break
+
+            if self.push():
+                # We did it! Hurray!
+                self.info("Great success!")
+                break
+            # If we're here, then the push failed. It also stripped any
+            # outgoing commits, so we should be in a pristine state again
+            # Empty our local cache of manifests so they get loaded again next
+            # time through this loop. This makes sure we get fresh upstream
+            # manifests, and avoids problems like bug 979080
+            self.device_manifests = {}
+
+            # Sleep before trying again
+            self.info("Sleeping 60 before trying again")
+            time.sleep(60)
+        else:
+            self.fatal("Didn't complete successfully (hit max_retries)")
+
+
+# __main__ {{{1
+if __name__ == '__main__':
+    bumper = L10nBumper()
+    bumper.run_and_exit()