bug 1339706 - l10n-bumper should build platforms from in-tree files. r=rail a=release
authorAki Sasaki <asasaki@mozilla.com>
Thu, 16 Feb 2017 14:17:03 -0800
changeset 372793 7fe88797039fe30fffce5a54b8a8abeb859bd039
parent 372646 a4fb1ed3d4f10410cd9e61e849716e872093edca
child 372794 9571f5beb2aeecd59aab1daf9438f1d496e8db59
push id10863
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 23:02:23 +0000
treeherdermozilla-aurora@0931190cd725 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrail, release
bugs1339706
milestone54.0a1
bug 1339706 - l10n-bumper should build platforms from in-tree files. r=rail a=release We were downloading the json from the dashboard, but the l10n team wants to discontinue that. Instead, we download the revisions from the dashboard for beta and jamun, and use 'default' revisions on central and aurora. We build the platform lists from all-locales and maemo-locales. I updated the mozilla-beta config to be live, but since we have the mozharness repo and revision pinned in puppet, we will be able to pin to a new revision and point at the jamun config file. MozReview-Commit-ID: JQcJKMVqcFr
testing/mozharness/configs/l10n_bumper/jamun.py
testing/mozharness/configs/l10n_bumper/mozilla-aurora.py
testing/mozharness/configs/l10n_bumper/mozilla-beta.py
testing/mozharness/configs/l10n_bumper/mozilla-central.py
testing/mozharness/scripts/l10n_bumper.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/jamun.py
@@ -0,0 +1,34 @@
+MULTI_REPO = "projects/jamun"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-aurora.py
@@ -0,0 +1,33 @@
+MULTI_REPO = "releases/mozilla-aurora"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
--- a/testing/mozharness/configs/l10n_bumper/mozilla-beta.py
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-beta.py
@@ -3,27 +3,32 @@ config = {
     "log_name": "l10n_bumper",
 
     "exes": {
         # Get around the https warnings
         "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
         "hgtool.py": ["/usr/local/bin/hgtool.py"],
     },
 
-#    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
-#    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
-    "gecko_pull_url": "https://hg.mozilla.org/projects/jamun",
-    "gecko_push_url": "ssh://hg.mozilla.org/projects/jamun",
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
 
     "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
     "ssh_key": "~/.ssh/ffxbld_rsa",
     "ssh_user": "ffxbld",
 
     "vcs_share_base": "/builds/hg-shared",
     "version_path": "browser/config/version.txt",
 
     "bump_configs": [{
         "path": "mobile/locales/l10n-changesets.json",
         "format": "json",
         "name": "Fennec l10n changesets",
-        "url": "https://l10n.mozilla.org/shipping/json-changesets?av=fennec%(MAJOR_VERSION)s&platforms=android&multi_android-multilocale_repo={}&multi_android-multilocale_rev=default&multi_android-multilocale_path=mobile/android/locales/maemo-locales".format(MULTI_REPO),
+        "revision_url": "https://l10n.mozilla.org/shipping/l10n-changesets?av=fennec%(MAJOR_VERSION)s",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
     }],
 }
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/l10n_bumper/mozilla-central.py
@@ -0,0 +1,33 @@
+MULTI_REPO = "mozilla-central"
+config = {
+    "log_name": "l10n_bumper",
+
+    "exes": {
+        # Get around the https warnings
+        "hg": ["/usr/local/bin/hg", "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
+        "hgtool.py": ["/usr/local/bin/hgtool.py"],
+    },
+
+    "gecko_pull_url": "https://hg.mozilla.org/{}".format(MULTI_REPO),
+    "gecko_push_url": "ssh://hg.mozilla.org/{}".format(MULTI_REPO),
+
+    "hg_user": "L10n Bumper Bot <release+l10nbumper@mozilla.com>",
+    "ssh_key": "~/.ssh/ffxbld_rsa",
+    "ssh_user": "ffxbld",
+
+    "vcs_share_base": "/builds/hg-shared",
+    "version_path": "browser/config/version.txt",
+
+    "bump_configs": [{
+        "path": "mobile/locales/l10n-changesets.json",
+        "format": "json",
+        "name": "Fennec l10n changesets",
+        "platform_configs": [{
+            "platforms": ["android-api-15", "android"],
+            "path": "mobile/android/locales/all-locales"
+        }, {
+            "platforms": ["android-multilocale"],
+            "path": "mobile/android/locales/maemo-locales"
+        }],
+    }],
+}
--- a/testing/mozharness/scripts/l10n_bumper.py
+++ b/testing/mozharness/scripts/l10n_bumper.py
@@ -24,17 +24,17 @@ try:
     assert json
 except ImportError:
     import json
 
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.base.errors import HgErrorList
 from mozharness.base.vcs.vcsbase import VCSScript
-from mozharness.base.log import ERROR
+from mozharness.base.log import ERROR, FATAL
 
 
 class L10nBumper(VCSScript):
     def __init__(self, require_config_file=True):
         super(L10nBumper, self).__init__(
             all_actions=[
                 'clobber',
                 'check-treestatus',
@@ -66,26 +66,27 @@ class L10nBumper(VCSScript):
             os.path.join(
                 abs_dirs['abs_work_dir'],
                 self.config.get('gecko_local_dir', os.path.basename(self.config['gecko_pull_url']))
             ),
         })
         self.abs_dirs = abs_dirs
         return self.abs_dirs
 
-    def hg_commit(self, repo_path, message):
+    def hg_commit(self, path, repo_path, message):
         """
         Commits changes in repo_path, with specified user and commit message
         """
         user = self.config['hg_user']
         hg = self.query_exe('hg', return_type='list')
+        env = self.query_env(partial_env={'LANG': 'en_US.UTF-8'})
+        cmd = hg + ['add', path]
+        self.run_command(cmd, cwd=repo_path, env=env)
         cmd = hg + ['commit', '-u', user, '-m', message]
-        env = self.query_env(partial_env={'LANG': 'en_US.UTF-8'})
         status = self.run_command(cmd, cwd=repo_path, env=env)
-        return status == 0
 
     def hg_push(self, repo_path):
         hg = self.query_exe('hg', return_type='list')
         command = hg + ["push", "-e",
                         "ssh -oIdentityFile=%s -l %s" % (
                             self.config["ssh_key"], self.config["ssh_user"],
                         ),
                         self.config["gecko_push_url"]]
@@ -118,20 +119,62 @@ class L10nBumper(VCSScript):
         return contents.split('.')
 
     def _build_locale_map(self, old_contents, new_contents):
         locale_map = {}
         for key in old_contents:
             if key not in new_contents:
                 locale_map[key] = "removed"
         for k,v in new_contents.items():
-            if old_contents.get(k) != v:
+            if old_contents.get(k, {}).get('revision') != v['revision']:
                 locale_map[k] = v['revision']
+            elif old_contents.get(k, {}).get('platforms') != v['platforms']:
+                locale_map[k] = v['platforms']
         return locale_map
 
+    def _build_platform_dict(self, bump_config):
+        dirs = self.query_abs_dirs()
+        repo_path = dirs['gecko_local_dir']
+        platform_dict = {}
+        for platform_config in bump_config['platform_configs']:
+            path = os.path.join(repo_path, platform_config['path'])
+            self.info("Reading %s for %s locales..." % (path, platform_config['platforms']))
+            contents = self.read_from_file(path)
+            for locale in contents.splitlines():
+                platforms = platform_dict.get(locale, {}).get('platforms', [])
+                platforms = sorted(list(platform_config['platforms']) + platforms)
+                platform_dict[locale] = {'platforms': platforms}
+        self.info("Built platform_dict:\n%s" % pprint.pformat(platform_dict))
+        return platform_dict
+
+    def _build_revision_dict(self, bump_config, version_list):
+        self.info("Building revision dict...")
+        platform_dict = self._build_platform_dict(bump_config)
+        revision_dict = {}
+        if bump_config.get('revision_url'):
+            repl_dict = {
+                'MAJOR_VERSION': version_list[0],
+            }
+
+            url = bump_config['revision_url'] % repl_dict
+            path = self.download_file(url, error_level=FATAL)
+            revision_info = self.read_from_file(path)
+            self.info("Got %s" % revision_info)
+            for line in revision_info.splitlines():
+                locale, revision = line.split(' ')
+                if locale in platform_dict:
+                    revision_dict[locale] = platform_dict[locale]
+                    revision_dict[locale]['revision'] = revision
+        else:
+            for k, v in platform_dict.items():
+                v['revision'] = 'default'
+                revision_dict[k] = v
+        self.info("revision_dict:\n%s" % pprint.pformat(revision_dict))
+        return revision_dict
+
     def build_commit_message(self, name, locale_map):
         revisions = []
         comments = ''
         for locale, revision in sorted(locale_map.items()):
             comments += "%s -> %s\n" % (locale, revision)
         message = 'Bumping %s a=l10n-bump\n\n' % (
             name,
         )
@@ -188,23 +231,18 @@ class L10nBumper(VCSScript):
                                 bump_config['path'])
             # For now, assume format == 'json'.  When we add desktop support,
             # we may need to add flatfile support
             if os.path.exists(path):
                 old_contents = self._read_json(path)
             else:
                 old_contents = {}
 
-            repl_dict = {
-                'MAJOR_VERSION': version_list[0],
-            }
+            new_contents = self._build_revision_dict(bump_config, version_list)
 
-            url = bump_config['url'] % repl_dict
-            new_contents = self.load_json_url(url)
-            self.info("Got %s" % pprint.pformat(new_contents))
             if new_contents == old_contents:
                 continue
             # super basic sanity check
             if not isinstance(new_contents, dict) or len(new_contents) < 5:
                 self.error("Cowardly refusing to land a broken-seeming changesets file!")
                 continue
 
             # Write to disk
@@ -213,17 +251,17 @@ class L10nBumper(VCSScript):
             fh.write(content_string + "\n")
             fh.close()
 
             locale_map = self._build_locale_map(old_contents, new_contents)
 
             # Commit
             message = self.build_commit_message(bump_config['name'],
                                                 locale_map)
-            self.hg_commit(repo_path, message)
+            self.hg_commit(path, repo_path, message)
             changes = True
         return changes
 
     def push(self):
         dirs = self.query_abs_dirs()
         repo_path = dirs['gecko_local_dir']
         return self.hg_push(repo_path)