Bug 1195865 - Upload android l10n repacks to Taskcluster; r=jlund
☠☠ backed out by 81f00129aedb ☠ ☠
authorMike Shal <mshal@mozilla.com>
Wed, 19 Aug 2015 15:28:43 -0400
changeset 259437 1c9a4b6596e673d0bb1f765a8fdd58b7b6f33b3a
parent 259436 cc9c846f859825c1baf602b40084ac1a4ad232bd
child 259438 81f00129aedb47b6d7f2cd2838b1a70311596d26
push id64249
push usermshal@mozilla.com
push dateWed, 26 Aug 2015 19:23:31 +0000
treeherdermozilla-inbound@1c9a4b6596e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlund
bugs1195865
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1195865 - Upload android l10n repacks to Taskcluster; r=jlund
testing/mozharness/mozharness/mozilla/l10n/locales.py
testing/mozharness/scripts/desktop_l10n.py
testing/mozharness/scripts/mobile_l10n.py
--- a/testing/mozharness/mozharness/mozilla/l10n/locales.py
+++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py
@@ -184,16 +184,25 @@ class LocalesMixin(ChunkingMixin):
                 'tag': tag,
                 'vcs': vcs
             })
         revs = self.vcs_checkout_repos(repo_list=locale_repos,
                                        parent_dir=parent_dir,
                                        tag_override=c.get('tag_override'))
         self.gecko_locale_revisions = revs
 
+    def query_repo(self):
+        # Find the name of our repository
+        mozilla_dir = self.config['mozilla_dir']
+        repo = None
+        for repository in self.config['repos']:
+            if repository.get('dest') == mozilla_dir:
+                repo = repository['repo']
+                break
+        return repo
 
 # GaiaLocalesMixin {{{1
 class GaiaLocalesMixin(object):
     gaia_locale_revisions = None
 
     def pull_gaia_locale_source(self, l10n_config, locales, base_dir):
         root = l10n_config['root']
         # urljoin will strip the last part of root if it doesn't end with "/"
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -156,16 +156,17 @@ class DesktopSingleLocale(LocalesMixin, 
         # fxbuild style:
         buildscript_kwargs = {
             'all_actions': [
                 "clobber",
                 "pull",
                 "list-locales",
                 "setup",
                 "repack",
+                "create-virtualenv",
                 "taskcluster-upload",
                 "funsize-props",
                 "submit-to-balrog",
                 "summary",
             ],
             'config': {
                 "buildbot_json_path": "buildprops.json",
                 "ignore_locales": ["en-US"],
@@ -1006,24 +1007,16 @@ class DesktopSingleLocale(LocalesMixin, 
         credentials = {}
         execfile(auth, credentials)
         client_id = credentials.get('taskcluster_clientId')
         access_token = credentials.get('taskcluster_accessToken')
         if not client_id or not access_token:
             self.warning('Skipping S3 file upload: No taskcluster credentials.')
             return
 
-        # We need to activate the virtualenv so that we can import taskcluster
-        # (and its dependent modules, like requests and hawk).  Normally we
-        # could create the virtualenv as an action, but due to some odd
-        # dependencies with query_build_env() being called from build(), which
-        # is necessary before the virtualenv can be created.
-        self.disable_mock()
-        self.create_virtualenv()
-        self.enable_mock()
         self.activate_virtualenv()
 
         # Enable Taskcluster debug logging, so at least we get some debug
         # messages while we are testing uploads.
         logging.getLogger('taskcluster').setLevel(logging.DEBUG)
 
         branch = self.config['branch']
         platform = self.config['platform']
--- a/testing/mozharness/scripts/mobile_l10n.py
+++ b/testing/mozharness/scripts/mobile_l10n.py
@@ -10,16 +10,17 @@ This currently supports nightly and rele
 Android.  This also creates nightly updates.
 """
 
 from copy import deepcopy
 import os
 import re
 import subprocess
 import sys
+import shlex
 
 try:
     import simplejson as json
     assert json
 except ImportError:
     import json
 
 # load modules from parent dir
@@ -32,22 +33,25 @@ from mozharness.mozilla.buildbot import 
 from mozharness.mozilla.purge import PurgeMixin
 from mozharness.mozilla.release import ReleaseMixin
 from mozharness.mozilla.signing import MobileSigningMixin
 from mozharness.mozilla.tooltool import TooltoolMixin
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.l10n.locales import LocalesMixin
 from mozharness.mozilla.mock import MockMixin
 from mozharness.mozilla.updates.balrog import BalrogMixin
+from mozharness.base.python import VirtualenvMixin
+from mozharness.mozilla.taskcluster_helper import Taskcluster
 
 
 # MobileSingleLocale {{{1
 class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin,
                          MobileSigningMixin, TransferMixin, TooltoolMixin,
-                         BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin):
+                         BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin,
+                         VirtualenvMixin):
     config_options = [[
         ['--locale', ],
         {"action": "extend",
          "dest": "locales",
          "type": "string",
          "help": "Specify the locale(s) to sign and update"
          }
     ], [
@@ -99,31 +103,45 @@ class MobileSingleLocale(MockMixin, Loca
         {"action": "store",
          "dest": "total_locale_chunks",
          "type": "int",
          "help": "Specify the total number of chunks of locales"
          }
     ]]
 
     def __init__(self, require_config_file=True):
-        LocalesMixin.__init__(self)
-        MercurialScript.__init__(
-            self,
-            config_options=self.config_options,
-            all_actions=[
+        buildscript_kwargs = {
+            'all_actions': [
                 "clobber",
                 "pull",
                 "list-locales",
                 "setup",
                 "repack",
                 "upload-repacks",
+                "create-virtualenv",
+                "taskcluster-upload",
                 "submit-to-balrog",
                 "summary",
             ],
-            require_config_file=require_config_file
+            'config': {
+                'taskcluster_credentials_file': 'oauth.txt',
+                'virtualenv_modules': [
+                    'requests==2.2.1',
+                    'PyHawk-with-a-single-extra-commit==0.1.5',
+                    'taskcluster==0.0.15',
+                ],
+                'virtualenv_path': 'venv',
+            },
+        }
+        LocalesMixin.__init__(self)
+        MercurialScript.__init__(
+            self,
+            config_options=self.config_options,
+            require_config_file=require_config_file,
+            **buildscript_kwargs
         )
         self.base_package_name = None
         self.buildid = None
         self.make_ident_output = None
         self.repack_env = None
         self.revision = None
         self.upload_env = None
         self.version = None
@@ -442,16 +460,79 @@ class MobileSingleLocale(MockMixin, Loca
             if status:
                 self.add_failure(locale, message="Errors verifying %s binary!" % locale)
                 # No need to rm because upload is per-locale
                 continue
             success_count += 1
         self.summarize_success_count(success_count, total_count,
                                      message="Repacked %d of %d binaries successfully.")
 
+    def taskcluster_upload(self):
+        auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file'])
+        credentials = {}
+        execfile(auth, credentials)
+        client_id = credentials.get('taskcluster_clientId')
+        access_token = credentials.get('taskcluster_accessToken')
+        if not client_id or not access_token:
+            self.warning('Skipping S3 file upload: No taskcluster credentials.')
+            return
+
+        self.activate_virtualenv()
+
+        dirs = self.query_abs_dirs()
+        locales = self.query_locales()
+        make = self.query_exe("make")
+        upload_env = self.query_upload_env()
+        cwd = dirs['abs_locales_dir']
+        branch = self.config['branch']
+        revision = self.query_revision()
+        repo = self.query_repo()
+        pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool')
+        routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
+                                   'testing/taskcluster/routes.json')
+        with open(routes_json) as f:
+            contents = json.load(f)
+            templates = contents['l10n']
+
+        for locale in locales:
+            output = self.get_output_from_command_m(
+                "%s echo-variable-UPLOAD_FILES AB_CD=%s" % (make, locale),
+                cwd=cwd,
+                env=upload_env,
+            )
+            files = shlex.split(output)
+            abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files]
+
+            routes = []
+            fmt = {
+                'index': self.config.get('taskcluster_index', 'index.garbage.staging'),
+                'project': branch,
+                'head_rev': revision,
+                'build_product': self.config['stage_product'],
+                'build_name': self.query_build_name(),
+                'build_type': self.query_build_type(),
+                'locale': locale,
+            }
+            for template in templates:
+                routes.append(template.format(**fmt))
+
+            self.info('Using routes: %s' % routes)
+            tc = Taskcluster(branch,
+                             pushinfo.pushdate, # Use pushdate as the rank
+                             client_id,
+                             access_token,
+                             self.log_obj,
+                             )
+            task = tc.create_task(routes)
+            tc.claim_task(task)
+
+            for upload_file in abs_files:
+                tc.create_artifact(task, upload_file)
+            tc.report_completed(task)
+
     def upload_repacks(self):
         c = self.config
         dirs = self.query_abs_dirs()
         locales = self.query_locales()
         make = self.query_exe("make")
         base_package_name = self.query_base_package_name()
         version = self.query_version()
         upload_env = self.query_upload_env()