Bug 1195865 - Upload android l10n repacks to Taskcluster; r=jlund
authorMike Shal <mshal@mozilla.com>
Wed, 19 Aug 2015 15:28:43 -0400
changeset 294180 53ac5605d91a66c3faa63ce21d7caab41170af6a
parent 294179 c57395522d53ed02fd64a395f70ca1c9cdffccaa
child 294181 444ea60893cde03b26b6eb2d7af33aef5d968a06
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlund
bugs1195865
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1195865 - Upload android l10n repacks to Taskcluster; r=jlund
testing/mozharness/mozharness/mozilla/l10n/locales.py
testing/mozharness/scripts/desktop_l10n.py
testing/mozharness/scripts/mobile_l10n.py
--- a/testing/mozharness/mozharness/mozilla/l10n/locales.py
+++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py
@@ -184,16 +184,25 @@ class LocalesMixin(ChunkingMixin):
                 'tag': tag,
                 'vcs': vcs
             })
         revs = self.vcs_checkout_repos(repo_list=locale_repos,
                                        parent_dir=parent_dir,
                                        tag_override=c.get('tag_override'))
         self.gecko_locale_revisions = revs
 
+    def query_l10n_repo(self):
+        # Find the name of our repository
+        mozilla_dir = self.config['mozilla_dir']
+        repo = None
+        for repository in self.config['repos']:
+            if repository.get('dest') == mozilla_dir:
+                repo = repository['repo']
+                break
+        return repo
 
 # GaiaLocalesMixin {{{1
 class GaiaLocalesMixin(object):
     gaia_locale_revisions = None
 
     def pull_gaia_locale_source(self, l10n_config, locales, base_dir):
         root = l10n_config['root']
         # urljoin will strip the last part of root if it doesn't end with "/"
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -156,16 +156,17 @@ class DesktopSingleLocale(LocalesMixin, 
         # fxbuild style:
         buildscript_kwargs = {
             'all_actions': [
                 "clobber",
                 "pull",
                 "list-locales",
                 "setup",
                 "repack",
+                "create-virtualenv",
                 "taskcluster-upload",
                 "funsize-props",
                 "submit-to-balrog",
                 "summary",
             ],
             'config': {
                 "buildbot_json_path": "buildprops.json",
                 "ignore_locales": ["en-US"],
@@ -986,54 +987,36 @@ class DesktopSingleLocale(LocalesMixin, 
             'branch': self.config['branch'],
             'appName': self.config['appName'],
             'platform': self.config['platform'],
         }
         self.info('funsize info: %s' % funsize_info)
         self.set_buildbot_property('funsize_info', json.dumps(funsize_info),
                                    write_to_file=True)
 
-    def query_repo(self):
-        # Find the name of our repository
-        mozilla_dir = self.config['mozilla_dir']
-        repo = None
-        for repository in self.config['repos']:
-            if repository.get('dest') == mozilla_dir:
-                repo = repository['repo']
-                break
-        return repo
-
     def taskcluster_upload(self):
         auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file'])
         credentials = {}
         execfile(auth, credentials)
         client_id = credentials.get('taskcluster_clientId')
         access_token = credentials.get('taskcluster_accessToken')
         if not client_id or not access_token:
             self.warning('Skipping S3 file upload: No taskcluster credentials.')
             return
 
-        # We need to activate the virtualenv so that we can import taskcluster
-        # (and its dependent modules, like requests and hawk).  Normally we
-        # could create the virtualenv as an action, but due to some odd
-        # dependencies with query_build_env() being called from build(), which
-        # is necessary before the virtualenv can be created.
-        self.disable_mock()
-        self.create_virtualenv()
-        self.enable_mock()
         self.activate_virtualenv()
 
         # Enable Taskcluster debug logging, so at least we get some debug
         # messages while we are testing uploads.
         logging.getLogger('taskcluster').setLevel(logging.DEBUG)
 
         branch = self.config['branch']
         platform = self.config['platform']
         revision = self._query_revision()
-        repo = self.query_repo()
+        repo = self.query_l10n_repo()
         if not repo:
             self.fatal("Unable to determine repository for querying the push info.")
         pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool')
 
         routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
                                    'testing/taskcluster/routes.json')
         with open(routes_json) as f:
             contents = json.load(f)
--- a/testing/mozharness/scripts/mobile_l10n.py
+++ b/testing/mozharness/scripts/mobile_l10n.py
@@ -10,16 +10,17 @@ This currently supports nightly and rele
 Android.  This also creates nightly updates.
 """
 
 from copy import deepcopy
 import os
 import re
 import subprocess
 import sys
+import shlex
 
 try:
     import simplejson as json
     assert json
 except ImportError:
     import json
 
 # load modules from parent dir
@@ -32,22 +33,25 @@ from mozharness.mozilla.buildbot import 
 from mozharness.mozilla.purge import PurgeMixin
 from mozharness.mozilla.release import ReleaseMixin
 from mozharness.mozilla.signing import MobileSigningMixin
 from mozharness.mozilla.tooltool import TooltoolMixin
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.l10n.locales import LocalesMixin
 from mozharness.mozilla.mock import MockMixin
 from mozharness.mozilla.updates.balrog import BalrogMixin
+from mozharness.base.python import VirtualenvMixin
+from mozharness.mozilla.taskcluster_helper import Taskcluster
 
 
 # MobileSingleLocale {{{1
 class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin,
                          MobileSigningMixin, TransferMixin, TooltoolMixin,
-                         BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin):
+                         BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin,
+                         VirtualenvMixin):
     config_options = [[
         ['--locale', ],
         {"action": "extend",
          "dest": "locales",
          "type": "string",
          "help": "Specify the locale(s) to sign and update"
          }
     ], [
@@ -99,31 +103,45 @@ class MobileSingleLocale(MockMixin, Loca
         {"action": "store",
          "dest": "total_locale_chunks",
          "type": "int",
          "help": "Specify the total number of chunks of locales"
          }
     ]]
 
     def __init__(self, require_config_file=True):
-        LocalesMixin.__init__(self)
-        MercurialScript.__init__(
-            self,
-            config_options=self.config_options,
-            all_actions=[
+        buildscript_kwargs = {
+            'all_actions': [
                 "clobber",
                 "pull",
                 "list-locales",
                 "setup",
                 "repack",
                 "upload-repacks",
+                "create-virtualenv",
+                "taskcluster-upload",
                 "submit-to-balrog",
                 "summary",
             ],
-            require_config_file=require_config_file
+            'config': {
+                'taskcluster_credentials_file': 'oauth.txt',
+                'virtualenv_modules': [
+                    'requests==2.2.1',
+                    'PyHawk-with-a-single-extra-commit==0.1.5',
+                    'taskcluster==0.0.15',
+                ],
+                'virtualenv_path': 'venv',
+            },
+        }
+        LocalesMixin.__init__(self)
+        MercurialScript.__init__(
+            self,
+            config_options=self.config_options,
+            require_config_file=require_config_file,
+            **buildscript_kwargs
         )
         self.base_package_name = None
         self.buildid = None
         self.make_ident_output = None
         self.repack_env = None
         self.revision = None
         self.upload_env = None
         self.version = None
@@ -442,16 +460,79 @@ class MobileSingleLocale(MockMixin, Loca
             if status:
                 self.add_failure(locale, message="Errors verifying %s binary!" % locale)
                 # No need to rm because upload is per-locale
                 continue
             success_count += 1
         self.summarize_success_count(success_count, total_count,
                                      message="Repacked %d of %d binaries successfully.")
 
+    def taskcluster_upload(self):
+        auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file'])
+        credentials = {}
+        execfile(auth, credentials)
+        client_id = credentials.get('taskcluster_clientId')
+        access_token = credentials.get('taskcluster_accessToken')
+        if not client_id or not access_token:
+            self.warning('Skipping S3 file upload: No taskcluster credentials.')
+            return
+
+        self.activate_virtualenv()
+
+        dirs = self.query_abs_dirs()
+        locales = self.query_locales()
+        make = self.query_exe("make")
+        upload_env = self.query_upload_env()
+        cwd = dirs['abs_locales_dir']
+        branch = self.config['branch']
+        revision = self.query_revision()
+        repo = self.query_l10n_repo()
+        pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool')
+        routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
+                                   'testing/taskcluster/routes.json')
+        with open(routes_json) as f:
+            contents = json.load(f)
+            templates = contents['l10n']
+
+        for locale in locales:
+            output = self.get_output_from_command_m(
+                "%s echo-variable-UPLOAD_FILES AB_CD=%s" % (make, locale),
+                cwd=cwd,
+                env=upload_env,
+            )
+            files = shlex.split(output)
+            abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files]
+
+            routes = []
+            fmt = {
+                'index': self.config.get('taskcluster_index', 'index.garbage.staging'),
+                'project': branch,
+                'head_rev': revision,
+                'build_product': self.config['stage_product'],
+                'build_name': self.query_build_name(),
+                'build_type': self.query_build_type(),
+                'locale': locale,
+            }
+            for template in templates:
+                routes.append(template.format(**fmt))
+
+            self.info('Using routes: %s' % routes)
+            tc = Taskcluster(branch,
+                             pushinfo.pushdate, # Use pushdate as the rank
+                             client_id,
+                             access_token,
+                             self.log_obj,
+                             )
+            task = tc.create_task(routes)
+            tc.claim_task(task)
+
+            for upload_file in abs_files:
+                tc.create_artifact(task, upload_file)
+            tc.report_completed(task)
+
     def upload_repacks(self):
         c = self.config
         dirs = self.query_abs_dirs()
         locales = self.query_locales()
         make = self.query_exe("make")
         base_package_name = self.query_base_package_name()
         version = self.query_version()
         upload_env = self.query_upload_env()