author | Mike Shal <mshal@mozilla.com> |
Wed, 19 Aug 2015 15:28:43 -0400 | |
changeset 261553 | 53ac5605d91a66c3faa63ce21d7caab41170af6a |
parent 261552 | c57395522d53ed02fd64a395f70ca1c9cdffccaa |
child 261554 | 444ea60893cde03b26b6eb2d7af33aef5d968a06 |
push id | 64765 |
push user | mshal@mozilla.com |
push date | Wed, 09 Sep 2015 12:26:57 +0000 |
treeherder | mozilla-inbound@53ac5605d91a [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | jlund |
bugs | 1195865 |
milestone | 43.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/mozharness/mozharness/mozilla/l10n/locales.py +++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py @@ -184,16 +184,25 @@ class LocalesMixin(ChunkingMixin): 'tag': tag, 'vcs': vcs }) revs = self.vcs_checkout_repos(repo_list=locale_repos, parent_dir=parent_dir, tag_override=c.get('tag_override')) self.gecko_locale_revisions = revs + def query_l10n_repo(self): + # Find the name of our repository + mozilla_dir = self.config['mozilla_dir'] + repo = None + for repository in self.config['repos']: + if repository.get('dest') == mozilla_dir: + repo = repository['repo'] + break + return repo # GaiaLocalesMixin {{{1 class GaiaLocalesMixin(object): gaia_locale_revisions = None def pull_gaia_locale_source(self, l10n_config, locales, base_dir): root = l10n_config['root'] # urljoin will strip the last part of root if it doesn't end with "/"
--- a/testing/mozharness/scripts/desktop_l10n.py +++ b/testing/mozharness/scripts/desktop_l10n.py @@ -156,16 +156,17 @@ class DesktopSingleLocale(LocalesMixin, # fxbuild style: buildscript_kwargs = { 'all_actions': [ "clobber", "pull", "list-locales", "setup", "repack", + "create-virtualenv", "taskcluster-upload", "funsize-props", "submit-to-balrog", "summary", ], 'config': { "buildbot_json_path": "buildprops.json", "ignore_locales": ["en-US"], @@ -986,54 +987,36 @@ class DesktopSingleLocale(LocalesMixin, 'branch': self.config['branch'], 'appName': self.config['appName'], 'platform': self.config['platform'], } self.info('funsize info: %s' % funsize_info) self.set_buildbot_property('funsize_info', json.dumps(funsize_info), write_to_file=True) - def query_repo(self): - # Find the name of our repository - mozilla_dir = self.config['mozilla_dir'] - repo = None - for repository in self.config['repos']: - if repository.get('dest') == mozilla_dir: - repo = repository['repo'] - break - return repo - def taskcluster_upload(self): auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file']) credentials = {} execfile(auth, credentials) client_id = credentials.get('taskcluster_clientId') access_token = credentials.get('taskcluster_accessToken') if not client_id or not access_token: self.warning('Skipping S3 file upload: No taskcluster credentials.') return - # We need to activate the virtualenv so that we can import taskcluster - # (and its dependent modules, like requests and hawk). Normally we - # could create the virtualenv as an action, but due to some odd - # dependencies with query_build_env() being called from build(), which - # is necessary before the virtualenv can be created. - self.disable_mock() - self.create_virtualenv() - self.enable_mock() self.activate_virtualenv() # Enable Taskcluster debug logging, so at least we get some debug # messages while we are testing uploads. logging.getLogger('taskcluster').setLevel(logging.DEBUG) branch = self.config['branch'] platform = self.config['platform'] revision = self._query_revision() - repo = self.query_repo() + repo = self.query_l10n_repo() if not repo: self.fatal("Unable to determine repository for querying the push info.") pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool') routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'], 'testing/taskcluster/routes.json') with open(routes_json) as f: contents = json.load(f)
--- a/testing/mozharness/scripts/mobile_l10n.py +++ b/testing/mozharness/scripts/mobile_l10n.py @@ -10,16 +10,17 @@ This currently supports nightly and rele Android. This also creates nightly updates. """ from copy import deepcopy import os import re import subprocess import sys +import shlex try: import simplejson as json assert json except ImportError: import json # load modules from parent dir @@ -32,22 +33,25 @@ from mozharness.mozilla.buildbot import from mozharness.mozilla.purge import PurgeMixin from mozharness.mozilla.release import ReleaseMixin from mozharness.mozilla.signing import MobileSigningMixin from mozharness.mozilla.tooltool import TooltoolMixin from mozharness.base.vcs.vcsbase import MercurialScript from mozharness.mozilla.l10n.locales import LocalesMixin from mozharness.mozilla.mock import MockMixin from mozharness.mozilla.updates.balrog import BalrogMixin +from mozharness.base.python import VirtualenvMixin +from mozharness.mozilla.taskcluster_helper import Taskcluster # MobileSingleLocale {{{1 class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin, MobileSigningMixin, TransferMixin, TooltoolMixin, - BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin): + BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin, + VirtualenvMixin): config_options = [[ ['--locale', ], {"action": "extend", "dest": "locales", "type": "string", "help": "Specify the locale(s) to sign and update" } ], [ @@ -99,31 +103,45 @@ class MobileSingleLocale(MockMixin, Loca {"action": "store", "dest": "total_locale_chunks", "type": "int", "help": "Specify the total number of chunks of locales" } ]] def __init__(self, require_config_file=True): - LocalesMixin.__init__(self) - MercurialScript.__init__( - self, - config_options=self.config_options, - all_actions=[ + buildscript_kwargs = { + 'all_actions': [ "clobber", "pull", "list-locales", "setup", "repack", "upload-repacks", + "create-virtualenv", + "taskcluster-upload", "submit-to-balrog", "summary", ], - require_config_file=require_config_file + 'config': { + 'taskcluster_credentials_file': 'oauth.txt', + 'virtualenv_modules': [ + 'requests==2.2.1', + 'PyHawk-with-a-single-extra-commit==0.1.5', + 'taskcluster==0.0.15', + ], + 'virtualenv_path': 'venv', + }, + } + LocalesMixin.__init__(self) + MercurialScript.__init__( + self, + config_options=self.config_options, + require_config_file=require_config_file, + **buildscript_kwargs ) self.base_package_name = None self.buildid = None self.make_ident_output = None self.repack_env = None self.revision = None self.upload_env = None self.version = None @@ -442,16 +460,79 @@ class MobileSingleLocale(MockMixin, Loca if status: self.add_failure(locale, message="Errors verifying %s binary!" % locale) # No need to rm because upload is per-locale continue success_count += 1 self.summarize_success_count(success_count, total_count, message="Repacked %d of %d binaries successfully.") + def taskcluster_upload(self): + auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file']) + credentials = {} + execfile(auth, credentials) + client_id = credentials.get('taskcluster_clientId') + access_token = credentials.get('taskcluster_accessToken') + if not client_id or not access_token: + self.warning('Skipping S3 file upload: No taskcluster credentials.') + return + + self.activate_virtualenv() + + dirs = self.query_abs_dirs() + locales = self.query_locales() + make = self.query_exe("make") + upload_env = self.query_upload_env() + cwd = dirs['abs_locales_dir'] + branch = self.config['branch'] + revision = self.query_revision() + repo = self.query_l10n_repo() + pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool') + routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'], + 'testing/taskcluster/routes.json') + with open(routes_json) as f: + contents = json.load(f) + templates = contents['l10n'] + + for locale in locales: + output = self.get_output_from_command_m( + "%s echo-variable-UPLOAD_FILES AB_CD=%s" % (make, locale), + cwd=cwd, + env=upload_env, + ) + files = shlex.split(output) + abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files] + + routes = [] + fmt = { + 'index': self.config.get('taskcluster_index', 'index.garbage.staging'), + 'project': branch, + 'head_rev': revision, + 'build_product': self.config['stage_product'], + 'build_name': self.query_build_name(), + 'build_type': self.query_build_type(), + 'locale': locale, + } + for template in templates: + routes.append(template.format(**fmt)) + + self.info('Using routes: %s' % routes) + tc = Taskcluster(branch, + pushinfo.pushdate, # Use pushdate as the rank + client_id, + access_token, + self.log_obj, + ) + task = tc.create_task(routes) + tc.claim_task(task) + + for upload_file in abs_files: + tc.create_artifact(task, upload_file) + tc.report_completed(task) + def upload_repacks(self): c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") base_package_name = self.query_base_package_name() version = self.query_version() upload_env = self.query_upload_env()