author | Wander Lairson Costa <wcosta@mozilla.com> |
Tue, 02 May 2017 16:12:51 -0300 | |
changeset 356595 | 971b9548d84c31878468f4890d283377987e2cf4 |
parent 356594 | 0c8c40eddb5eb6c4dbc91457159e16dc3a8ea800 |
child 356596 | 6b5f20da8bc90f3145980234cdc488e516c63325 |
push id | 89950 |
push user | cbook@mozilla.com |
push date | Fri, 05 May 2017 13:26:37 +0000 |
treeherder | mozilla-inbound@e95e5825fd40 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | dustin |
bugs | 1350413, 1338651 |
milestone | 55.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/taskcluster/ci/build/macosx.yml +++ b/taskcluster/ci/build/macosx.yml @@ -18,33 +18,32 @@ macosx64/debug: - builds/releng_base_mac_64_cross_builds.py - balrog/production.py script: "mozharness/scripts/fx_desktop_build.py" secrets: true custom-build-variant-cfg: cross-debug tooltool-downloads: internal macosx64/opt: - description: "MacOS X x64 Cross-compile" + description: "MacOS X x64" index: product: firefox job-name: macosx64-opt treeherder: platform: osx-10-7/opt symbol: tc(B) tier: 2 - worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64 + worker-type: buildbot-bridge/buildbot-bridge worker: - implementation: docker-worker - max-run-time: 36000 + implementation: buildbot-bridge run: using: mozharness actions: [get-secrets build generate-build-stats update] config: - - builds/releng_base_mac_64_cross_builds.py + - builds/releng_base_mac_64_builds.py - balrog/production.py script: "mozharness/scripts/fx_desktop_build.py" secrets: true tooltool-downloads: internal macosx64-qr/debug: description: "MacOS X x64 QuantumRender" index:
--- a/taskcluster/taskgraph/morph.py +++ b/taskcluster/taskgraph/morph.py @@ -152,12 +152,97 @@ def add_index_tasks(taskgraph, label_to_ if added: taskgraph, label_to_taskid = amend_taskgraph( taskgraph, label_to_taskid, added) logger.info('Added {} index tasks'.format(len(added))) return taskgraph, label_to_taskid +def make_s3_uploader_task(parent_task): + if parent_task.task['payload']['sourcestamp']['branch'] == 'try': + worker_type = 'buildbot-try' + else: + worker_type = 'buildbot' + + task_def = { + # The null-provisioner and buildbot worker type don't actually exist. + # So this task doesn't actually run - we just need to create the task so + # we have something to attach artifacts to. + "provisionerId": "null-provisioner", + "workerType": worker_type, + "created": {'relative-datestamp': '0 seconds'}, + "deadline": parent_task.task['deadline'], + "routes": [], + "payload": {}, + "metadata": { + "name": "Buildbot/mozharness S3 uploader", + "description": "Upload outputs of buildbot/mozharness builds to S3", + "owner": "mshal@mozilla.com", + "source": "http://hg.mozilla.org/build/mozharness/", + } + } + label = 's3-uploader-{}'.format(parent_task.label) + dependencies = {} + task = Task(kind='misc', label=label, attributes={}, task=task_def, + dependencies=dependencies) + task.task_id = parent_task.task['payload']['properties']['upload_to_task_id'] + return task + + +def update_test_tasks(taskid, build_taskid, taskgraph): + """Tests task must download artifacts from uploader task. + + Notice they don't need to depend on uploader task because it finishes + before the build task. + """ + # Notice we handle buildbot-bridge, native, and generic-worker payloads + # We can do better here in terms of graph searching + # We could do post order search and stop as soon as we + # reach the build task. Not worring about it because this is + # (supposed to be) a temporary solution. + for task in taskgraph.tasks.itervalues(): + if build_taskid in task.task.get('dependencies', []): + payload = task.task['payload'] + if 'command' in payload: + try: + payload['command'] = [ + cmd.replace(build_taskid, taskid) for cmd in payload['command'] + ] + except AttributeError: + # generic-worker command attribute is an list of lists + payload['command'] = [ + [cmd.replace(build_taskid, taskid) for cmd in x] + for x in payload['command'] + ] + if 'mounts' in payload: + for mount in payload['mounts']: + if mount.get('content', {}).get('taskId', '') == build_taskid: + mount['content']['taskId'] = taskid + if 'env' in payload: + payload['env'] = { + k: v.replace(build_taskid, taskid) for k, v in payload['env'].iteritems() + } + if 'properties' in payload: + payload['properties']['parent_task_id'] = taskid + + +def add_s3_uploader_task(taskgraph, label_to_taskid): + """The S3 uploader task is used by mozharness to upload buildbot artifacts.""" + for task in taskgraph.tasks.itervalues(): + if 'upload_to_task_id' in task.task.get('payload', {}).get('properties', {}): + added = make_s3_uploader_task(task) + taskgraph, label_to_taskid = amend_taskgraph( + taskgraph, label_to_taskid, [added]) + update_test_tasks(added.task_id, task.task_id, taskgraph) + logger.info('Added s3-uploader task') + return taskgraph, label_to_taskid + + def morph(taskgraph, label_to_taskid): """Apply all morphs""" - taskgraph, label_to_taskid = add_index_tasks(taskgraph, label_to_taskid) + morphs = [ + add_index_tasks, + add_s3_uploader_task, + ] + for m in morphs: + taskgraph, label_to_taskid = m(taskgraph, label_to_taskid) return taskgraph, label_to_taskid
--- a/taskcluster/taskgraph/transforms/job/mozharness.py +++ b/taskcluster/taskgraph/transforms/job/mozharness.py @@ -4,16 +4,17 @@ """ Support for running jobs via mozharness. Ideally, most stuff gets run this way, and certainly anything using mozharness should use this approach. """ from __future__ import absolute_import, print_function, unicode_literals +import slugid from textwrap import dedent from taskgraph.util.schema import Schema from voluptuous import Required, Optional, Any from taskgraph.transforms.job import run_job_using from taskgraph.transforms.job.common import ( @@ -239,8 +240,32 @@ def mozharness_on_generic_worker(config, r'icacls z:\build /grant *S-1-1-0:D /L', r'cd /d z:\build', ]) worker['command'].extend([ ' '.join(hg_command), ' '.join(mh_command) ]) + + +@run_job_using('buildbot-bridge', 'mozharness', schema=mozharness_run_schema) +def mozharness_on_buildbot_bridge(config, job, taskdesc): + run = job['run'] + worker = taskdesc['worker'] + branch = config.params['project'] + product = run.get('index', {}).get('product', 'firefox') + + worker.pop('env', None) + + worker.update({ + 'buildername': 'OS X 10.7 {} build'.format(branch), + 'sourcestamp': { + 'branch': branch, + 'repository': config.params['head_repository'], + 'revision': config.params['head_rev'], + }, + 'properties': { + 'product': product, + 'who': config.params['owner'], + 'upload_to_task_id': slugid.nice(), + } + })
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py +++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py @@ -205,27 +205,35 @@ def mozharness_test_on_generic_worker(co 'path': 'logs/log_raw.log', 'type': 'file' }, { 'name': 'public/logs/log_warning.log', 'path': 'logs/log_warning.log', 'type': 'file' }, - { + ] + + # jittest doesn't have blob_upload_dir + if test['test-name'] != 'jittest': + artifacts.append({ 'name': 'public/test_info', 'path': 'build/blobber_upload_dir', 'type': 'directory' - } - ] + }) build_platform = taskdesc['attributes']['build_platform'] + build_type = taskdesc['attributes']['build_type'] - target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) \ - if build_platform.startswith('win') else 'target' + if build_platform.startswith('win'): + target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) + elif build_platform == 'macosx64' and build_type == 'opt': + target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') + else: + target = 'target' installer_url = get_artifact_url('<build>', mozharness['build-artifact-name']) test_packages_url = get_artifact_url( '<build>', 'public/build/{}.test_packages.json'.format(target)) taskdesc['scopes'].extend( ['generic-worker:os-group:{}'.format(group) for group in test['os-groups']]) @@ -325,19 +333,24 @@ def mozharness_test_on_generic_worker(co @run_job_using('native-engine', 'mozharness-test', schema=mozharness_test_run_schema) def mozharness_test_on_native_engine(config, job, taskdesc): test = taskdesc['run']['test'] mozharness = test['mozharness'] worker = taskdesc['worker'] + build_platform = taskdesc['attributes']['build_platform'] + build_type = taskdesc['attributes']['build_type'] + target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') \ + if build_platform == 'macosx64' and build_type == 'opt' else 'target' + installer_url = get_artifact_url('<build>', mozharness['build-artifact-name']) test_packages_url = get_artifact_url('<build>', - 'public/build/target.test_packages.json') + 'public/build/{}.test_packages.json'.format(target)) mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip') worker['artifacts'] = [{ 'name': prefix.rstrip('/'), 'path': path.rstrip('/'), 'type': 'directory', } for (prefix, path) in ARTIFACTS]
--- a/taskcluster/taskgraph/transforms/tests.py +++ b/taskcluster/taskgraph/transforms/tests.py @@ -341,17 +341,23 @@ def set_defaults(config, tests): yield test @transforms.add def set_target(config, tests): for test in tests: build_platform = test['build-platform'] if build_platform.startswith('macosx'): - target = 'target.dmg' + if build_platform.split('/')[1] == 'opt': + target = 'firefox-{}.en-US.{}.dmg'.format( + get_firefox_version(), + 'mac', + ) + else: + target = 'target.dmg' elif build_platform.startswith('android'): if 'geckoview' in test['test-name']: target = 'geckoview_example.apk' else: target = 'target.apk' elif build_platform.startswith('win'): target = 'firefox-{}.en-US.{}.zip'.format( get_firefox_version(),
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -1414,28 +1414,32 @@ or run without that action (ie: --no-{ac 'locale': locale, } fmt.update(self.buildid_to_dict(self.query_buildid())) routes = [] for template in templates: routes.append(template.format(**fmt)) self.info("Using routes: %s" % routes) + taskid = self.buildbot_config['properties'].get('upload_to_task_id') tc = Taskcluster( branch=self.branch, rank=pushinfo.pushdate, # Use pushdate as the rank client_id=self.client_id, access_token=self.access_token, log_obj=self.log_obj, # `upload_to_task_id` is used by mozci to have access to where the artifacts # will be uploaded - task_id=self.buildbot_config['properties'].get('upload_to_task_id'), + task_id=taskid, ) - task = tc.create_task(routes) + if taskid: + task = tc.get_task(taskid) + else: + task = tc.create_task(routes) tc.claim_task(task) # Only those files uploaded with valid extensions are processed. # This ensures that we get the correct packageUrl from the list. valid_extensions = ( '.apk', '.dmg', '.mar',