Bug 1350413 part 1: move macosx64/opt build to buildbot-bridge. r=dustin
authorWander Lairson Costa <wcosta@mozilla.com>
Tue, 02 May 2017 16:12:51 -0300
changeset 356595 971b9548d84c31878468f4890d283377987e2cf4
parent 356594 0c8c40eddb5eb6c4dbc91457159e16dc3a8ea800
child 356596 6b5f20da8bc90f3145980234cdc488e516c63325
push id89950
push usercbook@mozilla.com
push dateFri, 05 May 2017 13:26:37 +0000
treeherdermozilla-inbound@e95e5825fd40 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1350413, 1338651
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1350413 part 1: move macosx64/opt build to buildbot-bridge. r=dustin Because of bug 1338651, we need to stick to BBB macosx64 builds for now. MozReview-Commit-ID: AwQc5r6ikUN
taskcluster/ci/build/macosx.yml
taskcluster/taskgraph/morph.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/job/mozharness_test.py
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -18,33 +18,32 @@ macosx64/debug:
             - builds/releng_base_mac_64_cross_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: cross-debug
         tooltool-downloads: internal
 
 macosx64/opt:
-    description: "MacOS X x64 Cross-compile"
+    description: "MacOS X x64"
     index:
         product: firefox
         job-name: macosx64-opt
     treeherder:
         platform: osx-10-7/opt
         symbol: tc(B)
         tier: 2
-    worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+    worker-type: buildbot-bridge/buildbot-bridge
     worker:
-        implementation: docker-worker
-        max-run-time: 36000
+        implementation: buildbot-bridge
     run:
         using: mozharness
         actions: [get-secrets build generate-build-stats update]
         config:
-            - builds/releng_base_mac_64_cross_builds.py
+            - builds/releng_base_mac_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: internal
 
 macosx64-qr/debug:
     description: "MacOS X x64 QuantumRender"
     index:
--- a/taskcluster/taskgraph/morph.py
+++ b/taskcluster/taskgraph/morph.py
@@ -152,12 +152,97 @@ def add_index_tasks(taskgraph, label_to_
     if added:
         taskgraph, label_to_taskid = amend_taskgraph(
             taskgraph, label_to_taskid, added)
         logger.info('Added {} index tasks'.format(len(added)))
 
     return taskgraph, label_to_taskid
 
 
+def make_s3_uploader_task(parent_task):
+    if parent_task.task['payload']['sourcestamp']['branch'] == 'try':
+        worker_type = 'buildbot-try'
+    else:
+        worker_type = 'buildbot'
+
+    task_def = {
+        # The null-provisioner and buildbot worker type don't actually exist.
+        # So this task doesn't actually run - we just need to create the task so
+        # we have something to attach artifacts to.
+        "provisionerId": "null-provisioner",
+        "workerType": worker_type,
+        "created": {'relative-datestamp': '0 seconds'},
+        "deadline": parent_task.task['deadline'],
+        "routes": [],
+        "payload": {},
+        "metadata": {
+            "name": "Buildbot/mozharness S3 uploader",
+            "description": "Upload outputs of buildbot/mozharness builds to S3",
+            "owner": "mshal@mozilla.com",
+            "source": "http://hg.mozilla.org/build/mozharness/",
+        }
+    }
+    label = 's3-uploader-{}'.format(parent_task.label)
+    dependencies = {}
+    task = Task(kind='misc', label=label, attributes={}, task=task_def,
+                dependencies=dependencies)
+    task.task_id = parent_task.task['payload']['properties']['upload_to_task_id']
+    return task
+
+
+def update_test_tasks(taskid, build_taskid, taskgraph):
+    """Tests task must download artifacts from uploader task.
+
+    Notice they don't need to depend on uploader task because it finishes
+    before the build task.
+    """
+    # Notice we handle buildbot-bridge, native, and generic-worker payloads
+    # We can do better here in terms of graph searching
+    # We could do post order search and stop as soon as we
+    # reach the build task. Not worring about it because this is
+    # (supposed to be) a temporary solution.
+    for task in taskgraph.tasks.itervalues():
+        if build_taskid in task.task.get('dependencies', []):
+            payload = task.task['payload']
+            if 'command' in payload:
+                try:
+                    payload['command'] = [
+                        cmd.replace(build_taskid, taskid) for cmd in payload['command']
+                    ]
+                except AttributeError:
+                    # generic-worker command attribute is an list of lists
+                    payload['command'] = [
+                        [cmd.replace(build_taskid, taskid) for cmd in x]
+                        for x in payload['command']
+                    ]
+            if 'mounts' in payload:
+                for mount in payload['mounts']:
+                    if mount.get('content', {}).get('taskId', '') == build_taskid:
+                        mount['content']['taskId'] = taskid
+            if 'env' in payload:
+                payload['env'] = {
+                    k: v.replace(build_taskid, taskid) for k, v in payload['env'].iteritems()
+                }
+            if 'properties' in payload:
+                payload['properties']['parent_task_id'] = taskid
+
+
+def add_s3_uploader_task(taskgraph, label_to_taskid):
+    """The S3 uploader task is used by mozharness to upload buildbot artifacts."""
+    for task in taskgraph.tasks.itervalues():
+        if 'upload_to_task_id' in task.task.get('payload', {}).get('properties', {}):
+            added = make_s3_uploader_task(task)
+            taskgraph, label_to_taskid = amend_taskgraph(
+                taskgraph, label_to_taskid, [added])
+            update_test_tasks(added.task_id, task.task_id, taskgraph)
+            logger.info('Added s3-uploader task')
+    return taskgraph, label_to_taskid
+
+
 def morph(taskgraph, label_to_taskid):
     """Apply all morphs"""
-    taskgraph, label_to_taskid = add_index_tasks(taskgraph, label_to_taskid)
+    morphs = [
+        add_index_tasks,
+        add_s3_uploader_task,
+    ]
+    for m in morphs:
+        taskgraph, label_to_taskid = m(taskgraph, label_to_taskid)
     return taskgraph, label_to_taskid
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -4,16 +4,17 @@
 """
 
 Support for running jobs via mozharness.  Ideally, most stuff gets run this
 way, and certainly anything using mozharness should use this approach.
 
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
+import slugid
 
 from textwrap import dedent
 
 from taskgraph.util.schema import Schema
 from voluptuous import Required, Optional, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
@@ -239,8 +240,32 @@ def mozharness_on_generic_worker(config,
             r'icacls z:\build /grant *S-1-1-0:D /L',
             r'cd /d z:\build',
         ])
 
     worker['command'].extend([
         ' '.join(hg_command),
         ' '.join(mh_command)
     ])
+
+
+@run_job_using('buildbot-bridge', 'mozharness', schema=mozharness_run_schema)
+def mozharness_on_buildbot_bridge(config, job, taskdesc):
+    run = job['run']
+    worker = taskdesc['worker']
+    branch = config.params['project']
+    product = run.get('index', {}).get('product', 'firefox')
+
+    worker.pop('env', None)
+
+    worker.update({
+        'buildername': 'OS X 10.7 {} build'.format(branch),
+        'sourcestamp': {
+            'branch': branch,
+            'repository': config.params['head_repository'],
+            'revision': config.params['head_rev'],
+        },
+        'properties': {
+            'product': product,
+            'who': config.params['owner'],
+            'upload_to_task_id': slugid.nice(),
+        }
+    })
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -205,27 +205,35 @@ def mozharness_test_on_generic_worker(co
             'path': 'logs/log_raw.log',
             'type': 'file'
         },
         {
             'name': 'public/logs/log_warning.log',
             'path': 'logs/log_warning.log',
             'type': 'file'
         },
-        {
+    ]
+
+    # jittest doesn't have blob_upload_dir
+    if test['test-name'] != 'jittest':
+        artifacts.append({
             'name': 'public/test_info',
             'path': 'build/blobber_upload_dir',
             'type': 'directory'
-        }
-    ]
+        })
 
     build_platform = taskdesc['attributes']['build_platform']
+    build_type = taskdesc['attributes']['build_type']
 
-    target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) \
-        if build_platform.startswith('win') else 'target'
+    if build_platform.startswith('win'):
+        target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform)
+    elif build_platform == 'macosx64' and build_type == 'opt':
+        target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
+    else:
+        target = 'target'
 
     installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
 
     test_packages_url = get_artifact_url(
         '<build>', 'public/build/{}.test_packages.json'.format(target))
 
     taskdesc['scopes'].extend(
         ['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
@@ -325,19 +333,24 @@ def mozharness_test_on_generic_worker(co
 
 
 @run_job_using('native-engine', 'mozharness-test', schema=mozharness_test_run_schema)
 def mozharness_test_on_native_engine(config, job, taskdesc):
     test = taskdesc['run']['test']
     mozharness = test['mozharness']
     worker = taskdesc['worker']
 
+    build_platform = taskdesc['attributes']['build_platform']
+    build_type = taskdesc['attributes']['build_type']
+    target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') \
+        if build_platform == 'macosx64' and build_type == 'opt' else 'target'
+
     installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
     test_packages_url = get_artifact_url('<build>',
-                                         'public/build/target.test_packages.json')
+                                         'public/build/{}.test_packages.json'.format(target))
     mozharness_url = get_artifact_url('<build>',
                                       'public/build/mozharness.zip')
 
     worker['artifacts'] = [{
         'name': prefix.rstrip('/'),
         'path': path.rstrip('/'),
         'type': 'directory',
     } for (prefix, path) in ARTIFACTS]
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -341,17 +341,23 @@ def set_defaults(config, tests):
         yield test
 
 
 @transforms.add
 def set_target(config, tests):
     for test in tests:
         build_platform = test['build-platform']
         if build_platform.startswith('macosx'):
-            target = 'target.dmg'
+            if build_platform.split('/')[1] == 'opt':
+                target = 'firefox-{}.en-US.{}.dmg'.format(
+                    get_firefox_version(),
+                    'mac',
+                )
+            else:
+                target = 'target.dmg'
         elif build_platform.startswith('android'):
             if 'geckoview' in test['test-name']:
                 target = 'geckoview_example.apk'
             else:
                 target = 'target.apk'
         elif build_platform.startswith('win'):
             target = 'firefox-{}.en-US.{}.zip'.format(
                 get_firefox_version(),
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1414,28 +1414,32 @@ or run without that action (ie: --no-{ac
             'locale': locale,
         }
         fmt.update(self.buildid_to_dict(self.query_buildid()))
         routes = []
         for template in templates:
             routes.append(template.format(**fmt))
         self.info("Using routes: %s" % routes)
 
+        taskid = self.buildbot_config['properties'].get('upload_to_task_id')
         tc = Taskcluster(
             branch=self.branch,
             rank=pushinfo.pushdate, # Use pushdate as the rank
             client_id=self.client_id,
             access_token=self.access_token,
             log_obj=self.log_obj,
             # `upload_to_task_id` is used by mozci to have access to where the artifacts
             # will be uploaded
-            task_id=self.buildbot_config['properties'].get('upload_to_task_id'),
+            task_id=taskid,
         )
 
-        task = tc.create_task(routes)
+        if taskid:
+            task = tc.get_task(taskid)
+        else:
+            task = tc.create_task(routes)
         tc.claim_task(task)
 
         # Only those files uploaded with valid extensions are processed.
         # This ensures that we get the correct packageUrl from the list.
         valid_extensions = (
             '.apk',
             '.dmg',
             '.mar',