Bug 1274980 part 2: Support Mac OS tests in tc intree config. r=dustin
authorWander Lairson Costa <wcosta@mozilla.com>
Tue, 08 Nov 2016 12:11:15 -0200
changeset 351614 9d6b5736f9c8feee1b11cce37e3b7b6e5c7c116f
parent 351613 ef393dc07cb861db2bdc19a7635b3f3cdec8bc6d
child 351615 680094d957919bd757685e3de5fe244a5e038658
child 351674 9796c87f3a2d847fbfe3d359c529b6be241543aa
push id6795
push userjlund@mozilla.com
push dateMon, 23 Jan 2017 14:19:46 +0000
treeherdermozilla-esr52@76101b503191 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1274980
milestone52.0a1
Bug 1274980 part 2: Support Mac OS tests in tc intree config. r=dustin We add support for Mac OSX tests in Taskcluster by support taskcluster-worker macosx engine. We also add a new configuration blob-upload-branch needed by mochitest-other. MozReview-Commit-ID: 5ZaYYRueRVr
taskcluster/ci/desktop-test/test-platforms.yml
taskcluster/ci/desktop-test/test-sets.yml
taskcluster/ci/desktop-test/tests.yml
taskcluster/scripts/tester/test-macosx.sh
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/transforms/tests/all_kinds.py
taskcluster/taskgraph/transforms/tests/desktop_test.py
taskcluster/taskgraph/transforms/tests/make_task_description.py
taskcluster/taskgraph/transforms/tests/test_description.py
--- a/taskcluster/ci/desktop-test/test-platforms.yml
+++ b/taskcluster/ci/desktop-test/test-platforms.yml
@@ -59,8 +59,15 @@ windows10-64-vm/opt:
 
 # win64 gpu
 #windows10-64/debug:
 #    build-platform: win64/debug
 #    test-set: windows-gpu-tests
 #windows10-64/opt:
 #    build-platform: win64/opt
 #    test-set: windows-gpu-tests
+
+macosx64/debug:
+    build-platform: macosx64/debug
+    test-set: macosx64-tests
+# macosx64/opt:
+#    build-platform: macosx64/opt
+#    test-set: macosx64-tests
--- a/taskcluster/ci/desktop-test/test-sets.yml
+++ b/taskcluster/ci/desktop-test/test-sets.yml
@@ -91,9 +91,29 @@ windows-vm-tests:
 ccov-code-coverage-tests:
     - mochitest
     - mochitest-browser-chrome
     - mochitest-devtools-chrome
     - xpcshell
 
 jsdcov-code-coverage-tests:
     - mochitest-browser-chrome
-    - mochitest-devtools-chrome
\ No newline at end of file
+    - mochitest-devtools-chrome
+
+macosx64-tests:
+    - cppunit
+    - crashtest
+    # - gtest
+    - jsreftest
+    # - marionette
+    # - mochitest
+    # - mochitest-browser-chrome
+    # - mochitest-clipboard
+    # - mochitest-devtools-chrome
+    # - mochitest-gpu
+    # - mochitest-jetpack
+    # - mochitest-media
+    # - mochitest-other
+    - mochitest-webgl
+    # - reftest
+    # - web-platform-tests
+    # - web-platform-tests-reftests
+    # - xpcshell
--- a/taskcluster/ci/desktop-test/tests.yml
+++ b/taskcluster/ci/desktop-test/tests.yml
@@ -12,16 +12,19 @@ cppunit:
     e10s: false
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --cppunittest-suite=cppunittest
 
 crashtest:
     description: "Crashtest run"
@@ -36,16 +39,19 @@ crashtest:
     mozharness:
         script: desktop_unittest.py
         chunked: true
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --reftest-suite=crashtest
 
 external-media-tests:
     description: "External Media Test run"
@@ -72,46 +78,51 @@ firefox-ui-functional-local:
     tier: 1
     docker-image: {"in-tree": "desktop1604-test"}
     mozharness:
         script: firefox_ui_tests/functional.py
         config:
             - firefox_ui_tests/taskcluster.py
             - remove_executables.py
         extra-options:
-            - "--tag local"
+            - "--tag"
+            - "local"
 
 firefox-ui-functional-remote:
     description: "Firefox-ui-tests functional run"
     suite: "firefox-ui/functional remote"
     treeherder-symbol: tc-Fxfn-r(en-US)
     max-run-time: 5400
     tier: 2
     docker-image: {"in-tree": "desktop1604-test"}
     mozharness:
         script: firefox_ui_tests/functional.py
         config:
             - firefox_ui_tests/taskcluster.py
             - remove_executables.py
         extra-options:
-            - "--tag remote"
+            - "--tag"
+            - "remote"
 
 gtest:
     description: "GTests run"
     suite: gtest
     treeherder-symbol: tc(GTest)
     e10s: false
     instance-size: xlarge
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --gtest-suite=gtest
 
 jittest:
     description: "JIT Test run"
@@ -124,16 +135,19 @@ jittest:
             default: 6
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --jittest-suite=jittest-chunked
 
 jsreftest:
     description: "JS Reftest run"
@@ -145,16 +159,19 @@ jsreftest:
             default: 2
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --reftest-suite=jsreftest
 
 marionette:
     description: "Marionette unittest run"
@@ -178,31 +195,35 @@ mochitest:
     treeherder-symbol: tc-M()
     loopback-video: true
     run-on-projects:
         by-test-platform:
             linux64-ccov/opt: []
             default: ['all']
     chunks:
         by-test-platform:
+            macosx.*: 5
             win.*: 5
             default: 10
     e10s:
         by-test-platform:
             linux64-ccov/opt: false
             default: both
     max-run-time: 5400
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             by-test-platform:
                 linux64-ccov/opt:
                     - --mochitest-suite=plain-chunked
                     - --code-coverage
@@ -221,16 +242,19 @@ mochitest-a11y:
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --mochitest-suite=a11y
 
 mochitest-browser-chrome:
     description: "Mochitest browser-chrome run"
@@ -262,16 +286,19 @@ mochitest-browser-chrome:
             default: 3600
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             by-test-platform:
                 linux64-jsdcov/opt:
                     - --mochitest-suite=browser-chrome-coverage
                 linux64-ccov/opt:
@@ -287,25 +314,31 @@ mochitest-browser-chrome:
             default: legacy
     allow-software-gl-layers: false
 
 mochitest-chrome:
     description: "Mochitest chrome run"
     suite: mochitest/chrome
     treeherder-symbol: tc-M(c)
     loopback-video: true
-    chunks: 3
+    chunks:
+      by-test-platform:
+        macosx.*: 1
+        default: 3
     e10s: false
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --mochitest-suite=chrome
 
 mochitest-clipboard:
     description: "Mochitest clipboard run"
@@ -316,16 +349,19 @@ mochitest-clipboard:
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --mochitest-suite=plain-clipboard,chrome-clipboard,browser-chrome-clipboard,jetpack-package-clipboard
 
 mochitest-devtools-chrome:
     description: "Mochitest devtools-chrome run"
@@ -354,16 +390,19 @@ mochitest-devtools-chrome:
             default: both
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             by-test-platform:
                 linux64-ccov/opt:
                     - --mochitest-suite=mochitest-devtools-chrome-chunked
                     - --code-coverage
@@ -391,16 +430,19 @@ mochitest-gpu:
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --mochitest-suite=plain-gpu,chrome-gpu,browser-chrome-gpu
 
 mochitest-jetpack:
     description: "Mochitest jetpack run"
@@ -412,16 +454,19 @@ mochitest-jetpack:
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --mochitest-suite=jetpack-package
             - --mochitest-suite=jetpack-addon
 
 mochitest-media:
@@ -435,87 +480,130 @@ mochitest-media:
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --mochitest-suite=mochitest-media
 
+mochitest-other:
+    description: "Mochitest other"
+    suite: mochitest/other
+    treeherder-symbol: tc-M(oth)
+    e10s: false
+    max-run-time: 5400
+    mozharness:
+        script: mozharness/scripts/desktop_unittest.py
+        no-read-buildbot-config: true
+        include-blob-upload-branch: true
+        chunked: true
+        config:
+          by-test-platform:
+            macosx.*:
+              - remove_executables.py
+              - unittests/mac_unittest.py
+            default:
+              - unittests/linux_unittest.py
+              - remove_executables.py
+        extra-options:
+            - --mochitest-suite=chrome,a11y
+
 mochitest-webgl:
     description: "Mochitest webgl run"
     suite: mochitest/mochitest-gl
     treeherder-symbol: tc-M(gl)
     chunks: 3
     loopback-video: true
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         chunked: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --mochitest-suite=mochitest-gl
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
 
 reftest:
     description: "Reftest run"
     suite: reftest/reftest
     treeherder-symbol: tc-R(R)
-    chunks: 8
     docker-image: {"in-tree": "desktop1604-test"}
+    chunks:
+      by-test-platform:
+        macosx.*: 1
+        default: 8
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --reftest-suite=reftest
 
 reftest-no-accel:
     description: "Reftest not accelerated run"
     suite: reftest/reftest-no-accel
     treeherder-symbol: tc-R(Ru)
-    chunks: 8
     docker-image: {"in-tree": "desktop1604-test"}
+    chunks:
+      by-test-platform:
+        macosx.*: 1
+        default: 8
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --reftest-suite=reftest-no-accel
 
 web-platform-tests:
     description: "Web platform test run"
     suite: web-platform-tests
     treeherder-symbol: tc-W()
-    chunks: 12
+    chunks:
+      by-test-platform:
+        macosx.*: 5
+        default: 12
     max-run-time: 7200
     instance-size: xlarge
     docker-image: {"in-tree": "desktop1604-test"}
     checkout: true
     mozharness:
         script: web_platform_tests.py
         no-read-buildbot-config: true
         config:
@@ -576,27 +664,31 @@ xpcshell:
     treeherder-symbol: tc-X()
     run-on-projects:
         by-test-platform:
             linux64-ccov/opt: []
             default: ['all']
     chunks:
         by-test-platform:
             # win.*: 1
+            macosx.*: 1
             linux64/debug: 10
             default: 8
     max-run-time: 5400
     e10s: false
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 win.*:
                     - unittests/win_taskcluster_unittest.py
+                macosx.*:
+                    - remove_executables.py
+                    - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             by-test-platform:
                 linux64-ccov/opt:
                     - --xpcshell-suite=xpcshell
                     - --code-coverage
new file mode 100644
--- /dev/null
+++ b/taskcluster/scripts/tester/test-macosx.sh
@@ -0,0 +1,52 @@
+#! /bin/bash -xe
+
+set -x -e
+
+echo "running as" $(id)
+
+####
+# Taskcluster friendly wrapper for performing fx Mac OSX tests via mozharness.
+####
+
+# Inputs, with defaults
+
+: MOZHARNESS_URL                ${MOZHARNESS_URL}
+: MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
+: MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
+
+WORKSPACE=$HOME
+cd $WORKSPACE
+
+rm -rf artifacts
+mkdir artifacts
+
+# test required parameters are supplied
+if [[ -z ${MOZHARNESS_URL} ]]; then fail "MOZHARNESS_URL is not set"; fi
+if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
+if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi
+
+# Unzip the mozharness ZIP file created by the build task
+if ! curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
+    fail "failed to download mozharness zip"
+fi
+rm -rf mozharness
+unzip -q mozharness.zip
+rm mozharness.zip
+
+# For telemetry purposes, the build process wants information about the
+# source it is running; tc-vcs obscures this a little, but we can provide
+# it directly.
+export MOZ_SOURCE_REPO="${GECKO_HEAD_REPOSITORY}"
+export MOZ_SOURCE_CHANGESET="${GECKO_HEAD_REV}"
+
+# support multiple, space delimited, config files
+config_cmds=""
+for cfg in $MOZHARNESS_CONFIG; do
+  config_cmds="${config_cmds} --config-file ${cfg}"
+done
+
+rm -rf build logs properties target.dmg
+
+# run the given mozharness script and configs, but pass the rest of the
+# arguments in from our own invocation
+python2.7 $WORKSPACE/mozharness/scripts/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}"
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -42,16 +42,17 @@ JOB_NAME_WHITELIST = set([
     'linux-debug',
     'linux-opt',
     'linux-pgo',
     'macosx64-debug',
     'macosx64-opt',
     'macosx64-st-an-opt',
     'nexus-5-l-eng-debug',
     'nexus-5-l-eng-opt',
+    'osx-10-10',
     'shell-haz-debug',
     'sm-arm64-sim-debug',
     'sm-arm-sim-debug',
     'sm-asan-opt',
     'sm-compacting-debug',
     'sm-mozjs-sys-debug',
     'sm-msan-opt',
     'sm-nonunified-debug',
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -245,16 +245,40 @@ task_description_schema = Schema({
             Optional('revision'): basestring,
             Optional('repository'): basestring,
             Optional('project'): basestring,
         },
         'properties': {
             'product': basestring,
             Extra: basestring,  # additional properties are allowed
         },
+    }, {
+        'implementation': 'macosx-engine',
+
+        # A link for an executable to download
+        Optional('link'): basestring,
+
+        # the command to run
+        Required('command'): [taskref_or_string],
+
+        # environment variables
+        Optional('env'): {basestring: taskref_or_string},
+
+        # artifacts to extract from the task image after completion
+        Optional('artifacts'): [{
+            # type of artifact -- simple file, or recursive directory
+            Required('type'): Any('file', 'directory'),
+
+            # task image path from which to read artifact
+            Required('path'): basestring,
+
+            # name of the produced artifact (root of the names for
+            # type=directory)
+            Required('name'): basestring,
+        }],
     }),
 
     # The "when" section contains descriptions of the circumstances
     # under which this task can be "optimized", that is, left out of the
     # task graph because it is unnecessary.
     Optional('when'): Any({
         # This task only needs to be run if a file matching one of the given
         # patterns has changed in the push.  The patterns use the mozpack
@@ -425,16 +449,33 @@ def build_generic_worker_payload(config,
         'maxRunTime': worker['max-run-time'],
         'osGroups': worker.get('os-groups', []),
     }
 
     if 'retry-exit-status' in worker:
         raise Exception("retry-exit-status not supported in generic-worker")
 
 
+@payload_builder('macosx-engine')
+def build_macosx_engine_payload(config, task, task_def):
+    worker = task['worker']
+    artifacts = map(lambda artifact: {
+        'name': artifact['name'],
+        'path': artifact['path'],
+        'type': artifact['type'],
+        'expires': task_def['expires'],
+    }, worker['artifacts'])
+
+    task_def['payload'] = {
+        'link': worker['link'],
+        'command': worker['command'],
+        'env': worker['env'],
+        'artifacts': artifacts,
+    }
+
 transforms = TransformSequence()
 
 
 @transforms.add
 def validate(config, tasks):
     for task in tasks:
         yield validate_schema(
             task_description_schema, task,
--- a/taskcluster/taskgraph/transforms/tests/all_kinds.py
+++ b/taskcluster/taskgraph/transforms/tests/all_kinds.py
@@ -21,16 +21,18 @@ transforms = TransformSequence()
 
 
 @transforms.add
 def set_worker_implementation(config, tests):
     """Set the worker implementation based on the test platform."""
     for test in tests:
         if test['test-platform'].startswith('win'):
             test['worker-implementation'] = 'generic-worker'
+        elif test['test-platform'].startswith('macosx'):
+            test['worker-implementation'] = 'macosx-engine'
         else:
             test['worker-implementation'] = 'docker-worker'
         yield test
 
 
 @transforms.add
 def set_tier(config, tests):
     """Set the tier based on policy for all test descriptions that do not
--- a/taskcluster/taskgraph/transforms/tests/desktop_test.py
+++ b/taskcluster/taskgraph/transforms/tests/desktop_test.py
@@ -16,31 +16,38 @@ from taskgraph.util.treeherder import sp
 import copy
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def set_defaults(config, tests):
     for test in tests:
-        test['mozharness']['build-artifact-name'] = 'public/build/target.tar.bz2'
+        build_platform = test['build-platform']
+        if build_platform.startswith('macosx'):
+            target = 'target.dmg'
+        else:
+            target = 'target.tar.bz2'
+        test['mozharness']['build-artifact-name'] = 'public/build/' + target
         # all desktop tests want to run the bits that require node
         test['mozharness']['set-moz-node-path'] = True
         yield test
 
 
 @transforms.add
 def set_treeherder_machine_platform(config, tests):
     """Set the appropriate task.extra.treeherder.machine.platform"""
     # Linux64 build platforms for asan and pgo are specified differently to
     # treeherder.  This is temporary until we can clean up the handling of
     # platforms
     translation = {
         'linux64-asan/opt': 'linux64/asan',
-        'linux64-pgo/opt': 'linux64/pgo'
+        'linux64-pgo/opt': 'linux64/pgo',
+        'macosx64/debug': 'osx-10-10/debug',
+        'macosx64/opt': 'osx-10-10/opt',
     }
     for test in tests:
         build_platform = test['build-platform']
         test_platform = test['test-platform']
         test['treeherder-machine-platform'] = translation.get(build_platform, test_platform)
         yield test
 
 
--- a/taskcluster/taskgraph/transforms/tests/make_task_description.py
+++ b/taskcluster/taskgraph/transforms/tests/make_task_description.py
@@ -20,31 +20,39 @@ for example - use `all_tests.py` instead
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.job.common import (
     docker_worker_support_vcs_checkout,
 )
 
 import logging
+import os.path
 
 ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 WORKER_TYPE = {
     # default worker types keyed by instance-size
     'large': 'aws-provisioner-v1/desktop-test-large',
     'xlarge': 'aws-provisioner-v1/desktop-test-xlarge',
     'legacy': 'aws-provisioner-v1/desktop-test',
     'default': 'aws-provisioner-v1/desktop-test-large',
     # windows worker types keyed by test-platform
     'windows7-32-vm': 'aws-provisioner-v1/gecko-t-win7-32',
     'windows7-32': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
     'windows10-64-vm': 'aws-provisioner-v1/gecko-t-win10-64',
     'windows10-64': 'aws-provisioner-v1/gecko-t-win10-64-gpu'
 }
 
+ARTIFACTS = [
+    # (artifact name prefix, in-image path)
+    ("public/logs/", "build/upload/logs/"),
+    ("public/test", "artifacts/"),
+    ("public/test_info/", "build/blobber_upload_dir/"),
+]
+
 logger = logging.getLogger(__name__)
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def make_task_description(config, tests):
     """Convert *test* descriptions to *task* descriptions (input to
@@ -152,17 +160,17 @@ def docker_worker_setup(config, test, ta
     worker['relengapi-proxy'] = False  # but maybe enabled for tooltool below
     worker['loopback-video'] = test['loopback-video']
     worker['loopback-audio'] = test['loopback-audio']
     worker['max-run-time'] = test['max-run-time']
     worker['retry-exit-status'] = test['retry-exit-status']
 
     worker['artifacts'] = [{
         'name': prefix,
-        'path': path,
+        'path': os.path.join('/home/worker/workspace', path),
         'type': 'directory',
     } for (prefix, path) in artifacts]
 
     worker['caches'] = [{
         'type': 'persistent',
         'name': 'level-{}-{}-test-workspace'.format(
             config.params['level'], config.params['project']),
         'mount-point': "/home/worker/workspace",
@@ -364,8 +372,74 @@ def generic_worker_setup(config, test, t
     worker['command'] = [
         'mkdir {} {}'.format(env['APPDATA'], env['TMP']),
         {'task-reference': 'c:\\mozilla-build\\wget\\wget.exe {}'.format(mozharness_url)},
         'c:\\mozilla-build\\info-zip\\unzip.exe mozharness.zip',
         {'task-reference': ' '.join(mh_command)},
         'xcopy build\\blobber_upload_dir public\\test_info /e /i',
         'copy /y logs\\*.* public\\logs\\'
     ]
+
+
+@worker_setup_function("macosx-engine")
+def macosx_engine_setup(config, test, taskdesc):
+    mozharness = test['mozharness']
+
+    installer_url = ARTIFACT_URL.format('<build>', mozharness['build-artifact-name'])
+    test_packages_url = ARTIFACT_URL.format('<build>',
+                                            'public/build/target.test_packages.json')
+    mozharness_url = ARTIFACT_URL.format('<build>',
+                                         'public/build/mozharness.zip')
+
+    # for now we have only 10.10 machines
+    taskdesc['worker-type'] = 'tc-worker-provisioner/gecko-t-osx-10-10'
+
+    worker = taskdesc['worker'] = {}
+    worker['implementation'] = test['worker-implementation']
+
+    worker['artifacts'] = [{
+        'name': prefix.rstrip('/'),
+        'path': path.rstrip('/'),
+        'type': 'directory',
+    } for (prefix, path) in ARTIFACTS]
+
+    worker['env'] = {
+        'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+        'GECKO_HEAD_REV': config.params['head_rev'],
+        'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
+        'MOZHARNESS_SCRIPT': mozharness['script'],
+        'MOZHARNESS_URL': {'task-reference': mozharness_url},
+        'MOZILLA_BUILD_URL': {'task-reference': installer_url},
+    }
+
+    # assemble the command line
+
+    worker['link'] = '{}/raw-file/{}/taskcluster/scripts/tester/test-macosx.sh'.format(
+        config.params['head_repository'], config.params['head_rev']
+    )
+
+    command = worker['command'] = ["./test-macosx.sh"]
+    if mozharness.get('no-read-buildbot-config'):
+        command.append("--no-read-buildbot-config")
+    command.extend([
+        {"task-reference": "--installer-url=" + installer_url},
+        {"task-reference": "--test-packages-url=" + test_packages_url},
+    ])
+    if mozharness.get('include-blob-upload-branch'):
+        command.append('--blob-upload-branch=' + config.params['project'])
+    command.extend(mozharness.get('extra-options', []))
+
+    # TODO: remove the need for run['chunked']
+    if mozharness.get('chunked') or test['chunks'] > 1:
+        # Implement mozharness['chunking-args'], modifying command in place
+        if mozharness['chunking-args'] == 'this-chunk':
+            command.append('--total-chunk={}'.format(test['chunks']))
+            command.append('--this-chunk={}'.format(test['this-chunk']))
+        elif mozharness['chunking-args'] == 'test-suite-suffix':
+            suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
+            for i, c in enumerate(command):
+                if isinstance(c, basestring) and c.startswith('--test-suite'):
+                    command[i] += suffix
+
+    if 'download-symbols' in mozharness:
+        download_symbols = mozharness['download-symbols']
+        download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
+        command.append('--download-symbols=' + download_symbols)
--- a/taskcluster/taskgraph/transforms/tests/test_description.py
+++ b/taskcluster/taskgraph/transforms/tests/test_description.py
@@ -101,16 +101,17 @@ test_description_schema = Schema({
     # using the GL compositor. May not be used with "legacy" sized instances
     # due to poor LLVMPipe performance (bug 1296086).
     Optional('allow-software-gl-layers', default=True): bool,
 
     # The worker implementation for this test, as dictated by policy and by the
     # test platform.
     Optional('worker-implementation'): Any(
         'docker-worker',
+        'macosx-engine',
         'generic-worker',
         # coming soon:
         'docker-engine',
         'buildbot-bridge',
     ),
 
     # For tasks that will run in docker-worker or docker-engine, this is the
     # name of the docker image or in-tree docker image to run the task in.  If
@@ -163,16 +164,19 @@ test_description_schema = Schema({
 
         # If true, tooltool downloads will be enabled via relengAPIProxy.
         Required('tooltool-downloads', default=False): bool,
 
         # This mozharness script also runs in Buildbot and tries to read a
         # buildbot config file, so tell it not to do so in TaskCluster
         Required('no-read-buildbot-config', default=False): bool,
 
+        # Add --blob-upload-branch=<project> mozharness parameter
+        Optional('include-blob-upload-branch'): bool,
+
         # The setting for --download-symbols (if omitted, the option will not
         # be passed to mozharness)
         Optional('download-symbols'): Any(True, 'ondemand'),
 
         # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the
         # environment.  This is more than just a helpful path setting -- it
         # causes xpcshell tests to start additional servers, and runs
         # additional tests.