Bug 1286075: add a spidermonkey kind; r=sfink
authorDustin J. Mitchell <dustin@mozilla.com>
Mon, 12 Sep 2016 18:04:24 +0000
changeset 357216 a84691d694b7ebb9344ce5a784943976e7ae5cc7
parent 357215 ae09f23c30124fcfb4cab32d355a96487280c0c1
child 357217 d2e2c2aad273cec54708acb67b80313690782107
push id1324
push usermtabara@mozilla.com
push dateMon, 16 Jan 2017 13:07:44 +0000
treeherdermozilla-release@a01c49833940 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfink
bugs1286075
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1286075: add a spidermonkey kind; r=sfink MozReview-Commit-ID: 4uI8LxHrSOS
taskcluster/ci/spidermonkey/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/job/spidermonkey.py
taskcluster/taskgraph/try_option_syntax.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/spidermonkey/kind.yml
@@ -0,0 +1,221 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+    treeherder:
+        platform: linux64/opt
+        kind: build
+        tier: 1
+    index:
+        product: firefox
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        max-run-time: 36000
+        docker-image: {in-tree: desktop-build}
+    run:
+        using: spidermonkey
+    when:
+        files-changed:
+            # any when.files-changed specified below in a job will be
+            # appended to this list
+            - js/public/**
+            - js/src/**
+
+jobs:
+    sm-package/opt:
+        description: "Spidermonkey source package and test"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-package.opt
+                gecko-v2: sm-package-opt
+        treeherder:
+            symbol: SM-tc(pkg)
+        run:
+            using: spidermonkey-package
+            spidermonkey-variant: plain
+        run-on-projects:
+            - integration
+            - release
+        when:
+            files-changed:
+                - build/**
+                - configure.py
+                - dom/bindings/**
+                - intl/icu/**
+                - js/moz.configure
+                - layout/tools/reftest/reftest/**
+                - media/webrtc/trunk/tools/gyp/**
+                - memory/**
+                - mfbt/**
+                - modules/fdlibm/**
+                - modules/zlib/src/**
+                - mozglue/**
+                - moz.configure
+                - nsprpub/**
+                - python/**
+                - taskcluster/moz.build
+                - testing/mozbase/**
+                - test.mozbuild
+                - toolkit/mozapps/installer/package-name.mk
+                - toolkit/mozapps/installer/upload-files.mk
+
+    sm-plain/debug:
+        description: "Spidermonkey Plain"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-plaindebug.debug
+                gecko-v2: sm-plaindebug-debug
+        treeherder:
+            platform: linux64/debug
+            symbol: SM-tc(p)
+        run:
+            spidermonkey-variant: plaindebug
+        run-on-projects:
+            - integration
+            - release
+
+    sm-plain/opt:
+        description: "Spidermonkey Plain"
+        index:
+            job-name: sm-plain-opt
+        treeherder:
+            symbol: SM-tc(p)
+        run:
+            spidermonkey-variant: plain
+        run-on-projects:
+            - integration
+            - release
+
+    sm-arm-sim/debug:
+        description: "Spidermonkey ARM sim"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-arm-sim.debug
+                gecko-v2: sm-arm-sim-debug
+        treeherder:
+            symbol: SM-tc(arm)
+        run:
+            spidermonkey-variant: arm-sim
+        run-on-projects:
+            - integration
+            - release
+
+    sm-arm64-sim/debug:
+        description: "Spidermonkey ARM64 sim"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-arm64-sim.debug
+                gecko-v2: sm-arm64-sim-debug
+        treeherder:
+            symbol: SM-tc(arm64)
+        run:
+            spidermonkey-variant: arm64-sim
+        run-on-projects:
+            - integration
+            - release
+
+    sm-asan/opt:
+        description: "Spidermonkey Address Sanitizer"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-asan.opt
+                gecko-v2: sm-asan-opt
+        treeherder:
+            symbol: SM-tc(asan)
+        run:
+            spidermonkey-variant: asan
+            tooltool-manifest: browser/config/tooltool-manifests/linux64/asan.manifest
+        run-on-projects:
+            - integration
+            - release
+
+    sm-compacting/debug:
+        description: "Spidermonkey Compacting"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-compacting.debug
+                gecko-v2: sm-compacting-debug
+        treeherder:
+            symbol: SM-tc(cgc)
+        run:
+            spidermonkey-variant: compacting
+        run-on-projects:
+            - integration
+            - release
+
+    sm-msan/opt:
+        description: "Spidermonkey Memory Sanitizer"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-msan.opt
+                gecko-v2: sm-msan-opt
+        treeherder:
+            symbol: SM-tc(msan)
+        run:
+            spidermonkey-variant: msan
+            tooltool-manifest: browser/config/tooltool-manifests/linux64/msan.manifest
+        run-on-projects:
+            - integration
+            - release
+
+    sm-tsan/opt:
+        description: "Spidermonkey Thread Sanitizer"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-tsan.opt
+                gecko-v2: sm-tsan-opt
+        treeherder:
+            symbol: SM-tc(tsan)
+            tier: 3
+        run-on-projects: []
+        run:
+            spidermonkey-variant: tsan
+            tooltool-manifest: browser/config/tooltool-manifests/linux64/tsan.manifest
+
+    sm-rootanalysis/debug:
+        description: "Spidermonkey Root Analysis"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-rootanalysis.debug
+                gecko-v2: sm-rootanalysis-debug
+        treeherder:
+            symbol: SM-tc(r)
+        run:
+            spidermonkey-variant: rootanalysis
+        run-on-projects:
+            - integration
+            - release
+
+    sm-nonunified/debug:
+        description: "Spidermonkey Non-Unified Debug"
+        index:
+            job-name:
+                buildbot: sm-plain
+                gecko-v1: sm-nonunified.debug
+                gecko-v2: sm-nonunified-debug
+        treeherder:
+            platform: linux64/debug
+            symbol: SM-tc(nu)
+        run:
+            spidermonkey-variant: nonunified
+        run-on-projects:
+            - integration
+            - release
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -59,16 +59,22 @@ artifact.
 
 toolchain
 ---------
 
 Toolchain builds create the compiler toolchains used to build Firefox.  These
 will eventually be dependencies of the builds themselves, but for the moment
 are run manually via try pushes and the results uploaded to tooltool.
 
+spidermonkey
+------------
+
+Spidermonkey tasks check out the full gecko source tree, then compile only the
+spidermonkey portion.  Each task runs specific tests after the build.
+
 Tests
 -----
 
 Test tasks for Gecko products are divided into several kinds, but share a
 common implementation.  The process goes like this, based on a set of YAML
 files named in ``kind.yml``:
 
  * For each build task, determine the related test platforms based on the build
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running spidermonkey jobs via dedicated scripts
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+from voluptuous import Schema, Required, Optional, Any
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_gecko_vcs_env_vars,
+    docker_worker_add_tc_vcs_cache,
+    docker_worker_add_public_artifacts
+)
+
+sm_run_schema = Schema({
+    Required('using'): Any('spidermonkey', 'spidermonkey-package'),
+
+    # The SPIDERMONKEY_VARIANT
+    Required('spidermonkey-variant'): basestring,
+
+    # The tooltool manifest to use; default from sm-tooltool-config.sh  is used
+    # if omitted
+    Optional('tooltool-manifest'): basestring,
+})
+
+
+@run_job_using("docker-worker", "spidermonkey")
+@run_job_using("docker-worker", "spidermonkey-package")
+def docker_worker_spidermonkey(config, job, taskdesc, schema=sm_run_schema):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['artifacts'] = []
+    worker['caches'] = []
+
+    if int(config.params['level']) > 1:
+        worker['caches'].append({
+            'type': 'persistent',
+            'name': 'level-{}-{}-build-spidermonkey-workspace'.format(
+                config.params['level'], config.params['project']),
+            'mount-point': "/home/worker/workspace",
+        })
+
+    # include secrets access
+    # TODO (taskdiff): this is unused? remove it
+    worker['taskcluster-proxy'] = True
+    taskdesc['scopes'].append(
+        'secrets:get:project/releng/gecko/build/level-{}/*'.format(config.params['level']))
+
+    docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZHARNESS_DISABLE': 'true',
+        'TOOLS_DISABLE': 'true',
+        'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
+        'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
+        'MOZ_SCM_LEVEL': config.params['level'],
+    })
+
+    # tooltool downloads; note that this script downloads using the API
+    # endpoiint directly, rather than via relengapi-proxy
+    worker['caches'].append({
+        'type': 'persistent',
+        'name': 'tooltool-cache',
+        'mount-point': '/home/worker/tooltool-cache',
+    })
+    env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+    env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+    env['TOOLTOOL_REV'] = 'master'
+    if run.get('tooltool-manifest'):
+        env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
+
+    script = "build-sm.sh"
+    if run['using'] == 'spidermonkey-package':
+        script = "build-sm-package.sh"
+
+    worker['command'] = [
+        "/bin/bash",
+        "-c",
+        "cd /home/worker/ "
+        "&& ./bin/checkout-sources.sh "
+        "&& ./workspace/build/src/taskcluster/scripts/builder/" + script
+    ]
--- a/taskcluster/taskgraph/try_option_syntax.py
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -25,16 +25,17 @@ BUILD_TYPE_ALIASES = {
 BUILD_KINDS = set([
     'build',
     'artifact-build',
     'hazard',
     'l10n',
     'upload-symbols',
     'valgrind',
     'static-analysis',
+    'spidermonkey',
 ])
 
 # anything in this list is governed by -j
 JOB_KINDS = set([
     'source-check',
     'toolchain',
 ])