Bug 1352113 - Shippable Builds - utilize run-on-projects more for previously excluded tests. r=jmaher
authorJustin Wood <Callek@gmail.com>
Wed, 27 Mar 2019 13:44:42 +0000
changeset 466349 cc1547a13f153514ee7bc3f38589893939e24824
parent 466348 ece439cc2d6dd49fb167390fc7715cde50776c04
child 466350 d7d2f0b6d43adc50dc1d5588e66a09b8ad391d1b
push id35768
push useropoprus@mozilla.com
push dateThu, 28 Mar 2019 09:55:54 +0000
treeherdermozilla-central@c045dd97faf2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1352113
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1352113 - Shippable Builds - utilize run-on-projects more for previously excluded tests. r=jmaher Effectively back out much of the run on projects from D22710 This also has the added affect of scheduling the shippable builds to run because of dependencies. Differential Revision: https://phabricator.services.mozilla.com/D22833
taskcluster/ci/test/misc.yml
taskcluster/ci/test/mochitest.yml
taskcluster/ci/test/talos.yml
taskcluster/ci/test/web-platform.yml
--- a/taskcluster/ci/test/misc.yml
+++ b/taskcluster/ci/test/misc.yml
@@ -87,17 +87,16 @@ test-verify:
     allow-software-gl-layers: false
     run-on-projects:
         by-test-platform:
             android-em-4.3-arm7-api-16/opt: ['try']
             # do not run on ccov
             .*-ccov/.*: []
             .*-asan/.*: []
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             # do not run on beta or release: usually just confirms earlier results
             default: ['trunk', 'try']
     tier: 2
     mozharness:
         script:
             by-test-platform:
                 android-em.*: android_emulator_unittest.py
                 default: desktop_unittest.py
@@ -132,17 +131,16 @@ test-verify-gpu:
     max-run-time: 10800
     allow-software-gl-layers: false
     run-on-projects:
         by-test-platform:
             # do not run on ccov
             .*-ccov/.*: []
             .*-asan/.*: []
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             # do not run on beta or release: usually just confirms earlier results
             default: ['trunk', 'try']
     tier: 2
     mozharness:
         script:
             by-test-platform:
                 android-em.*: android_emulator_unittest.py
                 default: desktop_unittest.py
--- a/taskcluster/ci/test/mochitest.yml
+++ b/taskcluster/ci/test/mochitest.yml
@@ -248,18 +248,17 @@ mochitest-devtools-webreplay:
     loopback-video: true
     tier: 2
     max-run-time: 900
     mozharness:
         mochitest-flavor: chrome
     allow-software-gl-layers: false
     run-on-projects:
         by-test-platform:
-            macosx64(!-shippable)*/opt: ['trunk', 'try']
-            .*shippable/opt: built-projects
+            macosx64.*/opt: ['trunk', 'try']
             default: []
 
 mochitest-gpu:
     description: "Mochitest GPU run"
     suite: mochitest/gpu
     treeherder-symbol: M(gpu)
     loopback-video: true
     instance-size:
--- a/taskcluster/ci/test/talos.yml
+++ b/taskcluster/ci/test/talos.yml
@@ -28,17 +28,16 @@ job-defaults:
 talos-bcv:
     description: "Talos basic compositor video"
     try-name: bcv
     treeherder-symbol: T(bcv)
     run-on-projects:
         by-test-platform:
             .*-qr/.*: []  # this test is not useful with webrender
             (?:windows10-64|windows7-32|linux64)/opt: ['mozilla-central', 'try']
-            .*shippable/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     tier:
         by-test-platform:
             .*-qr/.*: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
             default: default
     max-run-time: 1800
     mozharness:
         extra-options:
@@ -46,123 +45,115 @@ talos-bcv:
 
 talos-chrome:
     description: "Talos chrome"
     try-name: chromez
     treeherder-symbol: T(c)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     max-run-time: 1200
     mozharness:
         extra-options:
             - --suite=chromez
 
 talos-damp:
     description: "Talos devtools (damp)"
     try-name: damp
     treeherder-symbol: T(damp)
     max-run-time: 5400
     run-on-projects:
         by-test-platform:
             linux64-ccov/.*: ['try']  # Bug 1407593
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     mozharness:
         extra-options:
             - --suite=damp
 
 talos-dromaeojs:
     description: "Talos dromaeojs"
     try-name: dromaeojs
     treeherder-symbol: T(d)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     max-run-time: 2100
     mozharness:
         extra-options:
             - --suite=dromaeojs
 
 talos-flex:
     description: "Talos XUL flexbox emulation enabled"
     try-name: flex
     treeherder-symbol: T(f)
     tier: 3
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-central', 'try']
     max-run-time: 1800
     mozharness:
         extra-options:
             - --suite=flex
 
 talos-g1:
     description: "Talos g1"
     try-name: g1
     treeherder-symbol: T(g1)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     max-run-time:
         by-test-platform:
             linux64.*: 3600
             default: 7200
     mozharness:
         extra-options:
             - --suite=g1
 
 talos-g3:
     description: "Talos g3"
     try-name: g3
     treeherder-symbol: T(g3)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     max-run-time: 900
     mozharness:
         extra-options:
             - --suite=g3
 
 talos-g4:
     description: "Talos g4"
     try-name: g4
     treeherder-symbol: T(g4)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     max-run-time:
         by-test-platform:
             linux64.*: 1500
             default: 1800
     mozharness:
         extra-options:
             - --suite=g4
 
 talos-g5:
     description: "Talos g5"
     try-name: g5
     treeherder-symbol: T(g5)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     mozharness:
         extra-options:
             - --suite=g5
     max-run-time:
         by-test-platform:
             linux64.*: 1200
             default: 1800
@@ -219,59 +210,55 @@ talos-motionmark:
 
 talos-other:
     description: "Talos other"
     try-name: other
     treeherder-symbol: T(o)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     max-run-time: 1500
     mozharness:
         extra-options:
             - --suite=other
 
 talos-sessionrestore-many-windows:
     description: "Talos sessionrestore-many-windows"
     try-name: sessionrestore-many-windows
     treeherder-symbol: T(smw)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-central', 'try']
     max-run-time: 1500
     mozharness:
         extra-options:
             - --suite=sessionrestore-many-windows
 
 talos-perf-reftest:
     description: "Talos perf-reftest"
     try-name: perf-reftest
     treeherder-symbol: T(p)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['trunk', 'try']
     max-run-time: 1200
     mozharness:
         extra-options:
             - --suite=perf-reftest
 
 talos-perf-reftest-singletons:
     description: "Talos perf-reftest singletons"
     try-name: perf-reftest-singletons
     treeherder-symbol: T(ps)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['trunk', 'try']
     max-run-time: 1200
     mozharness:
         extra-options:
             - --suite=perf-reftest-singletons
 
 talos-speedometer:
     description: "Talos speedometer"
@@ -288,31 +275,29 @@ talos-speedometer:
 
 talos-svgr:
     description: "Talos svgr"
     try-name: svgr
     treeherder-symbol: T(s)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     max-run-time: 1800
     mozharness:
         extra-options:
             - --suite=svgr
 
 talos-tp5o:
     description: "Talos tp5o"
     try-name: tp5o
     treeherder-symbol: T(tp)
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     max-run-time: 1800
     mozharness:
         extra-options:
             - --suite=tp5o
 
 talos-tp6:
     description: "Talos tp6"
@@ -329,34 +314,32 @@ talos-tp6:
 
 talos-tp6-stylo-threads:
     description: "Talos Stylo sequential tp6"
     try-name: tp6-stylo-threads
     treeherder-symbol: Tss(tp6)
     max-run-time: 1200
     run-on-projects:
         by-test-platform:
-            macosx64(?!-shippable)*/.*: ['mozilla-beta', 'autoland', 'try']
+            macosx64.*: ['mozilla-beta', 'autoland', 'try']
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     mozharness:
         extra-options:
             - --suite=tp6-stylo-threads
 
 talos-tabswitch:
     description: "Talos page scroll (tabswitch)"
     try-name: tabswitch
     treeherder-symbol: T(tabswitch)
     max-run-time: 900
     run-on-projects:
         by-test-platform:
             linux64-ccov/.*: ['try']  # Bug 1407593
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
             default: ['mozilla-beta', 'trunk', 'try']
     mozharness:
         extra-options:
             - --suite=tabswitch
 
 talos-xperf:
     description: "Talos xperf"
     try-name: xperf
@@ -367,19 +350,18 @@ talos-xperf:
         by-test-platform:
             windows7-32.*: false
             windows10-aarch64.*: false
             windows10-64.*: true
     run-on-projects:
         by-test-platform:
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
             windows10-aarch64/opt: []
-            windows7-32-pgo(?:-qr)?/.*: ['mozilla-beta', 'trunk', 'try']
-            windows10-64-(?!qr|shippable).*/opt: ['mozilla-beta', 'trunk', 'try']
-            .*shippable(?:-qr)?/opt: built-projects
+            windows7-32-(?:pgo|shippable)(?:-qr)?/.*: ['mozilla-beta', 'trunk', 'try']
+            windows10-64-(?:pgo|shippable)(?:-qr)?/.*: ['mozilla-beta', 'trunk', 'try']
             default: []
     tier:
         by-test-platform:
             windows7-32.*: default
             windows10-64(-pgo)?/.*: default
             windows10-64-ccov/debug: 3
             default: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
     mozharness:
--- a/taskcluster/ci/test/web-platform.yml
+++ b/taskcluster/ci/test/web-platform.yml
@@ -46,18 +46,17 @@ web-platform-tests:
     e10s:
         by-test-platform:
             linux32/debug: both
             default: true
     run-on-projects:
         by-test-platform:
             android.*: ['mozilla-central', 'try']
             windows10-aarch64/opt: ['try', 'mozilla-central']
-            .*(?<!shippable)-qr/.*: ['release', 'try']  # skip on integration branches due to high load
-            .*shippable/opt: built-projects
+            .*-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
     tier:
         by-test-platform:
             android.*: 2
             linux64-asan/opt: 2
             .*-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
             default: default
     mozharness:
@@ -152,18 +151,17 @@ web-platform-tests-wdspec:
             default: 2
     mozharness:
         extra-options:
             - --test-type=wdspec
     run-on-projects:
         by-test-platform:
             windows10-aarch64/opt: ['try', 'mozilla-central']
             android.*: ['mozilla-central', 'try']
-            .*(?<!shippable)-qr/.*: ['release', 'try']  # skip on integration branches due to high load
-            .*shippable(?:-qr)?/opt: built-projects
+            .*-qr/.*: ['release', 'try']
             default: built-projects
     tier:
         by-test-platform:
             android.*: 3
             linux64-asan/opt: 2
             .*-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
             default: default
 
@@ -190,17 +188,16 @@ test-verify-wpt:
     treeherder-symbol: TVw
     max-run-time: 10800
     run-on-projects:
         by-test-platform:
             # do not run on ccov
             .*-ccov/.*: []
             .*-asan/.*: []
             (?:windows10-64|windows7-32|linux64)(?:-qr)?/opt: ['mozilla-central', 'try']
-            .*shippable/opt: built-projects
             # do not run on beta or release: usually just confirms earlier results
             default: ['trunk', 'try']
     tier: 2
     mozharness:
         extra-options:
             - --verify
 
 test-coverage-wpt: