Bug 1475541 - Make QR test jobs tier-1 by default. r=aryx
authorKartikaya Gupta <kgupta@mozilla.com>
Fri, 13 Jul 2018 09:44:06 -0400
changeset 426494 8af3291751e8bf79b1cc5054734ea64ea2afcec9
parent 426493 d93abdc258e13d0a2686da0c625b22a956c4d6aa
child 426495 37d172c987e66ad9fce9f381072f33c4f3b39e7f
push id34274
push usernerli@mozilla.com
push dateFri, 13 Jul 2018 21:51:36 +0000
treeherdermozilla-central@e37eeb019cf8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaryx
bugs1475541
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1475541 - Make QR test jobs tier-1 by default. r=aryx This patch makes the QR test platforms tier-1 by default, and removes the ad-hoc bits that were making individual QR jobs tier-1 before. However, it also explicitly downgrades some QR jobs to tier-2 or tier-3; comments in the yml files indicate why. MozReview-Commit-ID: 1UfPuhcMvIW
taskcluster/ci/test/mochitest.yml
taskcluster/ci/test/reftest.yml
taskcluster/ci/test/talos.yml
taskcluster/ci/test/web-platform.yml
taskcluster/taskgraph/transforms/tests.py
--- a/taskcluster/ci/test/mochitest.yml
+++ b/taskcluster/ci/test/mochitest.yml
@@ -38,16 +38,20 @@ mochitest:
             linux64-jsdcov/opt: mochitest/plain-chunked-coverage
             default: mochitest/plain-chunked
     treeherder-symbol: M()
     loopback-video: true
     run-on-projects:
         by-test-platform:
             windows10-64-qr/debug: []  # bug 1424755
             default: built-projects
+    tier:
+        by-test-platform:
+            windows10-64-qr/debug: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
+            default: default
     instance-size:
         by-test-platform:
             linux64-jsdcov/opt: xlarge
             android-em.*: xlarge
             default: default
     virtualization:
         by-test-platform:
             windows10-64-qr/.*: virtual-with-gpu
@@ -260,20 +264,16 @@ mochitest-gpu:
         mochitest-flavor: plain
         extra-options:
             by-test-platform:
                 android-em.*:
                     # note that Android runs fewer suites than other platforms
                     - --test-suite=mochitest-plain-gpu
                 default:
                     - --mochitest-suite=plain-gpu,chrome-gpu,browser-chrome-gpu
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-media:
     description: "Mochitest media run"
     suite: mochitest/mochitest-media
     treeherder-symbol: M(mda)
     max-run-time:
         by-test-platform:
             windows10-64-ccov/debug: 7200
@@ -300,17 +300,16 @@ mochitest-media:
             by-test-platform:
                 android.*: false
                 macosx64.*: false
                 windows10-64.*: false
                 default: true
     tier:
         by-test-platform:
             android-hw.*: 3
-            linux64-qr/.*: 1
             default: default
 
 mochitest-plain-headless:
     description: "Mochitest plain headless run"
     suite: mochitest/plain-chunked
     treeherder-symbol: M(h)
     loopback-video: true
     chunks:
@@ -361,20 +360,16 @@ mochitest-webgl1-core:
     max-run-time:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-webgl1-ext:
     description: "Mochitest webgl1-ext run"
     suite: mochitest/mochitest-webgl1-ext
     treeherder-symbol: M(gl1e)
     run-on-projects:
         by-test-platform:
             windows10-64-ccov/.*: []  # Do not run on Windows ccov, see bug 1419475.
@@ -388,20 +383,16 @@ mochitest-webgl1-ext:
     max-run-time:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-webgl2-core:
     description: "Mochitest webgl2-core run"
     suite: mochitest/mochitest-webgl2-core
     treeherder-symbol: M(gl2c)
     run-on-projects:
         by-test-platform:
             windows10-64-ccov/.*: []  # Do not run on Windows ccov, see bug 1419475.
@@ -415,20 +406,16 @@ mochitest-webgl2-core:
     max-run-time:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-webgl2-ext:
     description: "Mochitest webgl2-ext run"
     suite: mochitest/mochitest-webgl2-ext
     treeherder-symbol: M(gl2e)
     run-on-projects:
         by-test-platform:
             windows10-64-ccov/.*: []  # Do not run on Windows ccov, see bug 1419475.
@@ -444,20 +431,16 @@ mochitest-webgl2-ext:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
         chunked: true
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 mochitest-webgl2-deqp:
     description: "Mochitest webgl2-deqp run"
     suite: mochitest/mochitest-webgl2-deqp
     treeherder-symbol: M(gl2d)
     run-on-projects:
         by-test-platform:
             windows10-64-ccov/.*: []  # Do not run on Windows ccov, see bug 1419475.
@@ -473,12 +456,8 @@ mochitest-webgl2-deqp:
         by-test-platform:
             windows.*: 5400
             default: 3600
     # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety
     allow-software-gl-layers: false
     mozharness:
         mochitest-flavor: plain
         chunked: true
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
--- a/taskcluster/ci/test/reftest.yml
+++ b/taskcluster/ci/test/reftest.yml
@@ -42,20 +42,16 @@ crashtest:
             android-em-4.3-arm7-api-16/debug: 10
             android-em-7.0-x86/opt: 1
             android-em.*: 4
             default: 1
     e10s:
         by-test-platform:
             linux32/debug: both
             default: true
-    tier:
-        by-test-platform:
-            .*-qr/.*: 1
-            default: default
 
 jsreftest:
     description: "JS Reftest run"
     suite: reftest/jsreftest
     schedules-component: jsreftest  # scheduling for this reftest is different from the others..
     treeherder-symbol: R(J)
     run-on-projects: built-projects
     instance-size:
@@ -74,20 +70,16 @@ jsreftest:
             linux32/debug: 5
             macosx.*: 2
             default: 3
     max-run-time:
         by-test-platform:
             android-em.*: 7200
             windows10-64-ccov/debug: 7200
             default: 3600
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 reftest:
     description: "Reftest run"
     suite: reftest/reftest
     treeherder-symbol: R(R)
     run-on-projects: built-projects
     instance-size:
         by-test-platform:
@@ -119,20 +111,16 @@ reftest:
             default: 3600
     mozharness:
         chunked:
             by-test-platform:
                 android-em.*: false
                 macosx64/opt: false
                 windows10-64.*/opt: false
                 default: true
-    tier:
-        by-test-platform:
-            linux64-qr/.*: 1
-            default: default
 
 reftest-gpu:
     description: "Reftest GPU run"
     suite: reftest/reftest-gpu
     treeherder-symbol: R(Rg)
     chunks:
         by-test-platform:
             windows.*/opt: 2
--- a/taskcluster/ci/test/talos.yml
+++ b/taskcluster/ci/test/talos.yml
@@ -28,29 +28,37 @@ job-defaults:
 talos-bcv:
     description: "Talos basic compositor video"
     try-name: bcv
     treeherder-symbol: T(bcv)
     run-on-projects:
         by-test-platform:
             .*-qr/.*: []  # this test is not useful with webrender
             default: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
+    tier:
+        by-test-platform:
+            .*-qr/.*: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
+            default: default
     max-run-time: 1800
     mozharness:
         extra-options:
             - --suite=bcv
 
 talos-bcv-profiling:
     description: "Talos profiling basic compositor video"
     try-name: bcv-profiling
     treeherder-symbol: T-P(bcv)
     run-on-projects:
         by-test-platform:
             .*-qr/.*: []  # this test is not useful with webrender
             default: ['mozilla-central', 'try']
+    tier:
+        by-test-platform:
+            .*-qr/.*: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
+            default: default
     max-run-time: 1800
     mozharness:
         extra-options:
             - --suite=bcv
             - --geckoProfile
 
 talos-chrome:
     description: "Talos chrome"
@@ -479,15 +487,19 @@ talos-xperf:
     description: "Talos xperf"
     try-name: xperf
     treeherder-symbol: T(x)
     virtualization: virtual
     run-on-projects:
         by-test-platform:
             windows7-32.*: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
             default: []
+    tier:
+        by-test-platform:
+            windows7-32.*: default
+            default: 3  # this should be disabled but might run via try syntax anyway, so explicitly downgrade to tier-3
     mozharness:
         extra-options:
             - --suite=xperf
         config:
             by-test-platform:
                 windows.*:
                     - talos/windows_vm_config.py
--- a/taskcluster/ci/test/web-platform.yml
+++ b/taskcluster/ci/test/web-platform.yml
@@ -29,18 +29,22 @@ web-platform-tests:
             windows10-64-ccov/debug: 10800
             default: 7200
     e10s:
         by-test-platform:
             linux32/debug: both
             default: true
     run-on-projects:
         by-test-platform:
-            linux64-qr/.*: ['release', 'try']
+            linux64-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
+    tier:
+        by-test-platform:
+            linux64-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
+            default: default
     mozharness:
         chunked: true
         extra-options:
             - --test-type=testharness
 
 web-platform-tests-headless:
     description: "Web platform test headless run"
     suite: web-platform-tests
@@ -79,18 +83,22 @@ web-platform-tests-reftests:
             windows.*: 1
             default: 6
     e10s:
         by-test-platform:
             linux32/debug: both
             default: true
     run-on-projects:
         by-test-platform:
-            linux64-qr/.*: ['release', 'try']
+            linux64-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
+    tier:
+        by-test-platform:
+            linux64-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
+            default: default
     mozharness:
         extra-options:
             - --test-type=reftest
 
 web-platform-tests-reftests-headless:
     description: "Web platform reftest headless run"
     suite: web-platform-tests-reftests
     treeherder-symbol: W(WrH)
@@ -119,18 +127,22 @@ web-platform-tests-wdspec:
     description: "Web platform webdriver-spec run"
     suite: web-platform-tests-wdspec
     treeherder-symbol: W(Wd)
     mozharness:
         extra-options:
             - --test-type=wdspec
     run-on-projects:
         by-test-platform:
-            linux64-qr/.*: ['release', 'try']
+            linux64-qr/.*: ['release', 'try']  # skip on integration branches due to high load
             default: built-projects
+    tier:
+        by-test-platform:
+            linux64-qr/.*: 2  # can't be tier-1 if it's not running on integration branches
+            default: default
 
 web-platform-tests-wdspec-headless:
     description: "Web platform webdriver-spec headless run"
     suite: web-platform-tests-wdspec
     treeherder-symbol: W(WdH)
     run-on-projects: []  # disabled pending releng approval
     mozharness:
         extra-options:
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -592,31 +592,37 @@ def set_tier(config, tests):
                                          'linux32-nightly/opt',
                                          'linux32-devedition/opt',
                                          'linux64/opt',
                                          'linux64-nightly/opt',
                                          'linux64/debug',
                                          'linux64-pgo/opt',
                                          'linux64-devedition/opt',
                                          'linux64-asan/opt',
+                                         'linux64-qr/opt',
+                                         'linux64-qr/debug',
                                          'windows7-32/debug',
                                          'windows7-32/opt',
                                          'windows7-32-pgo/opt',
                                          'windows7-32-devedition/opt',
                                          'windows7-32-nightly/opt',
                                          'windows10-64/debug',
                                          'windows10-64/opt',
                                          'windows10-64-pgo/opt',
                                          'windows10-64-devedition/opt',
                                          'windows10-64-nightly/opt',
                                          'windows10-64-asan/opt',
+                                         'windows10-64-qr/opt',
+                                         'windows10-64-qr/debug',
                                          'macosx64/opt',
                                          'macosx64/debug',
                                          'macosx64-nightly/opt',
                                          'macosx64-devedition/opt',
+                                         'macosx64-qr/opt',
+                                         'macosx64-qr/debug',
                                          'android-em-4.3-arm7-api-16/opt',
                                          'android-em-4.3-arm7-api-16/debug',
                                          'android-em-4.2-x86/opt']:
                 test['tier'] = 1
             else:
                 test['tier'] = 2
 
         yield test