Bug 1380053 - Add Stylo tests for other desktop platforms. r=catlee
authorJ. Ryan Stinnett <jryans@gmail.com>
Tue, 18 Jul 2017 14:32:06 -0500
changeset 422608 30fb5403e45cdae7577bbfef400b384f944af66d
parent 422607 8f1268aef844ddf1fecca6512a57c04c8c40ae51
child 422609 8816877de730339d9355892a0cd175d99cd4a6c2
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscatlee
bugs1380053
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1380053 - Add Stylo tests for other desktop platforms. r=catlee Enable Stylo tests for win32, win64, and macosx64. Talos will be considered separately. MozReview-Commit-ID: BvTkddDX2Kk
taskcluster/ci/test/test-platforms.yml
taskcluster/ci/test/tests.yml
taskcluster/taskgraph/transforms/job/mozharness_test.py
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/configs/unittests/linux_unittest.py
testing/mozharness/configs/unittests/mac_unittest.py
testing/mozharness/configs/unittests/win_taskcluster_unittest.py
--- a/taskcluster/ci/test/test-platforms.yml
+++ b/taskcluster/ci/test/test-platforms.yml
@@ -80,16 +80,25 @@ linux64-pgo/opt:
 linux64-asan/opt:
     build-platform: linux64-asan/opt
     test-sets:
         - common-tests
         - headless
 
 # Stylo builds only run a subset of tests for the moment. So give them
 # their own test set.
+# Stylo doesn't work on 32-bit Linux yet (bug 1385025)
+# linux32-stylo/debug:
+#     build-platform: linux/debug
+#     test-sets:
+#         - stylo-tests
+# linux32-stylo/opt:
+#     build-platform: linux/opt
+#     test-sets:
+#         - stylo-tests
 linux64-stylo/debug:
     build-platform: linux64/debug
     test-sets:
         - stylo-tests
 linux64-stylo/opt:
     build-platform: linux64/opt
     test-sets:
         - stylo-tests
@@ -98,16 +107,41 @@ linux64-stylo-sequential/debug:
     build-platform: linux64/debug
     test-sets:
         - stylo-sequential-tests
 linux64-stylo-sequential/opt:
     build-platform: linux64/opt
     test-sets:
         - stylo-sequential-tests
         - talos
+macosx64-stylo/debug:
+    build-platform: macosx64/debug
+    test-sets:
+        - stylo-tests
+macosx64-stylo/opt:
+    build-platform: macosx64/opt
+    test-sets:
+        - stylo-tests
+# Windows needs more work to green up (bug 1385027)
+# windows7-32-stylo/debug:
+#     build-platform: win32/debug
+#     test-sets:
+#         - stylo-tests
+# windows7-32-stylo/opt:
+#     build-platform: win32/opt
+#     test-sets:
+#         - stylo-tests
+# windows10-64-stylo/debug:
+#     build-platform: win64/debug
+#     test-sets:
+#         - stylo-tests
+# windows10-64-stylo/opt:
+#     build-platform: win64/opt
+#     test-sets:
+#         - stylo-tests
 
 # QR builds just run a subset right now. Note that the tests in this
 # test set are further restricted in tests.yml to run on certain projects
 # only, to avoid adding too much infra load.
 linux64-qr/opt:
     build-platform: linux64/opt
     test-sets:
         - qr-tests
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -83,16 +83,18 @@ crashtest:
     chunks:
         by-test-platform:
             android-4.3-arm7-api-15/debug: 10
             android.*: 4
             default: 1
     e10s:
         by-test-platform:
             macosx64/debug: true
+            # Bug 1384701: e10s-off parallel reftest runs crash on shutdown
+            linux64-stylo/debug: true
             default: both
     mozharness:
         by-test-platform:
             android.*:
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     - android/androidarm_4_3.py
@@ -667,16 +669,18 @@ mochitest-chrome-style:
     docker-image: {"in-tree": "desktop1604-test"}
     e10s: false
     mozharness:
         mochitest-flavor: chrome
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                windows.*:
+                    - unittests/win_taskcluster_unittest.py
                 macosx.*:
                     - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --mochitest-suite=chrome-style
 
@@ -944,23 +948,29 @@ mochitest-media:
             default: default
 
 mochitest-style:
     description: "Mochitest plain run for style system"
     suite: mochitest/plain-style
     treeherder-symbol: tc-M(s)
     loopback-video: true
     docker-image: {"in-tree": "desktop1604-test"}
-    e10s: both
+    e10s:
+        by-test-platform:
+            # Bug 1384701: e10s-off parallel reftest runs crash on shutdown
+            linux64-stylo/debug: true
+            default: both
     mozharness:
         mochitest-flavor: plain
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                windows.*:
+                    - unittests/win_taskcluster_unittest.py
                 macosx.*:
                     - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --mochitest-suite=plain-style
 
@@ -1094,16 +1104,18 @@ reftest:
             default: 8
     max-run-time:
         by-test-platform:
             android.*: 10800
             default: 3600
     e10s:
         by-test-platform:
             macosx.*: true
+            # Bug 1384701: e10s-off parallel reftest runs crash on shutdown
+            linux64-stylo/debug: true
             default: both
     mozharness:
         by-test-platform:
             android.*:
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     - android/androidarm_4_3.py
@@ -1222,32 +1234,30 @@ reftest-no-accel:
 reftest-stylo:
     description: "Reftest run for Stylo"
     suite: reftest/reftest-stylo
     treeherder-symbol: tc-R(Rs)
     docker-image: {"in-tree": "desktop1604-test"}
     chunks: 16
     run-on-projects:
         by-test-platform:
-            linux64-stylo/opt: [ 'autoland', 'mozilla-central', 'try' ] # not on mozilla-inbound
+            .*-stylo/opt: [ 'autoland', 'mozilla-central', 'try' ] # not on mozilla-inbound
             linux64-stylo-sequential/opt: [ 'mozilla-central']
             default: built-projects
     e10s:
         # run only e10s tests for this platform see bug 1343301
         by-test-platform:
-            linux64-stylo/opt:
+            .*-stylo/opt:
                 # no opt on inbound see bug 1339604
                 by-project:
-                    stylo: true
                     autoland: true
                     mozilla-central: true
                     default: true
-            linux64-stylo/debug:
+            .*-stylo/debug:
                 by-project:
-                    stylo: true
                     autoland: true
                     mozilla-central: true
                     mozilla-inbound: true
                     default: true
             linux64-stylo-sequential/opt:
                 by-project:
                     mozilla-central: true
                     default: true
@@ -1255,16 +1265,18 @@ reftest-stylo:
                 by-project:
                     mozilla-central: true
                     default: true
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
+                windows.*:
+                    - unittests/win_taskcluster_unittest.py
                 macosx.*:
                     - unittests/mac_unittest.py
                 default:
                     - unittests/linux_unittest.py
                     - remove_executables.py
         extra-options:
             - --reftest-suite=reftest-stylo
 
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -32,21 +32,23 @@ BUILDER_NAME_PREFIX = {
     'macosx64-devedition': 'Rev7 MacOSX Yosemite 10.10.5 DevEdition',
     'android-4.3-arm7-api-15': 'Android 4.3 armv7 API 15+',
     'android-4.2-x86': 'Android 4.2 x86 Emulator',
     'android-4.3-arm7-api-15-gradle': 'Android 4.3 armv7 API 15+',
     'windows10-64': 'Windows 10 64-bit',
     'windows10-64-nightly': 'Windows 10 64-bit',
     'windows10-64-pgo': 'Windows 10 64-bit',
     'windows10-64-asan': 'Windows 10 64-bit',
+    'windows10-64-stylo': 'Windows 10 64-bit',
     'windows7-32': 'Windows 7 32-bit',
     ('windows7-32', 'virtual-with-gpu'): 'Windows 7 VM-GFX 32-bit',
     'windows7-32-nightly': 'Windows 7 32-bit',
     'windows7-32-devedition': 'Windows 7 32-bit DevEdition',
     'windows7-32-pgo': 'Windows 7 32-bit',
+    'windows7-32-stylo': 'Windows 7 32-bit',
     'windows8-64': 'Windows 8 64-bit',
     'windows8-64-nightly': 'Windows 8 64-bit',
     'windows8-64-devedition': 'Windows 8 64-bit DevEdition',
     'windows8-64-pgo': 'Windows 8 64-bit',
 }
 
 VARIANTS = [
     'nightly',
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -61,16 +61,21 @@ WINDOWS_WORKER_TYPES = {
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
       'hardware': 'releng-hardware/gecko-t-win7-32-hw',
     },
     'windows7-32-devedition': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
       'hardware': 'releng-hardware/gecko-t-win7-32-hw',
     },
+    'windows7-32-stylo': {
+      'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
+      'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
+      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+    },
     'windows10-64': {
       'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
       'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows10-64-pgo': {
       'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
@@ -81,16 +86,21 @@ WINDOWS_WORKER_TYPES = {
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
       'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows10-64-nightly': {
       'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
       'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
+    'windows10-64-stylo': {
+      'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
+      'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
+    },
     'windows10-64-asan': {
       'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
       'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     # These values don't really matter since BBB will be executing them
     'windows8-64': {
       'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
@@ -482,18 +492,28 @@ def set_treeherder_machine_platform(conf
         # The build names for Android platforms have partially evolved over the
         # years and need to be translated.
         'android-api-15/debug': 'android-4-3-armv7-api15/debug',
         'android-api-15/opt': 'android-4-3-armv7-api15/opt',
         'android-x86/opt': 'android-4-2-x86/opt',
         'android-api-15-gradle/opt': 'android-api-15-gradle/opt',
     }
     for test in tests:
-        test['treeherder-machine-platform'] = translation.get(
-            test['build-platform'], test['test-platform'])
+        # For most desktop platforms, the above table is not used for "regular"
+        # builds, so we'll always pick the test platform here.
+        # On macOS though, the regular builds are in the table.  This causes a
+        # conflict in `verify_task_graph_symbol` once you add a new test
+        # platform based on regular macOS builds, such as for Stylo.
+        # Since it's unclear if the regular macOS builds can be removed from
+        # the table, workaround the issue for Stylo.
+        if '-stylo' in test['test-platform']:
+            test['treeherder-machine-platform'] = test['test-platform']
+        else:
+            test['treeherder-machine-platform'] = translation.get(
+                test['build-platform'], test['test-platform'])
         yield test
 
 
 @transforms.add
 def set_tier(config, tests):
     """Set the tier based on policy for all test descriptions that do not
     specify a tier otherwise."""
     for test in tests:
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -223,17 +223,18 @@ config = {
         },
         "reftest": {
             "options": ["--suite=reftest"],
             "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
         "reftest-no-accel": {
             "options": ["--suite=reftest",
                         "--setpref=layers.acceleration.force-enabled=disabled"],
-            "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]},
+            "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
+        },
         "reftest-stylo": {
             "options": ["--suite=reftest",
                         "--setpref=reftest.compareStyloToGecko=true"],
             "tests": ["tests/reftest/tests/layout/reftests/reftest.list"],
         },
     },
     "all_xpcshell_suites": {
         "xpcshell": {
--- a/testing/mozharness/configs/unittests/mac_unittest.py
+++ b/testing/mozharness/configs/unittests/mac_unittest.py
@@ -165,31 +165,38 @@ config = {
         "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
         "mochitest-gl": ["--subsuite=webgl"],
         "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
         "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
         "jetpack-package": ["--flavor=jetpack-package"],
         "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
         "jetpack-addon": ["--flavor=jetpack-addon"],
         "a11y": ["--flavor=a11y"],
+        "plain-style": ["--failure-pattern-file=stylo-failures.md", "layout/style/test", "dom/smil/test", "dom/animation/test"],
+        "chrome-style": ["--flavor=chrome", "--failure-pattern-file=../stylo-failures.md", "layout/style/test/chrome", "dom/animation/test"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "crashtest": {
             'options': ["--suite=crashtest"],
             'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
         },
         "jsreftest": {
             'options':["--extra-profile-file=tests/jsreftest/tests/user.js"],
             'tests': ["tests/jsreftest/tests/jstests.list"]
         },
         "reftest": {
             'options': ["--suite=reftest"],
             'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
+        "reftest-stylo": {
+            "options": ["--suite=reftest",
+                        "--setpref=reftest.compareStyloToGecko=true"],
+            "tests": ["tests/reftest/tests/layout/reftests/reftest.list"],
+        },
     },
     "all_xpcshell_suites": {
         "xpcshell": {
             'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
                         "--manifest=tests/xpcshell/tests/xpcshell.ini"],
             'tests': []
         },
         "xpcshell-addons": {
--- a/testing/mozharness/configs/unittests/win_taskcluster_unittest.py
+++ b/testing/mozharness/configs/unittests/win_taskcluster_unittest.py
@@ -177,16 +177,18 @@ config = {
         "mochitest-gl": ["--subsuite=webgl"],
         "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
         "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
         "mochitest-metro-chrome": ["--flavor=browser", "--metro-immersive"],
         "jetpack-package": ["--flavor=jetpack-package"],
         "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
         "jetpack-addon": ["--flavor=jetpack-addon"],
         "a11y": ["--flavor=a11y"],
+        "plain-style": ["--failure-pattern-file=stylo-failures.md", "layout/style/test", "dom/smil/test", "dom/animation/test"],
+        "chrome-style": ["--flavor=chrome", "--failure-pattern-file=../stylo-failures.md", "layout/style/test/chrome", "dom/animation/test"],
     },
     # local reftest suites
     "all_reftest_suites": {
         "crashtest": {
             'options': ["--suite=crashtest"],
             'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
         },
         "jsreftest": {
@@ -202,16 +204,21 @@ config = {
                         "--setpref=layers.gpu-process.force-enabled=true"],
             'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
         "reftest-no-accel": {
             "options": ["--suite=reftest",
                         "--setpref=layers.acceleration.disabled=true"],
             "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
         },
+        "reftest-stylo": {
+            "options": ["--suite=reftest",
+                        "--setpref=reftest.compareStyloToGecko=true"],
+            "tests": ["tests/reftest/tests/layout/reftests/reftest.list"],
+        },
     },
     "all_xpcshell_suites": {
         "xpcshell": {
             'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
                         "--manifest=tests/xpcshell/tests/xpcshell.ini"],
             'tests': []
         },
         "xpcshell-addons": {