Bug 1330310: Switch macosx tests to buildbot-bridge. r=dustin,catlee draft
authorWander Lairson Costa <wcosta@mozilla.com>
Mon, 16 Jan 2017 19:00:52 -0200
changeset 462221 2928bd520bd24b834ff7e452dd640841038ea33a
parent 461393 d4d3a7b6d57e06891f158ccb4bebf507a8d55d32
child 542325 5606de2056498ca975d84496af011773cfa4ab36
push id41680
push userwcosta@mozilla.com
push dateMon, 16 Jan 2017 21:01:22 +0000
reviewersdustin, catlee
bugs1330310
milestone53.0a1
Bug 1330310: Switch macosx tests to buildbot-bridge. r=dustin,catlee We use buildbot-bridge to schedule macosx tests in buildbot, and disable scheduling on buildbot. Also, schedule a subset of unittests in taskcluster-worker Tier 3 machines. MozReview-Commit-ID: 6LyyODBaURI
taskcluster/ci/test/test-platforms.yml
taskcluster/ci/test/test-sets.yml
taskcluster/ci/test/tests.yml
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/configs/builds/releng_base_mac_64_builds.py
testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py
--- a/taskcluster/ci/test/test-platforms.yml
+++ b/taskcluster/ci/test/test-platforms.yml
@@ -119,21 +119,21 @@ windows10-64-vm/opt:
 
 ##
 # MacOS X platforms (matching /macosx.*/)
 
 macosx64/debug:
     build-platform: macosx64/debug
     test-sets:
         - macosx64-tests-debug
-# Disable bbb talos tests for mac until we disable bb scheduling
-# macosx64/opt:
-#     build-platform: macosx64/opt
-#     test-sets:
-#         - macosx64-tests-talos
+
+macosx64/opt:
+    build-platform: macosx64/opt
+    test-sets:
+        - macosx64-tests-talos
 
 ##
 # Android platforms (matching /android.*/)
 
 android-4.3-arm7-api-15/debug:
     build-platform: android-api-15/debug
     test-sets:
         - android-common-tests
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -119,32 +119,35 @@ ccov-code-coverage-tests:
 
 jsdcov-code-coverage-tests:
     - mochitest-browser-chrome
     - mochitest-devtools-chrome
 
 macosx64-tests-debug:
     - cppunit
     - crashtest
-    # - gtest
+    - external-media-tests
+    - gtest
+    - jittest
     - jsreftest
-    # - marionette
-    # - mochitest
-    # - mochitest-browser-chrome
-    # - mochitest-clipboard
-    # - mochitest-devtools-chrome
-    # - mochitest-gpu
-    # - mochitest-jetpack
-    # - mochitest-media
-    # - mochitest-other
+    - marionette
+    - media-youtube-tests
+    - mochitest
+    - mochitest-browser-chrome
+    - mochitest-clipboard
+    - mochitest-devtools-chrome
+    - mochitest-gpu
+    - mochitest-jetpack
+    - mochitest-media
+    - mochitest-other
     - mochitest-webgl
-    # - reftest
-    # - web-platform-tests
-    # - web-platform-tests-reftests
-    # - xpcshell
+    - reftest
+    - web-platform-tests
+    - web-platform-tests-reftests
+    - xpcshell
 
 macosx64-tests-talos:
     - talos-chrome
     - talos-dromaeojs
     - talos-g1
     - talos-g2
     - talos-g4
     - talos-other
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -63,17 +63,17 @@ crashtest:
                 config:
                     - android/androidarm_4_3.py
                     - remove_executables.py
                     - android/androidarm_4_3-tc.py
                 extra-options:
                     - --test-suite=crashtest
             default:
                 script: desktop_unittest.py
-                chunked: true
+                chunked: false
                 no-read-buildbot-config: true
                 config:
                     by-test-platform:
                         windows.*:
                             - unittests/win_taskcluster_unittest.py
                         macosx.*:
                             - remove_executables.py
                             - unittests/mac_unittest.py
@@ -95,16 +95,35 @@ external-media-tests:
         config:
             by-test-platform:
                 windows.*:
                     - mediatests/taskcluster_windows_config.py
                 default:
                     - mediatests/taskcluster_posix_config.py
                     - remove_executables.py
 
+media-youtube-tests:
+    description: "Media Tests MSE Youtube Playback"
+    suite: media-youtube-tests
+    treeherder-symbol: tc-VP(b-y)
+    e10s: false
+    tier: 2
+    max-run-time: 5400
+    mozharness:
+        script: firefox_media_tests_taskcluster.py
+        config:
+            by-test-platform:
+                windows.*:
+                    - mediatests/taskcluster_windows_config.py
+                default:
+                    - mediatests/taskcluster_posix_config.py
+        extra-options:
+          - "--suite"
+          - "media-youtube-tests"
+
 firefox-ui-functional-local:
     description: "Firefox-ui-tests functional run"
     suite: "firefox-ui/functional local"
     treeherder-symbol: tc-Fxfn-l(en-US)
     max-run-time: 5400
     tier: 1
     docker-image: {"in-tree": "desktop1604-test"}
     mozharness:
@@ -159,16 +178,20 @@ jittest:
     suite: jittest/jittest-chunked
     treeherder-symbol: tc(Jit)
     e10s: false
     chunks:
         by-test-platform:
             windows.*: 1
             default: 6
     mozharness:
+        chunked:
+          by-test-platform:
+            windows.*: false
+            default: true
         script: desktop_unittest.py
         no-read-buildbot-config: true
         config:
             by-test-platform:
                 windows.*:
                     - unittests/win_taskcluster_unittest.py
                 macosx.*:
                     - remove_executables.py
@@ -187,16 +210,17 @@ jsreftest:
         by-test-platform:
             android.*: xlarge
             default: default
     chunks:
         by-test-platform:
             android-4.3-arm7-api-15/debug: 20
             android.*: 6
             windows.*: 1
+            macosx.*: 1
             default: 2
     e10s:
         by-test-platform:
             # Bug 1321782
             windows.*: false
             android.*: false
             default: both
     max-run-time:
@@ -389,16 +413,17 @@ mochitest-browser-chrome:
             linux64-jsdcov/opt: 7200
             linux64-ccov/opt: 7200
             linux64/debug: 5400
             linux32/debug: 5400
             default: 3600
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
+        chunked: true
         config:
             by-test-platform:
                 windows.*:
                     - unittests/win_taskcluster_unittest.py
                 macosx.*:
                     - remove_executables.py
                     - unittests/mac_unittest.py
                 linux.*:
@@ -488,17 +513,17 @@ mochitest-clipboard:
                     - remove_executables.py
                     - android/androidarm_4_3-tc.py
                 extra-options:
                     # note that Android runs fewer suites than other platforms
                     - --test-suite=mochitest-plain-clipboard
             default:
                 script: desktop_unittest.py
                 no-read-buildbot-config: true
-                chunked: true
+                chunked: false
                 config:
                     by-test-platform:
                         windows.*:
                             - unittests/win_taskcluster_unittest.py
                         macosx.*:
                             - remove_executables.py
                             - unittests/mac_unittest.py
                         linux.*:
@@ -514,32 +539,34 @@ mochitest-devtools-chrome:
             linux64-jsdcov/opt: mochitest/mochitest-devtools-chrome-coverage
             default: mochitest/mochitest-devtools-chrome-chunked
     treeherder-symbol: tc-M(dt)
     loopback-video: true
     max-run-time: 5400
     chunks:
         by-test-platform:
             windows.*: 8
+            macosx.*: 8
             default: 10
     run-on-projects:
         by-test-platform:
             linux64-ccov/opt: []
             linux64-jsdcov/opt: []
             default: ['all']
     e10s:
         by-test-platform:
             linux64-ccov/opt: false
             linux64-jsdcov/opt: false
             # Bug 1304433: mochitest-devtools-chrome (e10s) not greened on windows
             windows.*: false
             default: both
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
+        chunked: true
         config:
             by-test-platform:
                 windows.*:
                     - unittests/win_taskcluster_unittest.py
                 macosx.*:
                     - remove_executables.py
                     - unittests/mac_unittest.py
                 linux.*:
@@ -566,33 +593,34 @@ mochitest-gpu:
     description: "Mochitest GPU run"
     suite: mochitest/gpu
     treeherder-symbol: tc-M(gpu)
     loopback-video: true
     e10s:
         by-test-platform:
             windows.*: both
             android.*: false
+            macosx.*: both
             default: true
     mozharness:
         by-test-platform:
             android.*:
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
                     - android/androidarm_4_3.py
                     - remove_executables.py
                     - android/androidarm_4_3-tc.py
                 extra-options:
                     # note that Android runs fewer suites than other platforms
                     - --test-suite=mochitest-plain-gpu
             default:
                 script: desktop_unittest.py
                 no-read-buildbot-config: true
-                chunked: true
+                chunked: false
                 config:
                     by-test-platform:
                         windows.*:
                             - unittests/win_taskcluster_unittest.py
                         macosx.*:
                             - remove_executables.py
                             - unittests/mac_unittest.py
                         linux.*:
@@ -606,17 +634,17 @@ mochitest-jetpack:
     suite: mochitest/jetpack-package
     treeherder-symbol: tc-M(JP)
     loopback-video: true
     e10s: false
     max-run-time: 5400
     mozharness:
         script: desktop_unittest.py
         no-read-buildbot-config: true
-        chunked: true
+        chunked: false
         config:
             by-test-platform:
                 windows.*:
                     - unittests/win_taskcluster_unittest.py
                 macosx.*:
                     - remove_executables.py
                     - unittests/mac_unittest.py
                 linux.*:
@@ -650,17 +678,17 @@ mochitest-media:
                     - android/androidarm_4_3.py
                     - remove_executables.py
                     - android/androidarm_4_3-tc.py
                 extra-options:
                     - --test-suite=mochitest-media
             default:
                 script: desktop_unittest.py
                 no-read-buildbot-config: true
-                chunked: true
+                chunked: false
                 config:
                     by-test-platform:
                         windows.*:
                             - unittests/win_taskcluster_unittest.py
                         macosx.*:
                             - remove_executables.py
                             - unittests/mac_unittest.py
                         linux.*:
@@ -674,17 +702,17 @@ mochitest-other:
     suite: mochitest/other
     treeherder-symbol: tc-M(oth)
     e10s: false
     max-run-time: 5400
     mozharness:
         script: mozharness/scripts/desktop_unittest.py
         no-read-buildbot-config: true
         include-blob-upload-branch: true
-        chunked: true
+        chunked: false
         config:
           by-test-platform:
             macosx.*:
               - remove_executables.py
               - unittests/mac_unittest.py
             linux.*:
               - unittests/linux_unittest.py
               - remove_executables.py
@@ -789,17 +817,18 @@ reftest:
     instance-size:
         by-test-platform:
             android.*: xlarge
             default: default
     chunks:
         by-test-platform:
             android-4.3-arm7-api-15/debug: 48
             android.*: 16
-            macosx.*: 1
+            macosx64/opt: 1
+            macosx64/debug: 2
             default: 8
     max-run-time:
         by-test-platform:
             android.*: 10800
             default: 3600
     mozharness:
         by-test-platform:
             android.*:
@@ -809,16 +838,20 @@ reftest:
                     - android/androidarm_4_3.py
                     - remove_executables.py
                     - android/androidarm_4_3-tc.py
                 extra-options:
                     - --test-suite=reftest
             default:
                 script: desktop_unittest.py
                 no-read-buildbot-config: true
+                chunked:
+                  by-test-platform:
+                    macosx64/opt: false
+                    default: true
                 config:
                     by-test-platform:
                         windows.*:
                             - unittests/win_taskcluster_unittest.py
                         macosx.*:
                             - remove_executables.py
                             - unittests/mac_unittest.py
                         linux.*:
@@ -1048,28 +1081,30 @@ talos-tp5o:
                     - talos/linux_config.py
                     - remove_executables.py
         extra-options:
             - --suite=tp5o
 
 web-platform-tests:
     description: "Web platform test run"
     suite: web-platform-tests
-    treeherder-symbol: tc-W()
+    treeherder-symbol: tc-W(wpt)
     chunks:
       by-test-platform:
-        macosx.*: 5
+        macosx64/opt: 5
+        macosx64/debug: 10
         default: 12
     max-run-time: 7200
     instance-size: xlarge
     docker-image: {"in-tree": "desktop1604-test"}
     checkout: true
     mozharness:
         script: web_platform_tests.py
         no-read-buildbot-config: true
+        chunked: true
         config:
             by-test-platform:
                 windows.*:
                     - web_platform_tests/prod_config_windows_taskcluster.py
                 default:
                     - web_platform_tests/prod_config.py
                     - remove_executables.py
         extra-options:
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -65,16 +65,23 @@ BUILDER_NAME_PREFIX = {
     'linux64-jsdcov': 'Ubuntu Code Coverage VM 12.04 x64',
     'linux64-stylo': 'Ubuntu VM 12.04 x64',
     'macosx64': 'Rev7 MacOSX Yosemite 10.10.5',
     'android-4.3-arm7-api-15': 'Android 4.3 armv7 API 15+',
     'android-4.2-x86': 'Android 4.2 x86 Emulator',
     'android-4.3-arm7-api-15-gradle': 'Android 4.3 armv7 API 15+',
 }
 
+TASKCLUSTER_WORKER_TESTS = [
+    'cppunit',
+    'crashtest',
+    'jsreftest',
+    'mochitest-webgl',
+]
+
 logger = logging.getLogger(__name__)
 
 transforms = TransformSequence()
 
 # Schema for a test description
 #
 # *****WARNING*****
 #
@@ -237,17 +244,19 @@ test_description_schema = Schema({
             # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the
             # environment.  This is more than just a helpful path setting -- it
             # causes xpcshell tests to start additional servers, and runs
             # additional tests.
             Required('set-moz-node-path', default=False): bool,
 
             # If true, include chunking information in the command even if the number
             # of chunks is 1
-            Required('chunked', default=False): bool,
+            Required('chunked', default=False): optionally_keyed_by(
+                'test-platform',
+                bool),
 
             # The chunking argument format to use
             Required('chunking-args', default='this-chunk'): Any(
                 # Use the usual --this-chunk/--total-chunk arguments
                 'this-chunk',
                 # Use --test-suite=<suite>-<chunk-suffix>; see chunk-suffix, below
                 'test-suite-suffix',
             ),
@@ -389,17 +398,24 @@ def set_asan_docker_image(config, tests)
 def set_worker_implementation(config, tests):
     """Set the worker implementation based on the test platform."""
     for test in tests:
         if test.get('suite', '') == 'talos':
             test['worker-implementation'] = 'buildbot-bridge'
         elif test['test-platform'].startswith('win'):
             test['worker-implementation'] = 'generic-worker'
         elif test['test-platform'].startswith('macosx'):
-            test['worker-implementation'] = 'macosx-engine'
+            test['worker-implementation'] = 'buildbot-bridge'
+            # Some tests must also run in taskcluster-worker
+            # small hack until we completely switch to taskcluster-worker
+            if test['test-name'] in TASKCLUSTER_WORKER_TESTS:
+                test_tc_worker = copy.deepcopy(test)
+                test_tc_worker['worker-implementation'] = 'macosx-engine'
+                test_tc_worker['test-name'] += '-worker'
+                yield test_tc_worker
         else:
             test['worker-implementation'] = 'docker-worker'
         yield test
 
 
 @transforms.add
 def set_tier(config, tests):
     """Set the tier based on policy for all test descriptions that do not
@@ -410,17 +426,18 @@ def set_tier(config, tests):
 
         # only override if not set for the test
         if 'tier' not in test or test['tier'] == 'default':
             if test['test-platform'] in ['linux64/debug',
                                          'linux64-asan/opt',
                                          'android-4.3-arm7-api-15/debug',
                                          'android-x86/opt']:
                 test['tier'] = 1
-            elif test['test-platform'].startswith('windows'):
+            elif test['test-platform'].startswith('windows') \
+                    or test['worker-implementation'] == 'macosx-engine':
                 test['tier'] = 3
             else:
                 test['tier'] = 2
         yield test
 
 
 @transforms.add
 def set_expires_after(config, tests):
@@ -480,17 +497,17 @@ def split_e10s(config, tests):
         test.setdefault('attributes', {})
         test['e10s'] = False
         test['attributes']['e10s'] = False
 
         if e10s == 'both':
             yield copy.deepcopy(test)
             e10s = True
         if e10s:
-            test['test-name'] += '-e10s'
+            test['test-name'] += '-10s'
             test['e10s'] = True
             test['attributes']['e10s'] = True
             group, symbol = split_symbol(test['treeherder-symbol'])
             if group != '?':
                 group += '-e10s'
             test['treeherder-symbol'] = join_symbol(group, symbol)
             test['mozharness']['extra-options'].append('--e10s')
         yield test
@@ -938,20 +955,41 @@ def macosx_engine_setup(config, test, ta
         download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
         command.append('--download-symbols=' + download_symbols)
 
 
 @worker_setup_function("buildbot-bridge")
 def buildbot_bridge_setup(config, test, taskdesc):
     branch = config.params['project']
     platform, build_type = test['build-platform'].split('/')
+
+    # mochitest e10s follows the pattern mochitest-e10s-<suffix>
+    # in buildbot, except for these special cases
+    buildbot_specials = [
+        'mochitest-webgl',
+        'mochitest-clipboard',
+        'mochitest-media',
+        'mochitest',
+    ]
     test_name = test.get('talos-try-name', test['test-name'])
+    if test_name.startswith('mochitest') \
+            and test_name.endswith('e10s') \
+            and test_name not in buildbot_specials:
+        split_mochitest = test_name.split('-')
+        test_name = '-'.join(
+            split_mochitest[0],
+            split_mochitest[-1],
+            '-'.join(split_mochitest[1:-1])
+        )
+    # in buildbot, mochitest-webgl is called mochitest-gl
+    test_name = test_name.replace('webgl', 'gl')
+
     mozharness = test['mozharness']
 
-    if test['e10s'] and not test_name.endswith('-e10s'):
+    if test['e10s'] and 'e10s' not in test_name:
         test_name += '-e10s'
 
     if mozharness.get('chunked', False):
         this_chunk = test.get('this-chunk')
         test_name = '{}-{}'.format(test_name, this_chunk)
 
     worker = taskdesc['worker'] = {}
     worker['implementation'] = test['worker-implementation']
--- a/testing/mozharness/configs/builds/releng_base_mac_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_mac_64_builds.py
@@ -26,18 +26,18 @@ config = {
     # decides whether we want to use moz_sign_cmd in env
     'enable_signing': True,
     'enable_ccache': True,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox',
     'tooltool_script': ["/builds/tooltool.py"],
     'tooltool_bootstrap': "setup.sh",
     'enable_count_ctors': False,
-    'enable_talos_sendchange': True,
-    'enable_unittest_sendchange': True,
+    'enable_talos_sendchange': False,
+    'enable_unittest_sendchange': False,
     #########################################################################
 
 
     #########################################################################
     ###### 64 bit specific ######
     'base_name': 'OS X 10.7 %(branch)s',
     'platform': 'macosx64',
     'stage_platform': 'macosx64',
--- a/testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py
@@ -24,18 +24,18 @@ config = {
     'enable_signing': True,
     'enable_ccache': True,
     'enable_check_test': False,
     'vcs_share_base': '/builds/hg-shared',
     'objdir': 'obj-firefox/',
     'tooltool_script': ["/builds/tooltool.py"],
     'tooltool_bootstrap': "setup.sh",
     'enable_count_ctors': False,
-    'enable_talos_sendchange': True,
-    'enable_unittest_sendchange': True,
+    'enable_talos_sendchange': False,
+    'enable_unittest_sendchange': False,
     #########################################################################
 
 
     #########################################################################
     ###### 64 bit specific ######
     'base_name': 'OS X 10.7 %(branch)s',
     'platform': 'macosx64',
     'stage_platform': 'macosx64',