Bug 1431161 - run windows 32 and 64 bit builds on windows10-64 hardware for talos performance tests. r=rwood, a=test-only
authorJoel Maher <jmaher@mozilla.com>
Fri, 30 Mar 2018 06:06:41 -0400
changeset 460561 156e4b64363d96c9eb60e5f124443ef3e36e1085
parent 460560 e8ac8996924c77580fb95348e64e2e9326e19bb5
child 460562 cfe7adda153dbf64d6a17fe0145f7ebbfedc7508
push id8984
push userryanvm@gmail.com
push dateThu, 05 Apr 2018 00:13:08 +0000
treeherdermozilla-beta@fb0f35c060ef [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood, test-only
bugs1431161
milestone60.0
Bug 1431161 - run windows 32 and 64 bit builds on windows10-64 hardware for talos performance tests. r=rwood, a=test-only
taskcluster/taskgraph/transforms/coalesce.py
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/configs/talos/windows_config.py
--- a/taskcluster/taskgraph/transforms/coalesce.py
+++ b/taskcluster/taskgraph/transforms/coalesce.py
@@ -30,15 +30,16 @@ def enable_coalescing(config, jobs):
         if int(config.params['level']) > 1 and job['worker-type'] not in [
             # These worker types don't currently support coalescing.
             # This list can be removed when bug 1399401 is fixed:
             #   https://bugzilla.mozilla.org/show_bug.cgi?id=1399401
             'aws-provisioner-v1/gecko-t-win7-32',
             'aws-provisioner-v1/gecko-t-win7-32-gpu',
             'aws-provisioner-v1/gecko-t-win10-64',
             'aws-provisioner-v1/gecko-t-win10-64-gpu',
+            'releng-hardware/gecko-t-win10-64-hw',
         ]:
             job['coalesce'] = {
                 'job-identifier': sha256(job["label"]).hexdigest()[:20],
                 'age': 3600,
                 'size': 5,
             }
         yield job
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -47,32 +47,32 @@ LINUX_WORKER_TYPES = {
     'default': 'aws-provisioner-v1/gecko-t-linux-large',
 }
 
 # windows worker types keyed by test-platform and virtualization
 WINDOWS_WORKER_TYPES = {
     'windows7-32': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
-      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows7-32-pgo': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
-      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows7-32-nightly': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
-      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows7-32-devedition': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
-      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
     },
     'windows7-32-stylo-disabled': {
       'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
       'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
       'hardware': 'releng-hardware/gecko-t-win7-32-hw',
     },
     'windows10-64': {
       'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
@@ -921,59 +921,45 @@ def single_stylo_traversal_tests(config,
 
 @transforms.add
 def set_worker_type(config, tests):
     """Set the worker type based on the test platform."""
     for test in tests:
         # during the taskcluster migration, this is a bit tortured, but it
         # will get simpler eventually!
         test_platform = test['test-platform']
-        try_options = config.params['try_options'] if config.params['try_options'] else {}
         if test.get('worker-type'):
             # This test already has its worker type defined, so just use that (yields below)
             pass
         elif test_platform.startswith('macosx'):
             test['worker-type'] = MACOSX_WORKER_TYPES['macosx64']
         elif test_platform.startswith('win'):
-            win_worker_type_platform = WINDOWS_WORKER_TYPES[
-                test_platform.split('/')[0]
-            ]
-            if test.get('suite', '') == 'talos' and 'ccov' not in test['build-platform']:
-                if try_options.get('taskcluster_worker'):
-                    test['worker-type'] = win_worker_type_platform['hardware']
-                elif test['virtualization'] == 'virtual':
-                    test['worker-type'] = win_worker_type_platform[test['virtualization']]
-                else:
-                    test['worker-type'] = 'buildbot-bridge/buildbot-bridge'
+            # figure out what platform the job needs to run on
+            if test['virtualization'] == 'hardware':
+                # some jobs like talos and reftest run on real h/w - those are all win10
+                win_worker_type_platform = WINDOWS_WORKER_TYPES['windows10-64']
             else:
-                test['worker-type'] = win_worker_type_platform[test['virtualization']]
+                # the other jobs run on a vm which may or may not be a win10 vm
+                win_worker_type_platform = WINDOWS_WORKER_TYPES[
+                    test_platform.split('/')[0]
+                ]
+            # now we have the right platform set the worker type accordingly
+            test['worker-type'] = win_worker_type_platform[test['virtualization']]
         elif test_platform.startswith('linux') or test_platform.startswith('android'):
             if test.get('suite', '') == 'talos' and test['build-platform'] != 'linux64-ccov/opt':
                 test['worker-type'] = 'releng-hardware/gecko-t-linux-talos'
             else:
                 test['worker-type'] = LINUX_WORKER_TYPES[test['instance-size']]
         else:
             raise Exception("unknown test_platform {}".format(test_platform))
 
         yield test
 
 
 @transforms.add
-def skip_win10_hardware(config, tests):
-    """Windows 10 hardware isn't ready yet, don't even bother scheduling
-    unless we're on try"""
-    for test in tests:
-        if 'releng-hardware/gecko-t-win10-64-hw' not in test['worker-type']:
-            yield test
-        if config.params == 'try':
-            yield test
-        # Silently drop the test on the floor if its win10 hardware and we're not try
-
-
-@transforms.add
 def make_job_description(config, tests):
     """Convert *test* descriptions to *job* descriptions (input to
     taskgraph.transforms.job)"""
 
     for test in tests:
         label = '{}-{}-{}'.format(config.kind, test['test-platform'], test['test-name'])
         if test['chunks'] > 1:
             label += '-{}'.format(test['this-chunk'])
--- a/testing/mozharness/configs/talos/windows_config.py
+++ b/testing/mozharness/configs/talos/windows_config.py
@@ -1,12 +1,13 @@
 import os
 import socket
+import sys
 
-PYTHON = 'c:/mozilla-build/python27/python.exe'
+PYTHON = sys.executable
 PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
 VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
 
 config = {
     "log_name": "talos",
     "buildbot_json_path": "buildprops.json",
     "installer_path": "installer.exe",
     "virtualenv_path": VENV_PATH,
@@ -17,17 +18,18 @@ config = {
     ],
     "virtualenv_modules": ['pywin32', 'talos', 'mozinstall'],
     "exes": {
         'python': PYTHON,
         'easy_install': ['%s/scripts/python' % VENV_PATH,
                          '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
         'mozinstall': ['%s/scripts/python' % VENV_PATH,
                        '%s/scripts/mozinstall-script.py' % VENV_PATH],
-        'hg': 'c:/mozilla-build/hg/hg',
+        'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg'),
+        'tooltool.py': [PYTHON, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
     },
     "title": socket.gethostname().split('.')[0],
     "default_actions": [
         "clobber",
         "read-buildbot-config",
         "download-and-extract",
         "populate-webroot",
         "create-virtualenv",