Bug 1474897: switch bitbar workers to g-w r=bc,tomprince
authorAndrew Erickson <aerickson@mozilla.com>
Mon, 15 Apr 2019 17:30:59 +0000
changeset 469534 b8f49a14c458
parent 469533 b02881ec59d7
child 469535 4b43bc564712
child 469604 afb20612c0e5
push id35873
push userccoroiu@mozilla.com
push dateMon, 15 Apr 2019 21:36:26 +0000
treeherdermozilla-central@b8f49a14c458 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbc, tomprince
bugs1474897
milestone68.0a1
first release with
nightly linux32
b8f49a14c458 / 68.0a1 / 20190415213626 / files
nightly linux64
b8f49a14c458 / 68.0a1 / 20190415213626 / files
nightly mac
b8f49a14c458 / 68.0a1 / 20190415213626 / files
nightly win32
b8f49a14c458 / 68.0a1 / 20190415213626 / files
nightly win64
b8f49a14c458 / 68.0a1 / 20190415213626 / files
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
releases
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1474897: switch bitbar workers to g-w r=bc,tomprince Much of this was already reviewed in D21473 (my test change where I developed the payload modifications and that pointed tests at my test queue). This change keeps the payload changes from D21473, but points at the new 'real' queues we'll be using. Differential Revision: https://phabricator.services.mozilla.com/D25009
taskcluster/ci/config.yml
taskcluster/taskgraph/transforms/job/mozharness_test.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/transforms/tests.py
taskcluster/taskgraph/util/workertypes.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -288,16 +288,21 @@ workers:
             implementation: docker-worker
             os: linux
             worker-type: 'gecko-{level}-{alias}'
         t-ap.*:
             provisioner: proj-autophone
             implementation: script-engine-autophone
             os: linux
             worker-type: 'gecko-{alias}'
+        t-bitbar-gw.*:
+            provisioner: proj-autophone
+            implementation: generic-worker
+            os: linux-bitbar
+            worker-type: 'gecko-{alias}'
         t-linux(-large|-xlarge):
             provisioner: aws-provisioner-v1
             implementation: docker-worker
             os: linux
             worker-type: 'gecko-{alias}'
         t-linux-talos:
             provisioner: releng-hardware
             implementation: generic-worker
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -183,19 +183,23 @@ def mozharness_test_on_docker(config, jo
 
 
 @run_job_using('generic-worker', 'mozharness-test', schema=mozharness_test_run_schema)
 def mozharness_test_on_generic_worker(config, job, taskdesc):
     test = taskdesc['run']['test']
     mozharness = test['mozharness']
     worker = taskdesc['worker']
 
+    bitbar_script = 'test-linux.sh'
+    bitbar_wrapper = '/builds/taskcluster/script.py'
+
     is_macosx = worker['os'] == 'macosx'
     is_windows = worker['os'] == 'windows'
-    is_linux = worker['os'] == 'linux'
+    is_linux = worker['os'] == 'linux' or worker['os'] == 'linux-bitbar'
+    is_bitbar = worker['os'] == 'linux-bitbar'
     assert is_macosx or is_windows or is_linux
 
     artifacts = [
         {
             'name': 'public/logs',
             'path': 'logs',
             'type': 'directory'
         },
@@ -204,16 +208,35 @@ def mozharness_test_on_generic_worker(co
     # jittest doesn't have blob_upload_dir
     if test['test-name'] != 'jittest':
         artifacts.append({
             'name': 'public/test_info',
             'path': 'build/blobber_upload_dir',
             'type': 'directory'
         })
 
+    if is_bitbar:
+        artifacts = [
+            {
+                'name': 'public/test/',
+                'path': 'artifacts/public',
+                'type': 'directory'
+            },
+            {
+                'name': 'public/logs/',
+                'path': 'workspace/logs',
+                'type': 'directory'
+            },
+            {
+                'name': 'public/test_info/',
+                'path': 'workspace/build/blobber_upload_dir',
+                'type': 'directory'
+            },
+        ]
+
     if 'installer-url' in mozharness:
         installer_url = mozharness['installer-url']
     else:
         upstream_task = '<build-signing>' if mozharness['requires-signed-builds'] else '<build>'
         installer_url = get_artifact_url(upstream_task, mozharness['build-artifact-name'])
 
     worker['os-groups'] = test['os-groups']
 
@@ -251,29 +274,49 @@ def mozharness_test_on_generic_worker(co
             'MOZ_NO_REMOTE': '1',
             'NO_FAIL_ON_TEST_ERRORS': '1',
             'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
             'SHELL': '/bin/bash',
             'XPCOM_DEBUG_BREAK': 'warn',
             'XPC_FLAGS': '0x0',
             'XPC_SERVICE_NAME': '0',
         })
+    elif is_bitbar:
+        env.update({
+            'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
+            'MOZHARNESS_SCRIPT': mozharness['script'],
+            'MOZHARNESS_URL': {'artifact-reference': '<build/public/build/mozharness.zip>'},
+            'MOZILLA_BUILD_URL': {'task-reference': installer_url},
+            "MOZ_NO_REMOTE": '1',
+            "NEED_XVFB": "false",
+            "XPCOM_DEBUG_BREAK": 'warn',
+            "NO_FAIL_ON_TEST_ERRORS": '1',
+            "MOZ_HIDE_RESULTS_TABLE": '1',
+            "MOZ_NODE_PATH": "/usr/local/bin/node",
+            'TASKCLUSTER_WORKER_TYPE': job['worker-type'],
+        })
 
     extra_config = {
         'installer_url': installer_url,
         'test_packages_url': test_packages_url(taskdesc),
     }
     env['EXTRA_MOZHARNESS_CONFIG'] = {'task-reference': json.dumps(extra_config)}
 
     if is_windows:
         mh_command = [
             'c:\\mozilla-build\\python\\python.exe',
             '-u',
             'mozharness\\scripts\\' + normpath(mozharness['script'])
         ]
+    if is_bitbar:
+        mh_command = [
+            bitbar_wrapper,
+            'bash',
+            "./{}".format(bitbar_script)
+        ]
     else:
         # is_linux or is_macosx
         mh_command = [
             'python2.7',
             '-u',
             'mozharness/scripts/' + mozharness['script']
         ]
 
@@ -312,16 +355,26 @@ def mozharness_test_on_generic_worker(co
         'content': {
             'artifact': get_artifact_path(taskdesc, 'mozharness.zip'),
             'task-id': {
                 'task-reference': '<build>'
             }
         },
         'format': 'zip'
     }]
+    if is_bitbar:
+        a_url = '{}/raw-file/{}/taskcluster/scripts/tester/{}'.format(
+            config.params['head_repository'], config.params['head_rev'], bitbar_script
+        )
+        worker['mounts'] = [{
+            'file': bitbar_script,
+            'content': {
+                'url': a_url,
+            },
+        }]
 
     if is_windows:
         worker['command'] = [' '.join(mh_command)]
     else:  # is_macosx
         worker['command'] = [mh_command]
 
 
 @run_job_using('script-engine-autophone', 'mozharness-test', schema=mozharness_test_run_schema)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -660,17 +660,17 @@ def build_docker_worker_payload(config, 
     # coalesce / superseding
     if 'coalesce' in task:
         payload['supersederUrl'] = superseder_url(config, task)
 
     check_caches_are_volumes(task)
 
 
 @payload_builder('generic-worker', schema={
-    Required('os'): Any('windows', 'macosx', 'linux'),
+    Required('os'): Any('windows', 'macosx', 'linux', 'linux-bitbar'),
     # see http://schemas.taskcluster.net/generic-worker/v1/payload.json
     # and https://docs.taskcluster.net/reference/workers/generic-worker/payload
 
     # command is a list of commands to run, sequentially
     # on Windows, each command is a string, on OS X and Linux, each command is
     # a string array
     Required('command'): Any(
         [taskref_or_string],   # Windows
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -1192,28 +1192,28 @@ def set_worker_type(config, tests):
                 # the other jobs run on a vm which may or may not be a win10 vm
                 win_worker_type_platform = WINDOWS_WORKER_TYPES[
                     test_platform.split('/')[0]
                 ]
             # now we have the right platform set the worker type accordingly
             test['worker-type'] = win_worker_type_platform[test['virtualization']]
         elif test_platform.startswith('android-hw-g5'):
             if test['suite'] != 'raptor':
-                test['worker-type'] = 'proj-autophone/gecko-t-ap-unit-g5'
+                test['worker-type'] = 't-bitbar-gw-unit-g5'
             elif '--power-test' in test['mozharness']['extra-options']:
-                test['worker-type'] = 'proj-autophone/gecko-t-ap-batt-g5'
+                test['worker-type'] = 't-bitbar-gw-batt-g5'
             else:
-                test['worker-type'] = 'proj-autophone/gecko-t-ap-perf-g5'
+                test['worker-type'] = 't-bitbar-gw-perf-g5'
         elif test_platform.startswith('android-hw-p2'):
             if test['suite'] != 'raptor':
-                test['worker-type'] = 'proj-autophone/gecko-t-ap-unit-p2'
+                test['worker-type'] = 't-bitbar-gw-unit-p2'
             elif '--power-test' in test['mozharness']['extra-options']:
-                test['worker-type'] = 'proj-autophone/gecko-t-ap-batt-p2'
+                test['worker-type'] = 't-bitbar-gw-batt-p2'
             else:
-                test['worker-type'] = 'proj-autophone/gecko-t-ap-perf-p2'
+                test['worker-type'] = 't-bitbar-gw-perf-p2'
         elif test_platform.startswith('android-em-7.0-x86'):
             test['worker-type'] = 'terraform-packet/gecko-t-linux'
         elif test_platform.startswith('linux') or test_platform.startswith('android'):
             if test.get('suite', '') in ['talos', 'raptor'] and \
                  not test['build-platform'].startswith('linux64-ccov'):
                 test['worker-type'] = 'releng-hardware/gecko-t-linux-talos'
             else:
                 test['worker-type'] = LINUX_WORKER_TYPES[test['instance-size']]
--- a/taskcluster/taskgraph/util/workertypes.py
+++ b/taskcluster/taskgraph/util/workertypes.py
@@ -47,22 +47,16 @@ WORKER_TYPES = {
     'scriptworker-prov-v1/balrogworker-v1': ('balrog', None),
     'scriptworker-prov-v1/beetmoverworker-v1': ('beetmover', None),
     'scriptworker-prov-v1/pushapk-v1': ('push-apk', None),
     "scriptworker-prov-v1/signing-linux-v1": ('scriptworker-signing', None),
     "scriptworker-prov-v1/shipit": ('shipit', None),
     "scriptworker-prov-v1/shipit-dev": ('shipit', None),
     "scriptworker-prov-v1/treescript-v1": ('treescript', None),
     'releng-hardware/gecko-t-osx-1010': ('generic-worker', 'macosx'),
-    'proj-autophone/gecko-t-ap-perf-g5': ('script-engine-autophone', 'linux'),
-    'proj-autophone/gecko-t-ap-unit-g5': ('script-engine-autophone', 'linux'),
-    'proj-autophone/gecko-t-ap-batt-g5': ('script-engine-autophone', 'linux'),
-    'proj-autophone/gecko-t-ap-perf-p2': ('script-engine-autophone', 'linux'),
-    'proj-autophone/gecko-t-ap-unit-p2': ('script-engine-autophone', 'linux'),
-    'proj-autophone/gecko-t-ap-batt-p2': ('script-engine-autophone', 'linux'),
     'terraform-packet/gecko-t-linux': ('docker-worker', 'linux'),
 }
 
 
 @memoize
 def worker_type_implementation(graph_config, worker_type):
     """Get the worker implementation and OS for the given workerType, where the
     OS represents the host system, not the target OS, in the case of