Bug 1436037 - [taskgraph] Support Windows generic-worker with run-task, r=gps
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 05 Dec 2018 19:19:56 +0000
changeset 508697 df5747b8931be1f0ac57aa34883f37db8985dd6d
parent 508696 8447262c404e08f623466e07ef0f56a9adb07639
child 508698 c88d2cb951ca8b2622ed90f921194902bd4bb712
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1436037
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1436037 - [taskgraph] Support Windows generic-worker with run-task, r=gps This enables Windows generic-worker based tasks to use the run-task script. Differential Revision: https://phabricator.services.mozilla.com/D10758
taskcluster/scripts/run-task
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/mach.py
taskcluster/taskgraph/transforms/job/python_test.py
taskcluster/taskgraph/transforms/job/run_task.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/scripts/run-task
+++ b/taskcluster/scripts/run-task
@@ -443,23 +443,24 @@ def vcs_checkout(source_repo, dest, stor
             '--config', 'hostsecurity.hg.mozilla.org:fingerprints=%s' % hgmo_fingerprint,
         ])
 
     if base_repo:
         args.extend(['--upstream', base_repo])
     if sparse_profile:
         args.extend(['--sparseprofile', sparse_profile])
 
+    dest = os.path.abspath(dest)
     args.extend([
         revision_flag, revision_value,
         source_repo, dest,
     ])
 
     res = run_and_prefix_output(b'vcs', args,
-                                extra_env={b'PYTHONUNBUFFERED': b'1'})
+                                extra_env={'PYTHONUNBUFFERED': '1'})
     if res:
         sys.exit(res)
 
     # Update the current revision hash and ensure that it is well formed.
     revision = subprocess.check_output(
         [hg_bin, 'log',
          '--rev', '.',
          '--template', '{node}'],
@@ -736,16 +737,19 @@ def main(args):
             branch=os.environ.get('COMM_HEAD_REF'))
 
     elif not os.environ.get('COMM_HEAD_REV') and \
             os.environ.get('COMM_HEAD_REF'):
         print('task should be defined in terms of non-symbolic revision')
         return 1
 
     try:
+        if 'GECKO_PATH' in os.environ:
+            os.environ['GECKO_PATH'] = os.path.abspath(os.environ['GECKO_PATH'])
+
         if 'MOZ_FETCHES' in os.environ:
             fetch_artifacts()
 
         return run_and_prefix_output(b'task', task_args)
     finally:
         fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
         if fetches_dir and os.path.isdir(fetches_dir):
             print_line(b'fetches', b'removing %s\n' % fetches_dir.encode('utf-8'))
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -59,45 +59,55 @@ def generic_worker_add_artifacts(config,
 
 
 def support_vcs_checkout(config, job, taskdesc, sparse=False):
     """Update a job/task with parameters to enable a VCS checkout.
 
     This can only be used with ``run-task`` tasks, as the cache name is
     reserved for ``run-task`` tasks.
     """
-    level = config.params['level']
+    is_win = job['worker']['os'] == 'windows'
 
-    # native-engine does not support caches (yet), so we just do a full clone
-    # every time :(
+    if is_win:
+        checkoutdir = './build'
+        geckodir = '{}/src'.format(checkoutdir)
+        hgstore = 'y:/hg-shared'
+    else:
+        checkoutdir = '{workdir}/checkouts'.format(**job['run'])
+        geckodir = '{}/gecko'.format(checkoutdir)
+        hgstore = '{}/hg-store'.format(checkoutdir)
+
+    level = config.params['level']
+    # native-engine and generic-worker do not support caches (yet), so we just
+    # do a full clone every time :(
     if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
         name = 'level-%s-checkouts' % level
 
         # comm-central checkouts need their own cache, because clobber won't
         # remove the comm-central checkout
         if job['run'].get('comm-checkout', False):
             name += '-comm'
 
         # Sparse checkouts need their own cache because they can interfere
         # with clients that aren't sparse aware.
         if sparse:
             name += '-sparse'
 
         taskdesc['worker'].setdefault('caches', []).append({
             'type': 'persistent',
             'name': name,
-            'mount-point': '{workdir}/checkouts'.format(**job['run']),
+            'mount-point': checkoutdir,
         })
 
     taskdesc['worker'].setdefault('env', {}).update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
-        'GECKO_PATH': '{workdir}/checkouts/gecko'.format(**job['run']),
-        'HG_STORE_PATH': '{workdir}/checkouts/hg-store'.format(**job['run']),
+        'GECKO_PATH': geckodir,
+        'HG_STORE_PATH': hgstore,
     })
 
     if 'comm_base_repository' in config.params:
         taskdesc['worker']['env'].update({
             'COMM_BASE_REPOSITORY': config.params['comm_base_repository'],
             'COMM_HEAD_REPOSITORY': config.params['comm_head_repository'],
             'COMM_HEAD_REV': config.params['comm_head_rev'],
         })
--- a/taskcluster/taskgraph/transforms/job/mach.py
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -25,19 +25,24 @@ mach_schema = Schema({
     # gecko checkout
     Required('comm-checkout'): bool,
 
     # Base work directory used to set up the task.
     Required('workdir'): basestring,
 })
 
 
-@run_job_using("docker-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False})
-@run_job_using("native-engine", "mach", schema=mach_schema, defaults={'comm-checkout': False})
-@run_job_using("generic-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False})
-def docker_worker_mach(config, job, taskdesc):
+defaults = {
+    'comm-checkout': False,
+}
+
+
+@run_job_using("docker-worker", "mach", schema=mach_schema, defaults=defaults)
+@run_job_using("native-engine", "mach", schema=mach_schema, defaults=defaults)
+@run_job_using("generic-worker", "mach", schema=mach_schema, defaults=defaults)
+def configure_mach(config, job, taskdesc):
     run = job['run']
 
     # defer to the run_task implementation
-    run['command'] = 'cd {workdir}/checkouts/gecko && ./mach {mach}'.format(**run)
+    run['command'] = 'cd $GECKO_PATH && ./mach {mach}'.format(**run)
     run['using'] = 'run-task'
     del run['mach']
     configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])
--- a/taskcluster/taskgraph/transforms/job/python_test.py
+++ b/taskcluster/taskgraph/transforms/job/python_test.py
@@ -20,22 +20,25 @@ python_test_schema = Schema({
     # The subsuite to run
     Required('subsuite'): basestring,
 
     # Base work directory used to set up the task.
     Required('workdir'): basestring,
 })
 
 
-@run_job_using(
-    'docker-worker',
-    'python-test',
-    schema=python_test_schema,
-    defaults={'python-version': 2, 'subsuite': 'default'})
-def docker_worker_python_test(config, job, taskdesc):
+defaults = {
+    'python-version': 2,
+    'subsuite': 'default',
+}
+
+
+@run_job_using('docker-worker', 'python-test', schema=python_test_schema, defaults=defaults)
+@run_job_using('generic-worker', 'python-test', schema=python_test_schema, defaults=defaults)
+def configure_python_test(config, job, taskdesc):
     run = job['run']
 
     # defer to the mach implementation
     run['mach'] = 'python-test --python {python-version} --subsuite {subsuite}'.format(**run)
     run['using'] = 'mach'
     del run['python-version']
     del run['subsuite']
     configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -36,22 +36,22 @@ run_task_schema = Schema({
     # it will be included in a single argument to `bash -cx`.
     Required('command'): Any([taskref_or_string], taskref_or_string),
 
     # Base work directory used to set up the task.
     Required('workdir'): basestring,
 })
 
 
-def common_setup(config, job, taskdesc, command, checkoutdir):
+def common_setup(config, job, taskdesc, command, geckodir):
     run = job['run']
     if run['checkout']:
         support_vcs_checkout(config, job, taskdesc,
                              sparse=bool(run['sparse-profile']))
-        command.append('--vcs-checkout={}/gecko'.format(checkoutdir))
+        command.append('--vcs-checkout={}'.format(geckodir))
 
     if run['sparse-profile']:
         command.append('--sparse-profile=build/sparse-profiles/%s' %
                        run['sparse-profile'])
 
     taskdesc['worker'].setdefault('env', {})['MOZ_SCM_LEVEL'] = config.params['level']
 
 
@@ -68,17 +68,18 @@ def run_task_url(config):
                 config.params['head_repository'], config.params['head_rev'])
 
 
 @run_job_using("docker-worker", "run-task", schema=run_task_schema, defaults=worker_defaults)
 def docker_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     command = ['/builds/worker/bin/run-task']
-    common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run))
+    common_setup(config, job, taskdesc, command,
+                 geckodir='{workdir}/checkouts/gecko'.format(**run))
 
     if run.get('cache-dotcache'):
         worker['caches'].append({
             'type': 'persistent',
             'name': 'level-{level}-{project}-dotcache'.format(**config.params),
             'mount-point': '{workdir}/.cache'.format(**run),
             'skip-untrusted': True,
         })
@@ -95,17 +96,18 @@ def docker_worker_run_task(config, job, 
     worker['command'] = command
 
 
 @run_job_using("native-engine", "run-task", schema=run_task_schema, defaults=worker_defaults)
 def native_engine_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     command = ['./run-task']
-    common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run))
+    common_setup(config, job, taskdesc, command,
+                 geckodir='{workdir}/checkouts/gecko'.format(**run))
 
     worker['context'] = run_task_url(config)
 
     if run.get('cache-dotcache'):
         raise Exception("No cache support on native-worker; can't use cache-dotcache")
 
     run_command = run['command']
     if isinstance(run_command, basestring):
@@ -114,30 +116,48 @@ def native_engine_run_task(config, job, 
     command.extend(run_command)
     worker['command'] = command
 
 
 @run_job_using("generic-worker", "run-task", schema=run_task_schema, defaults=worker_defaults)
 def generic_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
-    command = ['./run-task']
-    common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run))
+    is_win = worker['os'] == 'windows'
+
+    if is_win:
+        command = ['C:/mozilla-build/python3/python3.exe', 'run-task']
+        geckodir = './build/src'
+    else:
+        command = ['./run-task']
+        geckodir = '{workdir}/checkouts/gecko'.format(**run)
+
+    common_setup(config, job, taskdesc, command, geckodir=geckodir)
 
     worker.setdefault('mounts', [])
     if run.get('cache-dotcache'):
         worker['mounts'].append({
             'cache-name': 'level-{level}-{project}-dotcache'.format(**config.params),
             'directory': '{workdir}/.cache'.format(**run),
         })
     worker['mounts'].append({
         'content': {
             'url': run_task_url(config),
         },
         'file': './run-task',
     })
 
     run_command = run['command']
     if isinstance(run_command, basestring):
+        if is_win:
+            run_command = '"{}"'.format(run_command)
         run_command = ['bash', '-cx', run_command]
+
     command.append('--')
     command.extend(run_command)
-    worker['command'] = [['chmod', '+x', 'run-task'], command]
+
+    if is_win:
+        worker['command'] = [' '.join(command)]
+    else:
+        worker['command'] = [
+            ['chmod', '+x', 'run-task'],
+            command,
+        ]
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -1875,18 +1875,18 @@ def check_caches_are_volumes(task):
     volumes = set(task['worker']['volumes'])
     paths = set(c['mount-point'] for c in task['worker'].get('caches', []))
     missing = paths - volumes
 
     if not missing:
         return
 
     raise Exception('task %s (image %s) has caches that are not declared as '
-                    'Docker volumes: %s'
-                    'Have you added them as VOLUMEs in the Dockerfile?'
+                    'Docker volumes: %s '
+                    '(have you added them as VOLUMEs in the Dockerfile?)'
                     % (task['label'], task['worker']['docker-image'],
                        ', '.join(sorted(missing))))
 
 
 @transforms.add
 def check_run_task_caches(config, tasks):
     """Audit for caches requiring run-task.