Bug 1462791 - Add helper function for VCS checkout on generic-worker; r=dustin
authorGregory Szorc <gps@mozilla.com>
Fri, 20 Apr 2018 13:34:45 -0700
changeset 476480 143be456f42025327408b28871585ed0d7b53742
parent 476479 b3350fc6da3aca627f12da57ff55805fdbe7c6b2
child 476481 3409c789902c4c2b439ee96c036d5c6b2b70326d
push id1757
push userffxbld-merge
push dateFri, 24 Aug 2018 17:02:43 +0000
treeherdermozilla-release@736023aebdb1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1462791
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1462791 - Add helper function for VCS checkout on generic-worker; r=dustin This functionality was implemented at least 3 times. Let's consolidate it to a central function. Returning multiple command strings is kind of funky. I preserved existing behavior and mozharness jobs are the only ones printing the forensic logging. We should probably move this logging into robustcheckout so we don't need to involve taskgraph with this. But that can be deferred to another day. MozReview-Commit-ID: I2LglJvfI6
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/mozharness.py
taskcluster/taskgraph/transforms/job/spidermonkey.py
taskcluster/taskgraph/transforms/job/toolchain.py
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -127,16 +127,44 @@ def support_vcs_checkout(config, job, ta
     # for hg.mozilla.org.
     taskdesc['scopes'].append('secrets:get:project/taskcluster/gecko/hgfingerprint')
 
     # only some worker platforms have taskcluster-proxy enabled
     if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
         taskdesc['worker']['taskcluster-proxy'] = True
 
 
+def generic_worker_hg_commands(base_repo, head_repo, head_rev, path):
+    """Obtain commands needed to obtain a Mercurial checkout on generic-worker.
+
+    Returns two command strings. One performs the checkout. Another logs.
+    """
+    args = [
+        r'"c:\Program Files\Mercurial\hg.exe"',
+        'robustcheckout',
+        '--sharebase', r'y:\hg-shared',
+        '--purge',
+        '--upstream', base_repo,
+        '--revision', head_rev,
+        head_repo,
+        path,
+    ]
+
+    logging_args = [
+        b":: TinderboxPrint:<a href={source_repo}/rev/{revision} "
+        b"title='Built from {repo_name} revision {revision}'>{revision}</a>"
+        b"\n".format(
+            revision=head_rev,
+            source_repo=head_repo,
+            repo_name=head_repo.split('/')[-1]),
+    ]
+
+    return ' '.join(args), ' '.join(logging_args)
+
+
 def docker_worker_setup_secrets(config, job, taskdesc):
     """Set up access to secrets via taskcluster-proxy.  The value of
     run['secrets'] should be a boolean or a list of secret names that
     can be accessed."""
     if not job['run'].get('secrets'):
         return
 
     taskdesc['worker']['taskcluster-proxy'] = True
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -19,16 +19,17 @@ from voluptuous import Required, Optiona
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_workspace_cache,
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_setup_secrets,
     docker_worker_add_artifacts,
     docker_worker_add_tooltool,
     generic_worker_add_artifacts,
+    generic_worker_hg_commands,
     support_vcs_checkout,
 )
 
 mozharness_run_schema = Schema({
     Required('using'): 'mozharness',
 
     # the mozharness script used to run this task, relative to the testing/
     # directory and using forward slashes even on Windows
@@ -287,53 +288,30 @@ def mozharness_on_generic_worker(config,
 
     for option in run.get('options', []):
         assert ' ' not in option
         mh_command.append('--' + option)
     if run.get('custom-build-variant-cfg'):
         mh_command.append('--custom-build-variant')
         mh_command.append(run['custom-build-variant-cfg'])
 
-    def checkout_repo(base_repo, head_repo, head_rev, path):
-        hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
-        hg_command.append('robustcheckout')
-        hg_command.extend(['--sharebase', 'y:\\hg-shared'])
-        hg_command.append('--purge')
-        hg_command.extend(['--upstream', base_repo])
-        hg_command.extend(['--revision', head_rev])
-        hg_command.append(head_repo)
-        hg_command.append(path)
-
-        logging_command = [
-            b":: TinderboxPrint:<a href={source_repo}/rev/{revision} "
-            b"title='Built from {repo_name} revision {revision}'>{revision}</a>\n".format(
-                revision=head_rev,
-                source_repo=head_repo,
-                repo_name=head_repo.split('/')[-1],
-            )]
-
-        return [
-            ' '.join(hg_command),
-            ' '.join(logging_command),
-        ]
-
-    hg_commands = checkout_repo(
+    hg_commands = generic_worker_hg_commands(
         base_repo=env['GECKO_BASE_REPOSITORY'],
         head_repo=env['GECKO_HEAD_REPOSITORY'],
         head_rev=env['GECKO_HEAD_REV'],
-        path='.\\build\\src')
+        path=r'.\build\src',
+    )
 
     if run['comm-checkout']:
         hg_commands.extend(
-            checkout_repo(
+            generic_worker_hg_commands(
                 base_repo=env['COMM_BASE_REPOSITORY'],
                 head_repo=env['COMM_HEAD_REPOSITORY'],
                 head_rev=env['COMM_HEAD_REV'],
-                path='.\\build\\src\\comm')
-        )
+                path=r'.\build\src\comm'))
 
     worker['command'] = []
     if taskdesc.get('needs-sccache'):
         worker['command'].extend([
             # Make the comment part of the first command, as it will help users to
             # understand what is going on, and why these steps are implemented.
             dedent('''\
             :: sccache currently uses the full compiler commandline as input to the
--- a/taskcluster/taskgraph/transforms/job/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -9,16 +9,17 @@ from __future__ import absolute_import, 
 
 from taskgraph.util.schema import Schema
 from voluptuous import Required, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_artifacts,
     generic_worker_add_artifacts,
+    generic_worker_hg_commands,
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_add_tooltool,
     support_vcs_checkout,
 )
 
 sm_run_schema = Schema({
     Required('using'): Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate',
                            'spidermonkey-rust-bindings'),
@@ -110,25 +111,22 @@ def generic_worker_spidermonkey(config, 
         script = "build-sm-mozjs-crate.sh"
         # Don't allow untested configurations yet
         raise Exception("spidermonkey-mozjs-crate is not a supported configuration")
     elif run['using'] == 'spidermonkey-rust-bindings':
         script = "build-sm-rust-bindings.sh"
         # Don't allow untested configurations yet
         raise Exception("spidermonkey-rust-bindings is not a supported configuration")
 
-    hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
-    hg_command.append('robustcheckout')
-    hg_command.extend(['--sharebase', 'y:\\hg-shared'])
-    hg_command.append('--purge')
-    hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
-    hg_command.extend(['--revision', env['GECKO_HEAD_REV']])
-    hg_command.append(env['GECKO_HEAD_REPOSITORY'])
-    hg_command.append('.\\src')
+    hg_command = generic_worker_hg_commands(
+        'https://hg.mozilla.org/mozilla-unified',
+        env['GECKO_HEAD_REPOSITORY'],
+        env['GECKO_HEAD_REV'],
+        r'.\src',
+    )[0]
 
     command = ['c:\\mozilla-build\\msys\\bin\\bash.exe '  # string concat
                '"./src/taskcluster/scripts/builder/%s"' % script]
 
-    worker['command'] = []
-    worker['command'].extend([
-        ' '.join(hg_command),
-        ' '.join(command)
-    ])
+    worker['command'] = [
+        hg_command,
+        ' '.join(command),
+    ]
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -12,16 +12,17 @@ from mozbuild.shellutil import quote as 
 from taskgraph.util.schema import Schema
 from voluptuous import Optional, Required, Any
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
     docker_worker_add_gecko_vcs_env_vars,
     docker_worker_add_artifacts,
     docker_worker_add_tooltool,
+    generic_worker_hg_commands,
     support_vcs_checkout,
 )
 from taskgraph.util.hash import hash_paths
 from taskgraph import GECKO
 from taskgraph.util.cached_tasks import add_optimization
 import taskgraph
 
 
@@ -200,37 +201,33 @@ def windows_toolchain(config, job, taskd
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'MOZ_AUTOMATION': '1',
     })
 
-    hg = r'c:\Program Files\Mercurial\hg.exe'
-    hg_command = ['"{}"'.format(hg)]
-    hg_command.append('robustcheckout')
-    hg_command.extend(['--sharebase', 'y:\\hg-shared'])
-    hg_command.append('--purge')
-    hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
-    hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
-    hg_command.append('%GECKO_HEAD_REPOSITORY%')
-    hg_command.append('.\\build\\src')
+    hg_command = generic_worker_hg_commands(
+        'https://hg.mozilla.org/mozilla-unified',
+        env['GECKO_HEAD_REPOSITORY'],
+        env['GECKO_HEAD_REV'],
+        r'.\build\src')[0]
 
     # Use `mach` to invoke python scripts so in-tree libraries are available.
     if run['script'].endswith('.py'):
         raise NotImplementedError("Python scripts don't work on Windows")
 
     args = run.get('arguments', '')
     if args:
         args = ' ' + shell_quote(*args)
 
     bash = r'c:\mozilla-build\msys\bin\bash'
     worker['command'] = [
-        ' '.join(hg_command),
+        hg_command,
         # do something intelligent.
         r'{} build/src/taskcluster/scripts/misc/{}{}'.format(
             bash, run['script'], args)
     ]
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
     if 'toolchain-alias' in run: