Bug 1391427 - Execute python toolchain scripts with mach. r=glandium
authorRalph Giles <giles@mozilla.com>
Tue, 12 Sep 2017 16:01:26 -0700
changeset 383302 1db94c2f3321dca877079f52e539531036e33586
parent 383301 fde231f9bb5194f4b353e14b1e654d4e1df89122
child 383303 5e5e9ad6ed9abb45733d78082c429df0d7b6f17a
push id95539
push userkwierso@gmail.com
push dateThu, 28 Sep 2017 00:01:12 +0000
treeherdermozilla-inbound@72de90e66155 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1391427
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1391427 - Execute python toolchain scripts with mach. r=glandium Run scripts with a `.py` filename extension through `./mach python` so the normal enviroment with in-tree python libraries is available. This is helpful for the Rust upstream release download and repackaging steps, which are more easily expressed in python than in an sh-based build script like we use for clang and other tools. Invocation of `mach python` on Windows-hosted generic workers fails because of some missing environment pieces. For the purposes of this bug we can just run the script for Windows targets in a docker-worker so Python support was left unimplemented for generic workers. MozReview-Commit-ID: 4XUQ1XrVBc2
taskcluster/taskgraph/transforms/job/toolchain.py
--- a/taskcluster/taskgraph/transforms/job/toolchain.py
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -24,17 +24,19 @@ from taskgraph.util.hash import hash_pat
 from taskgraph import GECKO
 
 
 TOOLCHAIN_INDEX = 'gecko.cache.level-{level}.toolchains.v1.{name}.{digest}'
 
 toolchain_run_schema = Schema({
     Required('using'): 'toolchain-script',
 
-    # the script (in taskcluster/scripts/misc) to run
+    # The script (in taskcluster/scripts/misc) to run.
+    # Python scripts are invoked with `mach python` so vendored libraries
+    # are available.
     Required('script'): basestring,
 
     # If not false, tooltool downloads will be enabled via relengAPIProxy
     # for either just public files, or all files.  Not supported on Windows
     Required('tooltool-downloads', default=False): Any(
         False,
         'public',
         'internal',
@@ -115,26 +117,32 @@ def docker_worker_toolchain(config, job,
         'TOOLS_DISABLE': 'true',
         'MOZ_AUTOMATION': '1',
     })
 
     if run['tooltool-downloads']:
         internal = run['tooltool-downloads'] == 'internal'
         docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
 
+    # Use `mach` to invoke python scripts so in-tree libraries are available.
+    if run['script'].endswith('.py'):
+        wrapper = 'workspace/build/src/mach python '
+    else:
+        wrapper = ''
+
     worker['command'] = [
         '/builds/worker/bin/run-task',
         '--vcs-checkout=/builds/worker/workspace/build/src',
         '--sparse-profile', 'build/sparse-profiles/toolchain-build',
         '--',
         'bash',
         '-c',
         'cd /builds/worker && '
-        './workspace/build/src/taskcluster/scripts/misc/{}'.format(
-            run['script'])
+        '{}workspace/build/src/taskcluster/scripts/misc/{}'.format(
+            wrapper, run['script'])
     ]
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
     if 'toolchain-alias' in run:
         attributes['toolchain-alias'] = run['toolchain-alias']
 
     add_optimization(config, run, taskdesc)
@@ -167,16 +175,20 @@ def windows_toolchain(config, job, taskd
     hg_command.append('robustcheckout')
     hg_command.extend(['--sharebase', 'y:\\hg-shared'])
     hg_command.append('--purge')
     hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
     hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
     hg_command.append('%GECKO_HEAD_REPOSITORY%')
     hg_command.append('.\\build\\src')
 
+    # Use `mach` to invoke python scripts so in-tree libraries are available.
+    if run['script'].endswith('.py'):
+        raise NotImplementedError("Python scripts don't work on Windows")
+
     bash = r'c:\mozilla-build\msys\bin\bash'
     worker['command'] = [
         ' '.join(hg_command),
         # do something intelligent.
         r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
     ]
 
     attributes = taskdesc.setdefault('attributes', {})