Bug 1568460 - Set MOZ_FETCHES_DIR more appropriately. r=tomprince
authorMike Hommey <mh+mozilla@glandium.org>
Mon, 29 Jul 2019 20:48:40 +0000
changeset 485199 d33c8a70c9b1992fb9e18ebc2c8960b6693d2dca
parent 485198 d6ce560581af17b5aadc7957d5c0472faa27ccb6
child 485200 fedfb88f7b3404dbcf60e5d6f5ee96d97b97a1f2
push id36361
push userapavel@mozilla.com
push dateTue, 30 Jul 2019 09:49:32 +0000
treeherdermozilla-central@639f502ded6b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstomprince
bugs1568460
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1568460 - Set MOZ_FETCHES_DIR more appropriately. r=tomprince For some tasks, the workdir known to the decision task doesn't actually match the workdir used in the task, which makes MOZ_FETCHES_DIR wrong when the decision task derives it from the workdir. On other tasks, MOZ_FETCHES_DIR is set to a relative directory, which may work in some places where MOZ_FETCHES_DIR is used, but not in others, that happen to be executed from a different directory. To solve both problems, we set MOZ_FETCHES_DIR as a relative directory everywhere, and we make run-task normalize it to an absolute path before executing the task. Differential Revision: https://phabricator.services.mozilla.com/D39152
taskcluster/ci/generate-profile/kind.yml
taskcluster/ci/source-test/jsshell.yml
taskcluster/scripts/run-task
taskcluster/taskgraph/transforms/job/__init__.py
--- a/taskcluster/ci/generate-profile/kind.yml
+++ b/taskcluster/ci/generate-profile/kind.yml
@@ -152,17 +152,17 @@ jobs:
                   name: public/build/profile-run-1.log
                   path: build/src/artifacts/profile-run-1.log
                 - type: file
                   name: public/build/profile-run-2.log
                   path: build/src/artifacts/profile-run-2.log
         run:
             using: run-task
             sparse-profile: profile-generate
-            command: mv fetches build/src && cd build/src && ./taskcluster/scripts/misc/run-profileserver.sh
+            command: cd build/src && ./taskcluster/scripts/misc/run-profileserver.sh
 
     win64-shippable/opt:
         description: "Win64 Profile Generation"
         shipping-phase: build
         shipping-product: firefox
         index:
             product: firefox
             job-name: win64-profile
@@ -181,9 +181,9 @@ jobs:
                   name: public/build/profile-run-1.log
                   path: build/src/artifacts/profile-run-1.log
                 - type: file
                   name: public/build/profile-run-2.log
                   path: build/src/artifacts/profile-run-2.log
         run:
             using: run-task
             sparse-profile: profile-generate
-            command: mv fetches build/src && cd build/src && ./taskcluster/scripts/misc/run-profileserver.sh
+            command: cd build/src && ./taskcluster/scripts/misc/run-profileserver.sh
--- a/taskcluster/ci/source-test/jsshell.yml
+++ b/taskcluster/ci/source-test/jsshell.yml
@@ -10,28 +10,28 @@ job-defaults:
             linux64.*: t-linux-talos
     worker:
         by-platform:
             linux64.*:
                 env:
                     by-shell:
                         sm:
                             SHELL: /bin/bash
-                            JSSHELL: /home/cltbld/fetches/js
+                            JSSHELL: js
                         v8:
                             SHELL: /bin/bash
-                            JSSHELL: /home/cltbld/fetches/d8/release/d8
+                            JSSHELL: d8/release/d8
                 max-run-time: 1800
     treeherder:
         kind: test
         tier: 2
     run:
         using: mach
         workdir: /home/cltbld
-        mach: jsshell-bench --perfherder={shell} --binary=$JSSHELL {test}
+        mach: jsshell-bench --perfherder={shell} --binary=$MOZ_FETCHES_DIR/$JSSHELL {test}
     run-on-projects:
         by-shell:
             sm: ['mozilla-central', 'try', 'integration']
             default: ['mozilla-central', 'try']
     fetches:
         build:
             - target.jsshell.zip
         toolchain:
--- a/taskcluster/scripts/run-task
+++ b/taskcluster/scripts/run-task
@@ -824,18 +824,22 @@ def main(args):
         os.umask(0o22)
         os.setresgid(gid, gid, gid)
         os.setresuid(uid, uid, uid)
 
     vcs_checkout_from_args(args, 'gecko')
     vcs_checkout_from_args(args, 'comm')
 
     try:
-        if 'GECKO_PATH' in os.environ:
-            os.environ['GECKO_PATH'] = os.path.abspath(os.environ['GECKO_PATH'])
+        for k in ('GECKO_PATH', 'MOZ_FETCHES_DIR'):
+            if k in os.environ:
+                os.environ[k] = os.path.abspath(os.environ[k])
+                print_line(b'setup', b'%s is %s\n' % (
+                    k.encode('utf-8'),
+                    os.environ[k].encode('utf-8')))
 
         if 'MOZ_FETCHES' in os.environ:
             fetch_artifacts()
 
         return run_and_prefix_output(b'task', task_args)
     finally:
         fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
         if fetches_dir and os.path.isdir(fetches_dir):
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -224,21 +224,17 @@ def use_fetches(config, jobs):
                     }
                     if dest is not None:
                         fetch['dest'] = dest
                     job_fetches.append(fetch)
 
         env = worker.setdefault('env', {})
         env['MOZ_FETCHES'] = {'task-reference': json.dumps(job_fetches, sort_keys=True)}
 
-        if worker['os'] in ('windows', 'macosx'):
-            env.setdefault('MOZ_FETCHES_DIR', 'fetches')
-        else:
-            workdir = job['run'].get('workdir', '/builds/worker')
-            env.setdefault('MOZ_FETCHES_DIR', '{}/fetches'.format(workdir))
+        env.setdefault('MOZ_FETCHES_DIR', 'fetches')
 
         yield job
 
 
 @transforms.add
 def make_task_description(config, jobs):
     """Given a build description, create a task description"""
     # import plugin modules first, before iterating over jobs