Bug 1482344 - [raptor] Fix fetch tasks for native-engine mozharness_test based tasks, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 20 Aug 2018 14:04:57 +0000
changeset 487455 aa6f46eaec1bf5e7a930fc5f1dd6fa0bf2542b37
parent 487454 95e33848279667746af656bcc0358bcf223584e2
child 487456 f66ffcb2e82af2f8c16b899af352834bd5bc34a9
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1482344
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1482344 - [raptor] Fix fetch tasks for native-engine mozharness_test based tasks, r=jmaher This unbreaks some tier 3 raptor tasks. There are a few fixes rolled together here: 1) Stop overwriting the 'env' in mozharness_test.py's 'native-engine' implementation 2) Set the workdir to /home/cltbld (which makes sure the fetches are downloaded to there) 3) Download the fetches via mozharness in the 'raptor' script (since they don't use run-task anymore) Depends on D3651 Differential Revision: https://phabricator.services.mozilla.com/D3652
taskcluster/ci/test/raptor.yml
taskcluster/taskgraph/transforms/job/mozharness_test.py
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/testing/raptor.py
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -1,11 +1,12 @@
 job-defaults:
     max-run-time: 1800
     suite: raptor
+    workdir: /home/cltbld
     tier:
         by-test-platform:
             windows10-64-ccov/.*: 3
             linux64-ccov/.*: 3
             default: 2
     virtualization:
         by-test-platform:
             windows10-64-ccov/.*: virtual
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -332,31 +332,32 @@ def mozharness_test_on_native_engine(con
     ]]
 
     if test['reboot']:
         worker['reboot'] = test['reboot']
 
     if test['max-run-time']:
         worker['max-run-time'] = test['max-run-time']
 
-    worker['env'] = env = {
+    env = worker.setdefault('env', {})
+    env.update({
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
         'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
         'MOZHARNESS_SCRIPT': mozharness['script'],
         'MOZHARNESS_URL': {'task-reference': mozharness_url},
         'MOZILLA_BUILD_URL': {'task-reference': installer_url},
         "MOZ_NO_REMOTE": '1',
         "NO_EM_RESTART": '1',
         "XPCOM_DEBUG_BREAK": 'warn',
         "NO_FAIL_ON_TEST_ERRORS": '1',
         "MOZ_HIDE_RESULTS_TABLE": '1',
         "MOZ_NODE_PATH": "/usr/local/bin/node",
         'MOZ_AUTOMATION': '1',
-    }
+    })
     # talos tests don't need Xvfb
     if is_talos:
         env['NEED_XVFB'] = 'false'
 
     script = 'test-macosx.sh' if is_macosx else 'test-linux.sh'
     worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/{}'.format(
         config.params['head_repository'], config.params['head_rev'], script
     )
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -141,16 +141,19 @@ test_description_schema = Schema({
     # description of the suite, for the task metadata
     'description': basestring,
 
     # test suite name, or <suite>/<flavor>
     Required('suite'): optionally_keyed_by(
         'test-platform',
         basestring),
 
+    # base work directory used to set up the task.
+    Optional('workdir'): basestring,
+
     # the name by which this test suite is addressed in try syntax; defaults to
     # the test-name.  This will translate to the `unittest_try_name` or
     # `talos_try_name` attribute.
     Optional('try-name'): basestring,
 
     # additional tags to mark up this type of test
     Optional('tags'): {basestring: object},
 
@@ -1092,16 +1095,19 @@ def make_job_description(config, tests):
         else:
             # otherwise just use skip-unless-schedules
             jobdesc['optimization'] = {'skip-unless-schedules': schedules}
 
         run = jobdesc['run'] = {}
         run['using'] = 'mozharness-test'
         run['test'] = test
 
+        if 'workdir' in test:
+            run['workdir'] = test.pop('workdir')
+
         jobdesc['worker-type'] = test.pop('worker-type')
         if test.get('fetches'):
             jobdesc['fetches'] = test.pop('fetches')
 
         yield jobdesc
 
 
 def normpath(path):
--- a/testing/mozharness/mozharness/mozilla/testing/raptor.py
+++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py
@@ -294,16 +294,18 @@ class Raptor(TestingMixin, MercurialScri
         )
         if self.config.get('run_local'):
             self.raptor_path = os.path.join(self.repo_path, 'testing', 'raptor')
 
     # Action methods. {{{1
     # clobber defined in BaseScript
 
     def download_and_extract(self, extract_dirs=None, suite_categories=None):
+        if 'MOZ_FETCHES' in os.environ:
+            self.fetch_content()
         return super(Raptor, self).download_and_extract(
             suite_categories=['common', 'raptor']
         )
 
     def create_virtualenv(self, **kwargs):
         """VirtualenvMixin.create_virtualenv() assuemes we're using
         self.config['virtualenv_modules']. Since we are installing
         raptor from its source, we have to wrap that method here."""