Bug 1461980 - [taskgraph] Use run-task's 'use-artifacts' key to setup artifacts in mochitest and reftest selftests, r=dustin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 18 May 2018 08:22:36 -0400
changeset 419300 bbeba612682291bca2733a58552af2be4a12c0c4
parent 419299 aceeafcc22c998693a68adb607d8c0fb1e8a59c2
child 419301 bf4962739d38ac21ba6ba216fa61f572e7976354
push id64289
push userahalberstadt@mozilla.com
push dateTue, 22 May 2018 13:02:11 +0000
treeherderautoland@bbeba6126822 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1461980
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1461980 - [taskgraph] Use run-task's 'use-artifacts' key to setup artifacts in mochitest and reftest selftests, r=dustin Instead of downloading the build artifacts (rather hackily) in moztest.fixtures, this now happens directly in the taskgraph module via the run-task script. For now extraction and setup happens in the task's command key. It might be a good idea to figure out a syntax to tell run-task to do this extraction, e.g something like: run: using-artifacts: build: target.tar.bz2: extract: true path: /home/worker/build name: firefox But for now I wanted to avoid this extra complexity, so maybe it could be done in a follow-up. MozReview-Commit-ID: KOhFFpFdP7Y
layout/tools/reftest/selftest/conftest.py
taskcluster/ci/source-test/kind.yml
taskcluster/ci/source-test/python.yml
taskcluster/taskgraph/transforms/source_test.py
testing/mozbase/moztest/moztest/selftest/fixtures.py
--- a/layout/tools/reftest/selftest/conftest.py
+++ b/layout/tools/reftest/selftest/conftest.py
@@ -43,16 +43,19 @@ def runtests(setup_test_harness, binary,
     if not os.path.isdir(build.bindir):
         package_root = os.path.dirname(harness_root)
         options.update({
             'extraProfileFiles': [os.path.join(package_root, 'bin', 'plugins')],
             'reftestExtensionPath': os.path.join(harness_root, 'reftest'),
             'sandboxReadWhitelist': [here, os.environ['PYTHON_TEST_TMP']],
             'utilityPath': os.path.join(package_root, 'bin'),
         })
+
+        if 'USE_ARTIFACT_PATH' in os.environ:
+            options['sandboxReadWhitelist'].append(os.environ['USE_ARTIFACT_PATH'])
     else:
         options.update({
             'extraProfileFiles': [os.path.join(build.topobjdir, 'dist', 'plugins')],
             'sandboxReadWhitelist': [build.topobjdir, build.topsrcdir],
         })
 
     def normalize(test):
         if os.path.isabs(test):
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -18,17 +18,11 @@ jobs-from:
    - mozlint.yml
    - python.yml
    - webidl.yml
    - manifest-upload.yml
 
 # This is used by run-task based tasks to lookup which build task it
 # should depend on based on its own platform.
 dependent-build-platforms:
-   linux64-asan/opt:
-      label: build-linux64-asan/opt
-      target-name: target.tar.bz2
-   linux64/debug:
-      label: build-linux64/debug
-      target-name: target.tar.bz2
-   linux64.*:
-      label: build-linux64/opt
-      target-name: target.tar.bz2
+   linux64-asan/opt: build-linux64-asan/opt
+   linux64/debug: build-linux64/debug
+   linux64.*: build-linux64/opt
--- a/taskcluster/ci/source-test/python.yml
+++ b/taskcluster/ci/source-test/python.yml
@@ -55,19 +55,30 @@ mochitest-harness:
         symbol: py(mch)
     worker:
         by-platform:
             linux64.*:
                 docker-image: {in-tree: "desktop1604-test"}
                 max-run-time: 3600
     run:
         using: run-task
+        use-artifacts:
+            build:
+                - target.tar.bz2
+                - target.common.tests.zip
+                - target.mochitest.tests.zip
         command: >
             source /builds/worker/scripts/xvfb.sh &&
             start_xvfb '1600x1200x24' 0 &&
+            cd $USE_ARTIFACT_PATH/build &&
+            tar -xf target.tar.bz2 &&
+            unzip -q -d tests target.common.tests.zip &&
+            unzip -q -d tests target.mochitest.tests.zip &&
+            export GECKO_BINARY_PATH=$USE_ARTIFACT_PATH/build/firefox/firefox &&
+            export TEST_HARNESS_ROOT=$USE_ARTIFACT_PATH/build/tests &&
             cd /builds/worker/checkouts/gecko &&
             ./mach python-test --subsuite mochitest
     when:
         files-changed:
             - 'testing/mochitest/**'
             - 'testing/mozbase/moztest/moztest/selftest/**'
             - 'testing/mozharness/mozharness/base/log.py'
             - 'testing/mozharness/mozharness/mozilla/structuredlog.py'
@@ -151,19 +162,30 @@ reftest-harness:
         symbol: py(ref)
     worker:
         by-platform:
             linux64.*:
                 docker-image: {in-tree: "desktop1604-test"}
                 max-run-time: 3600
     run:
         using: run-task
+        use-artifacts:
+            build:
+                - target.tar.bz2
+                - target.common.tests.zip
+                - target.reftest.tests.zip
         command: >
             source /builds/worker/scripts/xvfb.sh &&
             start_xvfb '1600x1200x24' 0 &&
+            cd $USE_ARTIFACT_PATH/build &&
+            tar -xf target.tar.bz2 &&
+            unzip -q -d tests target.common.tests.zip &&
+            unzip -q -d tests target.reftest.tests.zip &&
+            export GECKO_BINARY_PATH=$USE_ARTIFACT_PATH/build/firefox/firefox &&
+            export TEST_HARNESS_ROOT=$USE_ARTIFACT_PATH/build/tests &&
             cd /builds/worker/checkouts/gecko &&
             ./mach python-test --subsuite reftest
     when:
         files-changed:
             - 'layout/tools/reftest/**'
             - 'testing/mozbase/moztest/moztest/selftest/**'
             - 'testing/mozharness/mozharness/base/log.py'
             - 'testing/mozharness/mozharness/mozilla/structuredlog.py'
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -112,27 +112,20 @@ def add_build_dependency(config, job):
     matches = keymatch(build_labels, key)
     if not matches:
         raise Exception("No build platform found for '{}'. "
                         "Define 'dependent-build-platforms' in the kind config.".format(key))
 
     if len(matches) > 1:
         raise Exception("More than one build platform found for '{}'.".format(key))
 
-    label = matches[0]['label']
-    target = matches[0]['target-name']
+    label = matches[0]
     deps = job.setdefault('dependencies', {})
     deps.update({'build': label})
 
-    build_artifact = 'public/build/{}'.format(target)
-    installer_url = ARTIFACT_URL.format('<build>', build_artifact)
-
-    env = job['worker'].setdefault('env', {})
-    env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
-
 
 @transforms.add
 def handle_platform(config, jobs):
     """
     Handle the 'platform' property, setting up treeherder context as well as
     try-related attributes.
     """
     fields = [
--- a/testing/mozbase/moztest/moztest/selftest/fixtures.py
+++ b/testing/mozbase/moztest/moztest/selftest/fixtures.py
@@ -6,20 +6,18 @@ in test harness selftests.
 """
 
 from __future__ import absolute_import
 
 import os
 import shutil
 import sys
 
-import mozfile
 import mozinstall
 import pytest
-import requests
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 try:
     from mozbuild.base import MozbuildObject
     build = MozbuildObject.from_environment(cwd=here)
 except ImportError:
     build = None
@@ -33,39 +31,20 @@ environment variable is required.
 
 def _get_test_harness(suite, install_dir):
     # Check if there is a local build
     if build:
         harness_root = os.path.join(build.topobjdir, '_tests', install_dir)
         if os.path.isdir(harness_root):
             return harness_root
 
-    # Check if it was previously set up by another test
-    harness_root = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests', suite)
-    if os.path.isdir(harness_root):
-        return harness_root
-
-    # Check if there is a test package to download
-    if 'GECKO_INSTALLER_URL' in os.environ:
-        base_url = os.environ['GECKO_INSTALLER_URL'].rsplit('/', 1)[0]
-        test_packages = requests.get(base_url + '/target.test_packages.json').json()
-
-        dest = os.path.join(os.environ['PYTHON_TEST_TMP'], 'tests')
-        for name in test_packages[suite]:
-            url = base_url + '/' + name
-            bundle = os.path.join(os.environ['PYTHON_TEST_TMP'], name)
-
-            r = requests.get(url, stream=True)
-            with open(bundle, 'w+b') as fh:
-                for chunk in r.iter_content(chunk_size=1024):
-                    fh.write(chunk)
-
-            mozfile.extract(bundle, dest)
-
-        return os.path.join(dest, suite)
+    if 'TEST_HARNESS_ROOT' in os.environ:
+        harness_root = os.path.join(os.environ['TEST_HARNESS_ROOT'], suite)
+        if os.path.isdir(harness_root):
+            return harness_root
 
     # Couldn't find a harness root, let caller do error handling.
     return None
 
 
 @pytest.fixture(scope='session')
 def setup_test_harness(request):
     """Fixture for setting up a mozharness-based test harness like
@@ -80,17 +59,17 @@ def setup_test_harness(request):
             if files_dir:
                 test_root = os.path.join(harness_root, 'tests', 'selftests')
                 if not os.path.exists(test_root):
                     if hasattr(os, 'symlink'):
                         os.symlink(files_dir, test_root)
                     else:
                         shutil.copytree(files_dir, test_root)
 
-        elif 'GECKO_INSTALLER_URL' in os.environ:
+        elif 'TEST_HARNESS_ROOT' in os.environ:
             # The mochitest tests will run regardless of whether a build exists or not.
             # In a local environment, they should simply be skipped if setup fails. But
             # in automation, we'll need to make sure an error is propagated up.
             pytest.fail(HARNESS_ROOT_NOT_FOUND)
         else:
             # Tests will be marked skipped by the calls to pytest.importorskip() below.
             # We are purposefully not failing here because running |mach python-test|
             # without a build is a perfectly valid use case.
@@ -109,15 +88,10 @@ def binary():
     app = 'firefox'
     bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app)
     if os.path.isdir(bindir):
         try:
             return mozinstall.get_binary(bindir, app_name=app)
         except Exception:
             pass
 
-    if 'GECKO_INSTALLER_URL' in os.environ:
-        bindir = mozinstall.install(
-            os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
-        return mozinstall.get_binary(bindir, app_name='firefox')
-
     if 'GECKO_BINARY_PATH' in os.environ:
         return os.environ['GECKO_BINARY_PATH']