Bug 1482344 - [mozharness] Refactor codecoverage fetch downloading into a standalone mixin, r=tvijiala
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 20 Aug 2018 14:04:41 +0000
changeset 487454 95e33848279667746af656bcc0358bcf223584e2
parent 487453 a72f317a470e0222e91eee1885ea9463a0f70c65
child 487455 aa6f46eaec1bf5e7a930fc5f1dd6fa0bf2542b37
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstvijiala
bugs1482344
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1482344 - [mozharness] Refactor codecoverage fetch downloading into a standalone mixin, r=tvijiala We need to grab fetches from several place in mozharness, this creates a dedicated mixin that can be used from anywhere. If the 'fetch-content' script is detected that will be used, otherwise we download the fetches manually. Differential Revision: https://phabricator.services.mozilla.com/D3651
testing/mozharness/mozharness/mozilla/fetches.py
testing/mozharness/mozharness/mozilla/testing/codecoverage.py
testing/mozharness/mozharness/mozilla/testing/per_test_base.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/mozharness/mozilla/fetches.py
@@ -0,0 +1,59 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from distutils.spawn import find_executable
+
+import mozfile
+
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
+
+
+class FetchesMixin(object):
+    """Utility class to download artifacts via `MOZ_FETCHES` and the
+    `fetch-content` script."""
+
+    @property
+    def fetch_script(self):
+        if getattr(self, '_fetch_script', None):
+            return self._fetch_script
+
+        self._fetch_script = find_executable('fetch-content')
+        if not self._fetch_script and 'GECKO_PATH' in os.environ:
+            self._fetch_script = os.path.join(os.environ['GECKO_PATH'],
+                'taskcluster', 'script', 'misc', 'fetch-content')
+        return self._fetch_script
+
+    def fetch_content(self):
+        if not os.environ.get('MOZ_FETCHES'):
+            self.warning('no fetches to download')
+            return
+
+        fetches = os.environ['MOZ_FETCHES'].split()
+
+        if not self.fetch_script or not os.path.isfile(self.fetch_script):
+            self.warning("fetch-content script not found, downloading manually")
+            self._download_fetches(fetches)
+            return
+
+        cmd = [self.fetch_script, 'task-artifacts'] + fetches
+        self.run_command(cmd, env=os.environ, throw_exception=True)
+
+    def _download_fetches(self, fetches):
+        # TODO: make sure fetch-content script is available everywhere
+        #       so this isn't needed
+        for word in fetches:
+            artifact, task = word.split('@', 1)
+            extdir = os.environ['MOZ_FETCHES_DIR']
+
+            if '>' in artifact:
+                artifact, subdir = artifact.rsplit('>', 1)
+                extdir = os.path.join(extdir, subdir)
+
+            url = ARTIFACT_URL.format(artifact=artifact, task=task)
+            self.download_file(url)
+
+            filename = os.path.basename(artifact)
+            mozfile.extract(filename, extdir)
+            os.remove(filename)
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -3,17 +3,16 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import posixpath
 import shutil
 import sys
-import tarfile
 import tempfile
 import zipfile
 import uuid
 
 import mozinfo
 from mozharness.base.script import (
     PreScriptAction,
     PostScriptAction,
@@ -139,41 +138,24 @@ class CodeCoverageMixin(SingleTestMixin)
         with zipfile.ZipFile(classfiles_zip_path, 'r') as z:
             z.extractall(self.classfiles_dir)
         os.remove(classfiles_zip_path)
 
         # Create the directory where the emulator coverage file will be placed.
         self.java_coverage_output_path = os.path.join(tempfile.mkdtemp(),
                                                       'junit-coverage.ec')
 
-    def _download_grcov(self):
-        fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
-        if fetches_dir and os.path.isfile(os.path.join(fetches_dir, 'grcov')):
-            self.grcov_dir = fetches_dir
-        else:
-            # Create the grcov directory, then download it.
-            # TODO: use the fetch-content script to download artifacts.
-            self.grcov_dir = tempfile.mkdtemp()
-            ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
-            for word in os.getenv('MOZ_FETCHES').split():
-                artifact, task = word.split('@', 1)
-                filename = os.path.basename(artifact)
-                url = ARTIFACT_URL.format(artifact=artifact, task=task)
-                self.download_file(url, parent_dir=self.grcov_dir)
-
-                with tarfile.open(os.path.join(self.grcov_dir, filename), 'r') as tar:
-                    tar.extractall(self.grcov_dir)
-                os.remove(os.path.join(self.grcov_dir, filename))
-
     @PostScriptAction('download-and-extract')
     def setup_coverage_tools(self, action, success=None):
         if not self.code_coverage_enabled and not self.java_code_coverage_enabled:
             return
 
-        self._download_grcov()
+        self.grcov_dir = os.environ['MOZ_FETCHES_DIR']
+        if not os.path.isfile(os.path.join(self.grcov_dir, 'grcov')):
+            self.fetch_content()
 
         if self.code_coverage_enabled:
             self._setup_cpp_js_coverage_tools()
 
         if self.java_code_coverage_enabled:
             self._setup_java_coverage_tools()
 
     @PostScriptAction('download-and-extract')
--- a/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
+++ b/testing/mozharness/mozharness/mozilla/testing/per_test_base.py
@@ -8,18 +8,20 @@
 import math
 import os
 import posixpath
 import re
 import sys
 import mozinfo
 from manifestparser import TestManifest
 
+from mozharness.mozilla.fetches import FetchesMixin
 
-class SingleTestMixin(object):
+
+class SingleTestMixin(FetchesMixin):
     """Utility functions for per-test testing like test verification and per-test coverage."""
 
     def __init__(self):
         self.suites = {}
         self.tests_downloaded = False
         self.reftest_test_dir = None
         self.jsreftest_test_dir = None
         # Map from full test path on the test machine to a relative path in the source checkout.
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -15,16 +15,17 @@ from urlparse import urlparse, ParseResu
 from mozharness.base.errors import BaseErrorList
 from mozharness.base.log import FATAL, WARNING
 from mozharness.base.python import (
     ResourceMonitoringMixin,
     VirtualenvMixin,
     virtualenv_config_options,
 )
 from mozharness.mozilla.automation import AutomationMixin, TBPL_WARNING
+from mozharness.mozilla.fetches import FetchesMixin
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
 from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options
 from mozharness.mozilla.testing.verify_tools import VerifyToolsMixin, verify_config_options
 from mozharness.mozilla.tooltool import TooltoolMixin
 
 from mozharness.lib.python.authentication import get_credentials
 
@@ -97,17 +98,17 @@ testing_config_options = [
       }],
 ] + copy.deepcopy(virtualenv_config_options) \
   + copy.deepcopy(try_config_options) \
   + copy.deepcopy(verify_config_options)
 
 
 # TestingMixin {{{1
 class TestingMixin(VirtualenvMixin, AutomationMixin, ResourceMonitoringMixin,
-                   TooltoolMixin, TryToolsMixin, VerifyToolsMixin):
+                   TooltoolMixin, TryToolsMixin, VerifyToolsMixin, FetchesMixin):
     """
     The steps to identify + download the proper bits for [browser] unit
     tests and Talos.
     """
 
     installer_url = None
     installer_path = None
     binary_path = None