Bug 1472236 - Add grcov code coverage builds by using a fetch task. r=marco,gps draft
authorTudor-Gabriel Vîjială <tvijiala@mozilla.com>
Mon, 02 Jul 2018 16:55:56 +0100
changeset 814469 eb9f2c24b361e40bc76469531b6fb6e3730e21a5
parent 813062 9c02d2ecf22050bfee5d70c04a359d8aaff6eb91
push id115219
push userbmo:tvijiala@mozilla.com
push dateThu, 05 Jul 2018 12:26:03 +0000
reviewersmarco, gps
bugs1472236
milestone63.0a1
Bug 1472236 - Add grcov code coverage builds by using a fetch task. r=marco,gps MozReview-Commit-ID: 3PZMNDRqxRM
taskcluster/ci/fetch/toolchains.yml
taskcluster/ci/test/kind.yml
taskcluster/taskgraph/loader/test.py
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/testing/codecoverage.py
--- a/taskcluster/ci/fetch/toolchains.yml
+++ b/taskcluster/ci/fetch/toolchains.yml
@@ -188,8 +188,38 @@ mpfr-3.1.5:
   run:
     using: fetch-url
     url: http://www.mpfr.org/mpfr-3.1.5/mpfr-3.1.5.tar.bz2
     sha256: ca498c1c7a74dd37a576f353312d1e68d490978de4395fa28f1cbd46a364e658
     size: 1279489
     gpg-signature:
       sig-url: "{url}.asc"
       key-path: build/unix/build-gcc/07F3DBBECC1A39605078094D980C197698C3739D.key
+
+grcov-linux-x86_64:
+  description: grcov 0.2.1 binary release
+  treeherder:
+    symbol: grcov-linux-x86_64
+  run:
+    using: fetch-url
+    url: https://github.com/mozilla/grcov/releases/download/v0.2.1/grcov-linux-x86_64.tar.bz2
+    sha256: 8f0af5b49080980b12972ecf214d7afddc277a52a50c6a0bab8b40623b59d442
+    size: 1881829
+
+grcov-osx-x86_64:
+  description: grcov 0.2.0 binary release
+  treeherder:
+    symbol: grcov-osx-x86_64
+  run:
+    using: fetch-url
+    url: https://github.com/mozilla/grcov/releases/download/v0.2.0/grcov-osx-x86_64.tar.bz2
+    sha256: 8b612c603465cb46934e5c3dacaf109a8d9d18f42b64cb8851b27564b6745f98
+    size: 938901
+
+grcov-win-i686:
+  description: grcov 0.2.0 binary release
+  treeherder:
+    symbol: grcov-win-i686
+  run:
+    using: fetch-url
+    url: https://github.com/mozilla/grcov/releases/download/v0.2.0/grcov-win-i686.tar.bz2
+    sha256: 0877e7ea1545a902e04c18ce22666bc07f75fa77f243f6ea036822ebee7147c6
+    size: 923160
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -1,16 +1,18 @@
 loader: taskgraph.loader.test:loader
 
 kind-dependencies:
     - build
     - build-signing
+    - fetch
 
 transforms:
     - taskgraph.transforms.tests:transforms
+    - taskgraph.transforms.use_fetches:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.coalesce:transforms
     - taskgraph.transforms.task:transforms
 
 # Each stanza in a file pointed to by 'jobs-from' describes a particular test
 # suite or sub-suite. These are processed through the transformations described
 # above to produce a bunch of tasks. See the schema in
 # `taskcluster/taskgraph/transforms/tests.py` for a description of the fields
--- a/taskcluster/taskgraph/loader/test.py
+++ b/taskcluster/taskgraph/loader/test.py
@@ -13,22 +13,16 @@ from ..util.yaml import load_yaml
 logger = logging.getLogger(__name__)
 
 
 def loader(kind, path, config, params, loaded_tasks):
     """
     Generate tasks implementing Gecko tests.
     """
 
-    # the kind on which this one depends
-    if len(config.get('kind-dependencies', [])) != 2:
-        raise Exception(
-            'Test kinds must have exactly 2 items in kind-dependencies'
-        )
-
     builds_by_platform = get_builds_by_platform(dep_kind='build', loaded_tasks=loaded_tasks)
     signed_builds_by_platform = get_builds_by_platform(
         dep_kind='build-signing', loaded_tasks=loaded_tasks
     )
 
     # get the test platforms for those build tasks
     test_platforms_cfg = load_yaml(path, 'test-platforms.yml')
     test_platforms = get_test_platforms(
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -165,16 +165,22 @@ def make_task_description(config, jobs):
         taskdesc.setdefault('scopes', [])
         taskdesc.setdefault('extra', {})
 
         # give the function for job.run.using on this worker implementation a
         # chance to set up the task description.
         configure_taskdesc_for_run(config, job, taskdesc, impl)
         del taskdesc['run']
 
+        # get all environment variables from the job, without overwriting the task's variables
+        jobenv = job.get('worker', {}).get('env', {})
+        taskenv = taskdesc.get('worker', {}).get('env', {})
+        for key, value in jobenv.iteritems():
+            taskenv.setdefault(key, value)
+
         # yield only the task description, discarding the job description
         yield taskdesc
 
 
 # A registry of all functions decorated with run_job_using
 registry = {}
 
 
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -718,16 +718,24 @@ def enable_code_coverage(config, tests):
                 test['run-on-projects'] = 'built-projects'
 
             # Ensure we don't optimize test suites out.
             # We always want to run all test suites for coverage purposes.
             test.pop('schedules-component', None)
             test.pop('when', None)
             test['optimization'] = None
 
+            # Add a fetch task for the grcov binary.
+            if 'linux' in test['build-platform']:
+                test['fetches'] = ['grcov-linux-x86_64']
+            elif 'osx' in test['build-platform']:
+                test['fetches'] = ['grcov-osx-x86_64']
+            elif 'win' in test['build-platform']:
+                test['fetches'] = ['grcov-win-i686']
+
             if 'talos' in test['test-name']:
                 test['max-run-time'] = 7200
                 if 'linux' in test['build-platform']:
                     test['docker-image'] = {"in-tree": "desktop1604-test"}
                 test['mozharness']['extra-options'].append('--add-option')
                 test['mozharness']['extra-options'].append('--cycles,1')
                 test['mozharness']['extra-options'].append('--add-option')
                 test['mozharness']['extra-options'].append('--tppagecycles,1')
@@ -1043,16 +1051,18 @@ def make_job_description(config, tests):
             # otherwise just use skip-unless-schedules
             jobdesc['optimization'] = {'skip-unless-schedules': schedules}
 
         run = jobdesc['run'] = {}
         run['using'] = 'mozharness-test'
         run['test'] = test
 
         jobdesc['worker-type'] = test.pop('worker-type')
+        if test.get('fetches'):
+            jobdesc['fetches'] = test.pop('fetches')
 
         yield jobdesc
 
 
 def normpath(path):
     return path.replace('/', '\\')
 
 
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -106,44 +106,30 @@ class CodeCoverageMixin(SingleTestMixin)
             raise Exception('Unexpected OS: {}'.format(mozinfo.os))
 
         os.environ['GCOV_PREFIX_STRIP'] = str(strip_count)
 
         # Install grcov on the test machine
         # Get the path to the build machines gcno files.
         self.url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
         self.url_to_chrome_map = self.query_build_dir_url('chrome-map.json')
-        dirs = self.query_abs_dirs()
-
-        # Create the grcov directory, get the tooltool manifest, and finally
-        # download and unpack the grcov binary.
-        self.grcov_dir = tempfile.mkdtemp()
 
-        if mozinfo.os == 'linux':
-            platform = 'linux64'
-            tar_file = 'grcov-linux-x86_64.tar.bz2'
-        elif mozinfo.os == 'win':
-            platform = 'win32'
-            tar_file = 'grcov-win-i686.tar.bz2'
-        elif mozinfo.os == 'mac':
-            platform = 'macosx64'
-            tar_file = 'grcov-osx-x86_64.tar.bz2'
+        # Create the grcov directory, then download it.
+        # TODO: use the fetch-content script to download artifacts.
+        self.grcov_dir = tempfile.mkdtemp()
+        ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
+        for word in os.getenv('MOZ_FETCHES').split():
+            artifact, task = word.split('@', 1)
+            filename = os.path.basename(artifact)
+            url = ARTIFACT_URL.format(artifact=artifact, task=task)
+            self.download_file(url, parent_dir=self.grcov_dir)
 
-        manifest = os.path.join(dirs.get('abs_test_install_dir',
-                                         os.path.join(dirs['abs_work_dir'], 'tests')),
-                                'config/tooltool-manifests/%s/ccov.manifest' % platform)
-
-        self.tooltool_fetch(
-            manifest=manifest,
-            output_dir=self.grcov_dir,
-            cache=self.config.get('tooltool_cache')
-        )
-
-        with tarfile.open(os.path.join(self.grcov_dir, tar_file)) as tar:
-            tar.extractall(self.grcov_dir)
+            with tarfile.open(os.path.join(self.grcov_dir, filename), 'r') as tar:
+                tar.extractall(self.grcov_dir)
+            os.remove(os.path.join(self.grcov_dir, filename))
 
         # Download the gcno archive from the build machine.
         self.download_file(self.url_to_gcno, parent_dir=self.grcov_dir)
 
         # Download the chrome-map.json file from the build machine.
         self.download_file(self.url_to_chrome_map, parent_dir=self.grcov_dir)
 
     @PostScriptAction('download-and-extract')