Bug 1473313 - Part 3: Extend CodeCoverageMixin to handle java code coverage tools. r=marco,gbrown draft
authorTudor-Gabriel Vîjială <tvijiala@mozilla.com>
Tue, 24 Jul 2018 11:51:54 +0100
changeset 825870 4d6d0b55435905933e452e03f649bb3382efee62
parent 825869 b2e4f3c1c4d2313c3108ecdedb86bfbad1f02fb7
child 825871 bc8db8e9cc8748e095f7e9a8fa73430ab88598fc
push id118194
push userbmo:tvijiala@mozilla.com
push dateThu, 02 Aug 2018 15:08:25 +0000
reviewersmarco, gbrown
bugs1473313
milestone63.0a1
Bug 1473313 - Part 3: Extend CodeCoverageMixin to handle java code coverage tools. r=marco,gbrown MozReview-Commit-ID: Lld5XLUMb3Y
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/testing/codecoverage.py
testing/mozharness/scripts/android_emulator_unittest.py
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -721,16 +721,22 @@ def handle_suite_category(config, tests)
 def enable_code_coverage(config, tests):
     """Enable code coverage for the ccov and jsdcov build-platforms"""
     for test in tests:
         if 'ccov' in test['build-platform']:
             # do not run tests on fuzzing or opt build
             if 'opt' in test['build-platform'] or 'fuzzing' in test['build-platform']:
                 test['run-on-projects'] = []
                 continue
+            # Skip this transform for android code coverage builds.
+            if 'android' in test['build-platform']:
+                test.setdefault('fetches', {}).setdefault('fetch', []).append('grcov-linux-x86_64')
+                test['mozharness'].setdefault('extra-options', []).append('--java-code-coverage')
+                yield test
+                continue
             test['mozharness'].setdefault('extra-options', []).append('--code-coverage')
             test['instance-size'] = 'xlarge'
             # Ensure we always run on the projects defined by the build, unless the test
             # is try only or shouldn't run at all.
             if test['run-on-projects'] not in [[], ['try']]:
                 test['run-on-projects'] = 'built-projects'
 
             # Ensure we don't optimize test suites out.
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -40,16 +40,22 @@ code_coverage_config_options = [
       "help": "Whether test run should package and upload code coverage data."
       }],
     [["--jsd-code-coverage"],
      {"action": "store_true",
       "dest": "jsd_code_coverage",
       "default": False,
       "help": "Whether JSDebugger code coverage should be run."
       }],
+    [["--java-code-coverage"],
+     {"action": "store_true",
+      "dest": "java_code_coverage",
+      "default": False,
+      "help": "Whether Java code coverage should be run."
+      }],
 ]
 
 
 class CodeCoverageMixin(SingleTestMixin):
     """
     Mixin for setting GCOV_PREFIX during test execution, packaging up
     the resulting .gcda files and uploading them to blobber.
     """
@@ -84,39 +90,65 @@ class CodeCoverageMixin(SingleTestMixin)
 
     @property
     def jsd_code_coverage_enabled(self):
         try:
             return bool(self.config.get('jsd_code_coverage'))
         except (AttributeError, KeyError, TypeError):
             return False
 
-    @PostScriptAction('download-and-extract')
-    def setup_coverage_tools(self, action, success=None):
-        if not self.code_coverage_enabled:
-            return
+    @property
+    def java_code_coverage_enabled(self):
+        try:
+            return bool(self.config.get('java_code_coverage'))
+        except (AttributeError, KeyError, TypeError):
+            return False
 
+    def _setup_cpp_js_coverage_tools(self):
         if mozinfo.os == 'linux' or mozinfo.os == 'mac':
             self.prefix = '/builds/worker/workspace/build/src/'
             strip_count = self.prefix.count('/')
         elif mozinfo.os == 'win':
             self.prefix = 'z:/build/build/src/'
             # Add 1 as on Windows the path where the compiler tries to write the
             # gcda files has an additional 'obj-firefox' component.
             strip_count = self.prefix.count('/') + 1
         else:
             raise Exception('Unexpected OS: {}'.format(mozinfo.os))
 
         os.environ['GCOV_PREFIX_STRIP'] = str(strip_count)
 
-        # Install grcov on the test machine
-        # Get the path to the build machines gcno files.
-        self.url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
-        self.url_to_chrome_map = self.query_build_dir_url('chrome-map.json')
+        # Download the gcno archive from the build machine.
+        url_to_gcno = self.query_build_dir_url('target.code-coverage-gcno.zip')
+        self.download_file(url_to_gcno, parent_dir=self.grcov_dir)
+
+        # Download the chrome-map.json file from the build machine.
+        url_to_chrome_map = self.query_build_dir_url('chrome-map.json')
+        self.download_file(url_to_chrome_map, parent_dir=self.grcov_dir)
+
+    def _setup_java_coverage_tools(self):
+        # Download and extract jacoco-cli from the build task.
+        url_to_jacoco = self.query_build_dir_url('target.jacoco-cli.jar')
+        self.jacoco_jar = os.path.join(tempfile.mkdtemp(), 'target.jacoco-cli.jar')
+        self.download_file(url_to_jacoco, self.jacoco_jar)
 
+        # Download and extract class files from the build task.
+        self.classfiles_dir = tempfile.mkdtemp()
+        url_to_classfiles = self.query_build_dir_url('target.geckoview_classfiles.zip')
+        classfiles_zip_path = os.path.join(self.classfiles_dir, 'target.geckoview_classfiles.zip')
+        self.download_file(url_to_classfiles, classfiles_zip_path)
+        with zipfile.ZipFile(classfiles_zip_path, 'r') as z:
+            z.extractall(self.classfiles_dir)
+        os.remove(classfiles_zip_path)
+
+        # Create the directory where the emulator coverage file will be placed.
+        self.java_coverage_output_path = os.path.join(tempfile.mkdtemp(),
+                                                      'junit-coverage.ec')
+
+    def _download_grcov(self):
         fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
         if fetches_dir and os.path.isfile(os.path.join(fetches_dir, 'grcov')):
             self.grcov_dir = fetches_dir
         else:
             # Create the grcov directory, then download it.
             # TODO: use the fetch-content script to download artifacts.
             self.grcov_dir = tempfile.mkdtemp()
             ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{task}/artifacts/{artifact}'
@@ -125,21 +157,28 @@ class CodeCoverageMixin(SingleTestMixin)
                 filename = os.path.basename(artifact)
                 url = ARTIFACT_URL.format(artifact=artifact, task=task)
                 self.download_file(url, parent_dir=self.grcov_dir)
 
                 with tarfile.open(os.path.join(self.grcov_dir, filename), 'r') as tar:
                     tar.extractall(self.grcov_dir)
                 os.remove(os.path.join(self.grcov_dir, filename))
 
-        # Download the gcno archive from the build machine.
-        self.download_file(self.url_to_gcno, parent_dir=self.grcov_dir)
+    @PostScriptAction('download-and-extract')
+    def setup_coverage_tools(self, action, success=None):
+        if not self.code_coverage_enabled and not self.java_code_coverage_enabled:
+            return
 
-        # Download the chrome-map.json file from the build machine.
-        self.download_file(self.url_to_chrome_map, parent_dir=self.grcov_dir)
+        self._download_grcov()
+
+        if self.code_coverage_enabled:
+            self._setup_cpp_js_coverage_tools()
+
+        if self.java_code_coverage_enabled:
+            self._setup_java_coverage_tools()
 
     @PostScriptAction('download-and-extract')
     def find_tests_for_coverage(self, action, success=None):
         """
            For each file modified on this push, determine if the modified file
            is a test, by searching test manifests. Populate self.verify_suites
            with test files, organized by suite.
 
@@ -435,16 +474,58 @@ class CodeCoverageMixin(SingleTestMixin)
 
             # Zip the JSVM coverage data and upload it.
             jsvm_zip_path = os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-jsvm.zip')
             with zipfile.ZipFile(jsvm_zip_path, 'w', zipfile.ZIP_DEFLATED) as z:
                 z.write(jsvm_output_file)
 
         shutil.rmtree(self.grcov_dir)
 
+    @PostScriptAction('run-tests')
+    def process_java_coverage_data(self, action, success=None):
+        '''
+        Run JaCoCo on the coverage.exec file in order to get a XML report.
+        After that, run grcov on the XML report to get a lcov report.
+        Finally, archive the lcov file and upload it, as process_coverage_data is doing.
+        '''
+        if not self.java_code_coverage_enabled:
+            return
+
+        # Don't try to get reports after a failed test run, as the coverage
+        # report probably won't exist and the error will prevent automatic task retry.
+        if not success:
+            return
+
+        dirs = self.query_abs_dirs()
+        xml_path = tempfile.mkdtemp()
+        jacoco_command = ['java', '-jar', self.jacoco_jar, 'report',
+                          self.java_coverage_output_path,
+                          '--classfiles', self.classfiles_dir,
+                          '--name', 'geckoview-junit',
+                          '--xml', os.path.join(xml_path, 'geckoview-junit.xml')]
+        self.run_command(jacoco_command, halt_on_failure=True)
+
+        grcov_command = [
+            os.path.join(self.grcov_dir, 'grcov'),
+            '-t', 'lcov',
+            xml_path,
+        ]
+        tmp_output_file, _ = self.get_output_from_command(
+            grcov_command,
+            silent=True,
+            save_tmpfiles=True,
+            return_type='files',
+            throw_exception=True,
+        )
+
+        if not self.ccov_upload_disabled:
+            grcov_zip_path = os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip')
+            with zipfile.ZipFile(grcov_zip_path, 'w', zipfile.ZIP_DEFLATED) as z:
+                z.write(tmp_output_file, 'grcov_lcov_output.info')
+
 
 def rm_baseline_cov(baseline_coverage, test_coverage):
     '''
     Returns the difference between test_coverage and
     baseline_coverage, such that what is returned
     is the unique coverage for the test in question.
     '''
 
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -21,17 +21,20 @@ sys.path.insert(1, os.path.dirname(sys.p
 
 from mozprocess import ProcessHandler
 
 from mozharness.base.log import FATAL
 from mozharness.base.script import BaseScript, PreScriptAction, PostScriptAction
 from mozharness.mozilla.automation import TBPL_RETRY, EXIT_STATUS_DICT
 from mozharness.mozilla.mozbase import MozbaseMixin
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
-from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin
+from mozharness.mozilla.testing.codecoverage import (
+    CodeCoverageMixin,
+    code_coverage_config_options
+)
 
 
 class AndroidEmulatorTest(TestingMixin, BaseScript, MozbaseMixin, CodeCoverageMixin):
     """
        A mozharness script for Android functional tests (like mochitests and reftests)
        run on an Android emulator. This script starts and manages an Android emulator
        for the duration of the required tests. This is like desktop_unittest.py, but
        for Android emulator test platforms.
@@ -72,17 +75,18 @@ class AndroidEmulatorTest(TestingMixin, 
          }
     ], [
         ["--log-tbpl-level"],
         {"action": "store",
          "dest": "log_tbpl_level",
          "default": "info",
          "help": "Set log level (debug|info|warning|error|critical|fatal)",
          }
-    ]] + copy.deepcopy(testing_config_options)
+    ]] + copy.deepcopy(testing_config_options) + \
+        copy.deepcopy(code_coverage_config_options)
 
     app_name = None
 
     def __init__(self, require_config_file=False):
         super(AndroidEmulatorTest, self).__init__(
             config_options=self.config_options,
             all_actions=['clobber',
                          'setup-avds',
@@ -482,16 +486,20 @@ class AndroidEmulatorTest(TestingMixin, 
         try_options, try_tests = self.try_args(self.test_suite)
         cmd.extend(try_options)
         if not self.verify_enabled and not self.per_test_coverage:
             cmd.extend(self.query_tests_args(
                 self.config["suite_definitions"][self.test_suite].get("tests"),
                 None,
                 try_tests))
 
+        if self.java_code_coverage_enabled:
+            cmd.extend(['--enable-coverage',
+                        '--coverage-output-path', self.java_coverage_output_path])
+
         return cmd
 
     def _get_repo_url(self, path):
         """
            Return a url for a file (typically a tooltool manifest) in this hg repo
            and using this revision (or mozilla-central/default if repo/rev cannot
            be determined).