Bug 1541147 - [Coverity] Create a try job that performs coverity static-analysis for patches. r=bastien
authorAndi-Bogdan Postelnicu <bpostelnicu@mozilla.com>
Mon, 08 Apr 2019 10:42:37 +0000
changeset 468339 674a4d255565bcee8e1fc30d6d64d353ae40b737
parent 468338 471b2063c817d3eb1dddf334963200e293953cdb
child 468347 85b7c2d2a42e1d73e7f4dab8012b5d2323d37ab9
push id82521
push userbpostelnicu@mozilla.com
push dateMon, 08 Apr 2019 10:44:17 +0000
treeherderautoland@674a4d255565 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbastien
bugs1541147
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1541147 - [Coverity] Create a try job that performs coverity static-analysis for patches. r=bastien Differential Revision: https://phabricator.services.mozilla.com/D26145
python/mozbuild/mozbuild/mach_commands.py
taskcluster/ci/config.yml
taskcluster/ci/source-test/coverity.yml
taskcluster/ci/source-test/kind.yml
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1,26 +1,28 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import hashlib
+import io
 import itertools
 import json
 import logging
 import ntpath
 import operator
 import os
 import re
 import shutil
 import subprocess
 import sys
+import tarfile
 import tempfile
 import xml.etree.ElementTree as ET
 import yaml
 
 from collections import OrderedDict
 
 import mozpack.path as mozpath
 
@@ -1763,16 +1765,277 @@ class StaticAnalysis(MachCommandBase):
         if rc != 0:
             return rc
         # if we are building firefox for android it might be nice to
         # also analyze the java code base
         if self.substs['MOZ_BUILD_APP'] == 'mobile/android':
             rc = self.check_java(source, jobs, strip, verbose, skip_export=True)
         return rc
 
+    @StaticAnalysisSubCommand('static-analysis', 'check-coverity',
+                              'Run coverity static-analysis tool on the given files. '
+                              'Can only be run by automation! '
+                              'It\'s result is stored as an json file on the artifacts server.')
+    @CommandArgument('source', nargs='*', default=['.*'],
+                     help='Source files to be analyzed by Coverity Static Analysis Tool. '
+                          'This is ran only in automation.')
+    @CommandArgument('--output', '-o', default=None,
+                     help='Write coverity output translated to json output in a file')
+    @CommandArgument('--coverity_output_path', '-co', default=None,
+                     help='Path where to write coverity results as cov-results.json. '
+                     'If no path is specified the default path from the coverity working directory, '
+                     '~./mozbuild/coverity is used.')
+    @CommandArgument('--outgoing', default=False, action='store_true',
+                     help='Run coverity on outgoing files from mercurial or git repository')
+    def check_coverity(self, source=None, output=None, coverity_output_path=None, outgoing=False, verbose=False):
+        self._set_log_level(verbose)
+        self.log_manager.enable_all_structured_loggers()
+
+        if 'MOZ_AUTOMATION' not in os.environ:
+            self.log(logging.INFO, 'static-analysis', {}, 'Coverity based static-analysis cannot be ran outside automation.')
+            return
+
+        # Use outgoing files instead of source files
+        if outgoing:
+            repo = get_repository_object(self.topsrcdir)
+            files = repo.get_outgoing_files()
+            source = map(os.path.abspath, files)
+
+        rc = self._build_compile_db(verbose=verbose)
+        rc = rc or self._build_export(jobs=2, verbose=verbose)
+
+        if rc != 0:
+            return rc
+
+        commands_list = self.get_files_with_commands(source)
+
+        if len(commands_list) == 0:
+            self.log(logging.INFO, 'static-analysis', {}, 'There are no files that need to be analyzed.')
+            return 0
+
+        rc = self.setup_coverity()
+        if rc != 0:
+            return rc
+
+        # First run cov-run-desktop --setup in order to setup the analysis env
+        cmd = [self.cov_run_desktop, '--setup']
+        self.log(logging.INFO, 'static-analysis', {}, 'Running {} --setup'.format(self.cov_run_desktop))
+
+        rc = self.run_process(args=cmd, cwd=self.cov_path, pass_thru=True)
+
+        if rc != 0:
+            self.log(logging.ERROR, 'static-analysis', {}, 'Running {} --setup failed!'.format(self.cov_run_desktop))
+            return rc
+
+        # Run cov-configure for clang
+        cmd = [self.cov_configure, '--clang']
+        self.log(logging.INFO, 'static-analysis', {}, 'Running {} --clang'.format(self.cov_configure))
+
+        rc = self.run_process(args=cmd, cwd=self.cov_path, pass_thru=True)
+
+        if rc != 0:
+            self.log(logging.ERROR, 'static-analysis', {}, 'Running {} --clang failed!'.format(self.cov_configure))
+            return rc
+
+        # For each element in commands_list run `cov-translate`
+        for element in commands_list:
+            cmd = [self.cov_translate, '--dir', self.cov_idir_path] + element['command'].split(' ')
+            self.log(logging.INFO, 'static-analysis', {}, 'Running Coverity Tranlate for {}'.format(cmd))
+            rc = self.run_process(args=cmd, cwd=element['directory'], pass_thru=True)
+            if rc != 0:
+                self.log(logging.ERROR, 'static-analysis', {}, 'Running Coverity Tranlate failed for {}'.format(cmd))
+                return cmd
+
+        if coverity_output_path is None:
+            cov_result = mozpath.join(self.cov_state_path, 'cov-results.json')
+        else:
+            cov_result = mozpath.join(coverity_output_path, 'cov-results.json')
+
+        # Once the capture is performed we need to do the actual Coverity Desktop analysis
+        cmd = [self.cov_run_desktop, '--json-output-v6', cov_result, '--strip-path', self.topsrcdir]
+        cmd += [element['file'] for element in commands_list]
+        self.log(logging.INFO, 'static-analysis', {}, 'Running Coverity Analysis for {}'.format(cmd))
+        rc = self.run_process(cmd, cwd=self.cov_state_path, pass_thru=True)
+        if rc != 0:
+            self.log(logging.ERROR, 'static-analysis', {}, 'Coverity Analysis failed!')
+
+        if output is not None:
+            self.dump_cov_artifact(cov_result, output)
+
+    def dump_cov_artifact(self, cov_results, output):
+        # Parse Coverity json into structured issues
+        with open(cov_results) as f:
+            result = json.load(f)
+
+            # Parse the issues to a standard json format
+            issues_dict = {'files': {}}
+
+            files_list = issues_dict['files']
+
+            def build_element(issue):
+                # We look only for main event
+                event_path = next((event for event in issue['events'] if event['main'] is True), None)
+
+                dict_issue = {
+                    'line': issue['mainEventLineNumber'],
+                    'flag': issue['checkerName'],
+                    'message': event_path['eventDescription'],
+                    'extra': []
+                }
+
+                # Embed all events into extra message
+                for event in issue['events']:
+                    dict_issue['extra'].append({'file_path': event['strippedFilePathname'],
+                                                'line_number': event['lineNumber'],
+                                                'path_type': event['eventTag'],
+                                                'description': event['eventDescription']})
+
+                return dict_issue
+
+            for issue in result['issues']:
+                path = issue['strippedMainEventFilePathname'].strip('/')
+                if path in files_list:
+                    files_list[path]['warnings'].append(build_element(issue))
+                else:
+                    files_list[path] = {'warnings': [build_element(issue)]}
+
+            with open(output, 'w') as f:
+                json.dump(issues_dict, f)
+
+    def get_coverity_secrets(self):
+        from taskgraph.util.taskcluster import get_root_url
+
+        secret_name = 'project/relman/coverity'
+        secrets_url = '{}/secrets/v1/secret/{}'.format(get_root_url(True), secret_name)
+
+        self.log(logging.INFO, 'static-analysis', {}, 'Using symbol upload token from the secrets service: "{}"'.format(secrets_url))
+
+        import requests
+        res = requests.get(secrets_url)
+        res.raise_for_status()
+        secret = res.json()
+        cov_config = secret['secret'] if 'secret' in secret else None
+
+        if cov_config is None:
+            self.log(logging.ERROR, 'static-analysis', {}, 'Ill formatted secret for Coverity. Aborting analysis.')
+            return 1
+
+        self.cov_analysis_url = cov_config.get('package_url')
+        self.cov_package_name = cov_config.get('package_name')
+        self.cov_url = cov_config.get('server_url')
+        self.cov_auth = cov_config.get('auth_key')
+        self.cov_package_ver = cov_config.get('package_ver')
+        self.cov_full_stack = cov_config.get('full_stack', False)
+
+        return 0
+
+    def download_coverity(self):
+        if self.cov_url is None or self.cov_analysis_url is None or self.cov_auth is None:
+            self.log(logging.ERROR, 'static-analysis', {}, 'Missing Coverity secret on try job!')
+            return 1
+
+        COVERITY_CONFIG = '''
+        {
+            "type": "Coverity configuration",
+            "format_version": 1,
+            "settings": {
+            "server": {
+                "host": "%s",
+                "ssl" : true,
+                "on_new_cert" : "trust",
+                "auth_key_file": "%s"
+            },
+            "stream": "Firefox",
+            "cov_run_desktop": {
+                "build_cmd": [],
+                "clean_cmd": []
+            }
+            }
+        }
+        '''
+        # Generate the coverity.conf and auth files
+        cov_auth_path = mozpath.join(self.cov_state_path, 'auth')
+        cov_setup_path = mozpath.join(self.cov_state_path, 'coverity.conf')
+        cov_conf = COVERITY_CONFIG % (self.cov_url, cov_auth_path)
+
+        def download(artifact_url, target):
+            import requests
+            resp = requests.get(artifact_url, verify=False, stream=True)
+            resp.raise_for_status()
+
+            # Extract archive into destination
+            with tarfile.open(fileobj=io.BytesIO(resp.content)) as tar:
+                tar.extractall(target)
+
+        download(self.cov_analysis_url, self.cov_state_path)
+
+        with open(cov_auth_path, 'w') as f:
+            f.write(self.cov_auth)
+
+        # Modify it's permission to 600
+        os.chmod(cov_auth_path, 0o600)
+
+        with open(cov_setup_path, 'a') as f:
+            f.write(cov_conf)
+
+    def setup_coverity(self, force_download=True):
+        rc, config, _ = self._get_config_environment()
+        rc = rc or self.get_coverity_secrets()
+
+        if rc != 0:
+            return rc
+
+        # Create a directory in mozbuild where we setup coverity
+        self.cov_state_path = mozpath.join(self._mach_context.state_dir, "coverity")
+
+        if force_download is True and os.path.exists(self.cov_state_path):
+            shutil.rmtree(self.cov_state_path)
+
+        os.mkdir(self.cov_state_path)
+
+        # Download everything that we need for Coverity from out private instance
+        self.download_coverity()
+
+        self.cov_path = mozpath.join(self.cov_state_path, self.cov_package_name)
+        self.cov_run_desktop = mozpath.join(self.cov_path, 'bin', 'cov-run-desktop')
+        self.cov_translate = mozpath.join(self.cov_path, 'bin', 'cov-translate')
+        self.cov_configure = mozpath.join(self.cov_path, 'bin', 'cov-configure')
+        self.cov_work_path = mozpath.join(self.cov_state_path, 'data-coverity')
+        self.cov_idir_path = mozpath.join(self.cov_work_path, self.cov_package_ver, 'idir')
+
+        if not os.path.exists(self.cov_path):
+            self.log(logging.ERROR, 'static-analysis', {}, 'Missing Coverity in {}'.format(self.cov_path))
+            return 1
+
+        return 0
+
+    def get_files_with_commands(self, source):
+        '''
+        Returns an array of dictionaries having file_path with build command
+        '''
+
+        compile_db = json.load(open(self._compile_db, 'r'))
+
+        commands_list = []
+
+        for f in source:
+            # It must be a C/C++ file
+            _, ext = os.path.splitext(f)
+
+            if ext.lower() not in self._format_include_extensions:
+                self.log(logging.INFO, 'static-analysis', {}, 'Skipping {}'.format(f))
+                continue
+            file_with_abspath = os.path.join(self.topsrcdir, f)
+            for f in compile_db:
+                # Found for a file that we are looking
+                if file_with_abspath == f['file']:
+                    commands_list.append(f)
+
+        return commands_list
+
     @StaticAnalysisSubCommand('static-analysis', 'check-java',
                               'Run infer on the java codebase.')
     @CommandArgument('source', nargs='*', default=['mobile'],
                      help='Source files to be analyzed. '
                           'Can be omitted, in which case the entire code base '
                           'is analyzed.  The source argument is ignored if '
                           'there is anything fed through stdin, in which case '
                           'the analysis is only performed on the files changed '
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -91,16 +91,17 @@ treeherder:
         'Snap': 'Snap image generation'
         'langpack': 'Langpack sigatures and uploads'
         'TPS': 'Sync tests'
         'UV': 'Update verify'
         'pipfu': 'pipfile update'
         'WR': 'WebRender standalone'
         'Gd': 'Geckodriver'
         'clang': 'Clang Tidy & Format'
+        'coverity': 'Coverity Static Analysis'
 
 index:
     products:
         - 'firefox'
         - 'fennec'
         - 'mobile'
         - 'static-analysis'
         - 'devedition'
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/source-test/coverity.yml
@@ -0,0 +1,52 @@
+job-defaults:
+    platform: linux64/opt
+    attributes:
+        code-review: true
+    worker-type:
+        by-platform:
+            linux64.*: aws-provisioner-v1/gecko-t-linux-xlarge
+    worker:
+        docker-image: {in-tree: debian7-amd64-build}
+        max-run-time: 5400
+    treeherder:
+        kind: other
+        tier: 2
+    run:
+        using: run-task
+        tooltool-downloads: public
+    toolchains:
+        - linux64-clang
+        - linux64-rust
+        - linux64-cbindgen
+        - linux64-nasm
+        - linux64-node
+    when:
+        # Extension list from https://hg.mozilla.org/mozilla-central/file/tip/python/mozbuild/mozbuild/mach_commands.py#l1664
+        files-changed:
+            - '**/*.c'
+            - '**/*.cpp'
+            - '**/*.cc'
+            - '**/*.cxx'
+            - '**/*.m'
+            - '**/*.mm'
+            - '**/*.h'
+            - '**/*.hh'
+            - '**/*.hpp'
+            - '**/*.hxx'
+
+coverity:
+    description: Run static-analysis (Coverity) on C/C++ patches
+    treeherder:
+        symbol: coverity(cvsa)
+    run:
+        command: >-
+            source $HOME/checkouts/gecko/taskcluster/scripts/misc/source-test-clang-setup.sh &&
+            cd $HOME/checkouts/gecko &&
+            ./mach --log-no-times static-analysis check-coverity --outgoing --output $HOME/coverity.json
+    scopes:
+        - secrets:get:project/relman/coverity
+    worker:
+        artifacts:
+            - type: file
+              name: public/code-review/coverity.json
+              path: /builds/worker/coverity.json
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -12,16 +12,17 @@ transforms:
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - fetch
    - toolchain
 
 jobs-from:
    - clang.yml
+   - coverity.yml
    - cram.yml
    - doc.yml
    - file-metadata.yml
    - jsshell.yml
    - mozlint.yml
    - node.yml
    - python.yml
    - webidl.yml