author | Robert Bartlensky <rbartlensky@mozilla.com> |
Thu, 26 Jul 2018 14:45:44 +0100 | |
changeset 430537 | 9fb094b1e5f6d7aa6e3976a842fbe6a788f19327 |
parent 430536 | 28d1ff679d50a7840db50c23b165efccec9e7138 |
child 430538 | 1f426c672aabc1b58a4aceeaca499309afd9b66f |
push id | 106196 |
push user | ncsoregi@mozilla.com |
push date | Wed, 08 Aug 2018 11:11:21 +0000 |
treeherder | mozilla-inbound@9fb094b1e5f6 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | gps |
bugs | 1473278 |
milestone | 63.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
.inferconfig | file | annotate | diff | comparison | revisions | |
python/mozbuild/mozbuild/mach_commands.py | file | annotate | diff | comparison | revisions | |
tools/infer/config.yaml | file | annotate | diff | comparison | revisions |
deleted file mode 100644 --- a/.inferconfig +++ /dev/null @@ -1,7 +0,0 @@ -{ - "infer-blacklist-path-regex": [ - // This is full of issues, and is a dependency we need to discard - // sooner rather than later anyway: - "mobile/android/thirdparty/ch/boye/httpclientandroidlib" - ] -} \ No newline at end of file
--- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -1625,17 +1625,17 @@ class StaticAnalysisMonitor(object): return (warning, True) @CommandProvider class StaticAnalysis(MachCommandBase): """Utilities for running C++ static analysis checks and format.""" # List of file extension to consider (should start with dot) - _format_include_extensions = ('.cpp', '.c', '.h') + _format_include_extensions = ('.cpp', '.c', '.h', '.java') # File contaning all paths to exclude from formatting _format_ignore_file = '.clang-format-ignore' @Command('static-analysis', category='testing', description='Run C++ static analysis checks') def static_analysis(self): # If not arguments are provided, just print a help message. mach = Mach(os.getcwd()) @@ -1707,17 +1707,151 @@ class StaticAnalysis(MachCommandBase): footer = StaticAnalysisFooter(self.log_manager.terminal, monitor) with StaticAnalysisOutputManager(self.log_manager, monitor, footer) as output: rc = self.run_process(args=args, line_handler=output.on_line, cwd=cwd) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_db)}, '{count} warnings present.') + if rc != 0: return rc + # if we are building firefox for android it might be nice to + # also analyze the java code base + if self.substs['MOZ_BUILD_APP'] == 'mobile/android': + rc = self.check_java(source, jobs, strip, verbose, skip_export=True) + return rc + + @StaticAnalysisSubCommand('static-analysis', 'check-java', + 'Run infer on the java codebase.') + @CommandArgument('source', nargs='*', default=['mobile'], + help='Source files to be analyzed. ' + 'Can be omitted, in which case the entire code base ' + 'is analyzed. The source argument is ignored if ' + 'there is anything fed through stdin, in which case ' + 'the analysis is only performed on the files changed ' + 'in the patch streamed through stdin. This is called ' + 'the diff mode.') + @CommandArgument('--checks', '-c', default=[], metavar='checks', nargs='*', + help='Static analysis checks to enable.') + @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int, + help='Number of concurrent jobs to run.' + ' Default is the number of CPUs.') + @CommandArgument('--task', '-t', type=str, + default='compileLocalWithGeckoBinariesNoMinApiDebugSources', + help='Which gradle tasks to use to compile the java codebase.') + def check_java(self, source=['mobile'], jobs=2, strip=1, verbose=False, checks=[], + task='compileLocalWithGeckoBinariesNoMinApiDebugSources', + skip_export=False): + self._set_log_level(verbose) + self.log_manager.enable_all_structured_loggers() + if self.substs['MOZ_BUILD_APP'] != 'mobile/android': + self.log(logging.WARNING, 'static-analysis', {}, + 'Cannot check java source code unless you are building for android!') + return 1 + # if source contains the whole mobile folder, then we just have to + # analyze everything + check_all = any(i.rstrip(os.sep).split(os.sep)[-1] == 'mobile' for i in source) + # gather all java sources from the source variable + java_sources = [] + if not check_all: + java_sources = self._get_java_files(source) + if not java_sources: + return 0 + if not skip_export: + rc = self._build_export(jobs=jobs, verbose=verbose) + if rc != 0: + return rc + rc = self._get_infer(verbose=verbose) + if rc != 0: + self.log(logging.WARNING, 'static-analysis', {}, + 'This command is only available for linux64!') + return rc + # which checkers to use, and which folders to exclude + all_checkers, third_party_path = self._get_infer_config() + checkers, excludes = self._get_infer_args( + checks=checks or all_checkers, + third_party_path=third_party_path + ) + gradlew = mozpath.join(self.topsrcdir, 'gradlew') + # infer capture command + capture_cmd = [self._infer_path, 'capture'] + excludes + \ + ['--', gradlew, task] + tmp_file, args = self._get_infer_source_args(java_sources) + # infer analyze command + analysis_cmd = [self._infer_path, 'analyze', '--keep-going'] + \ + checkers + args + # capture, then analyze the sources + for args in [[gradlew, 'clean'], capture_cmd, analysis_cmd]: + rc = self.run_process(args=args, cwd=self.topsrcdir, + pass_thru=True) + # if a command fails, break and close the tmp file before returning + if rc != 0: + break + if tmp_file: + tmp_file.close() + return rc + + def _get_java_files(self, sources): + java_sources = [] + for i in sources: + f = mozpath.join(self.topsrcdir, i) + if os.path.isdir(f): + for root, dirs, files in os.walk(f): + dirs.sort() + for file in sorted(files): + if file.endswith('.java'): + java_sources.append(mozpath.join(root, file)) + elif f.endswith('.java'): + java_sources.append(f) + return java_sources + + def _get_infer_source_args(self, sources): + '''Return the arguments to only analyze <sources>''' + if not sources: + return (None, []) + # create a temporary file in which we place all sources + # this is used by the analysis command to only analyze certain files + f = tempfile.NamedTemporaryFile() + for source in sources: + f.write(source+'\n') + f.flush() + return (f, ['--changed-files-index', f.name]) + + def _get_infer_config(self): + '''Load the infer config file.''' + import yaml + checkers = [] + tp_path = '' + with open(mozpath.join(self.topsrcdir, 'tools', + 'infer', 'config.yaml')) as f: + try: + config = yaml.safe_load(f) + for item in config['infer_checkers']: + if item['publish']: + checkers.append(item['name']) + tp_path = mozpath.join(self.topsrcdir, config['third_party']) + except Exception as e: + print('Looks like config.yaml is not valid, so we are unable ' + 'to determine default checkers, and which folder to ' + 'exclude, using defaults provided by infer') + return checkers, tp_path + + def _get_infer_args(self, checks, third_party_path): + '''Return the arguments which include the checkers <checks>, and + excludes all folder in <third_party_path>.''' + checkers = ['-a', 'checkers'] + excludes = [] + for checker in checks: + checkers.append('--' + checker) + with open(third_party_path) as f: + for line in f: + excludes.append('--skip-analysis-in-path') + excludes.append(line.strip('\n')) + return checkers, excludes def _get_clang_tidy_command(self, checks, header_filter, sources, jobs, fix): if checks == '-*': checks = self._get_checks() common_args = ['-clang-tidy-binary', self._clang_tidy_path, '-clang-apply-replacements-binary', self._clang_apply_replacements, @@ -1741,17 +1875,18 @@ class StaticAnalysis(MachCommandBase): @StaticAnalysisSubCommand('static-analysis', 'autotest', 'Run the auto-test suite in order to determine that' ' the analysis did not regress.') @CommandArgument('--dump-results', '-d', default=False, action='store_true', help='Generate the baseline for the regression test. Based on' ' this baseline we will test future results.') @CommandArgument('--intree-tool', '-i', default=False, action='store_true', help='Use a pre-aquired in-tree clang-tidy package.') - @CommandArgument('checker_names', nargs='*', default=[], help='Checkers that are going to be auto-tested.') + @CommandArgument('checker_names', nargs='*', default=[], + help='Checkers that are going to be auto-tested.') def autotest(self, verbose=False, dump_results=False, intree_tool=False, checker_names=[]): # If 'dump_results' is True than we just want to generate the issues files for each # checker in particulat and thus 'force_download' becomes 'False' since we want to # do this on a local trusted clang-tidy package. self._set_log_level(verbose) self._dump_results = dump_results force_download = True @@ -1788,29 +1923,31 @@ class StaticAnalysis(MachCommandBase): "clang", "clang-format-diff.py") # Ensure that clang-tidy is present rc = not os.path.exists(self._clang_tidy_path) else: rc = self._get_clang_tools(force=force_download, verbose=verbose) if rc != 0: - self.log(logging.ERROR, 'ERROR: static-analysis', {}, 'clang-tidy unable to locate package.') + self.log(logging.ERROR, 'ERROR: static-analysis', {}, + 'clang-tidy unable to locate package.') return self.TOOLS_FAILED_DOWNLOAD self._clang_tidy_base_path = mozpath.join(self.topsrcdir, "tools", "clang-tidy") # For each checker run it f = open(mozpath.join(self._clang_tidy_base_path, "config.yaml")) import yaml config = yaml.safe_load(f) platform, _ = self.platform if platform not in config['platforms']: - self.log(logging.ERROR, 'static-analysis', {},"RUNNING: clang-tidy autotest for platform {} not supported.".format(platform)) + self.log(logging.ERROR, 'static-analysis', {}, + "RUNNING: clang-tidy autotest for platform {} not supported.".format(platform)) return TOOLS_UNSUPORTED_PLATFORM import concurrent.futures import multiprocessing import shutil max_workers = multiprocessing.cpu_count() @@ -1870,50 +2007,65 @@ class StaticAnalysis(MachCommandBase): json.dump(compile_commands, file_handler) file_handler.flush() return directory @StaticAnalysisSubCommand('static-analysis', 'install', 'Install the static analysis helper tool') @CommandArgument('source', nargs='?', type=str, - help='Where to fetch a local archive containing the static-analysis and format helper tool.' - 'It will be installed in ~/.mozbuild/clang-tools/.' - 'Can be omitted, in which case the latest clang-tools ' - ' helper for the platform would be automatically ' - 'detected and installed.') + help='Where to fetch a local archive containing the static-analysis and ' + 'format helper tool.' + 'It will be installed in ~/.mozbuild/clang-tools and ~/.mozbuild/infer.' + 'Can be omitted, in which case the latest clang-tools and infer ' + 'helper for the platform would be automatically detected and installed.') @CommandArgument('--skip-cache', action='store_true', help='Skip all local caches to force re-fetching the helper tool.', default=False) def install(self, source=None, skip_cache=False, verbose=False): self._set_log_level(verbose) rc = self._get_clang_tools(force=True, skip_cache=skip_cache, source=source, verbose=verbose) + if rc == 0: + # XXX ignore the return code because if it fails or not, infer is + # not mandatory, but clang-tidy is + self._get_infer(force=True, skip_cache=skip_cache, verbose=verbose) return rc @StaticAnalysisSubCommand('static-analysis', 'clear-cache', 'Delete local helpers and reset static analysis helper tool cache') def clear_cache(self, verbose=False): self._set_log_level(verbose) rc = self._get_clang_tools(force=True, download_if_needed=True, skip_cache=True, verbose=verbose) + if rc == 0: + self._get_infer(force=True, download_if_needed=True, skip_cache=True, + verbose=verbose) if rc != 0: return rc - - self._artifact_manager.artifact_clear_cache() + return self._artifact_manager.artifact_clear_cache() @StaticAnalysisSubCommand('static-analysis', 'print-checks', 'Print a list of the static analysis checks performed by default') def print_checks(self, verbose=False): self._set_log_level(verbose) rc = self._get_clang_tools(verbose=verbose) + if rc == 0: + rc = self._get_infer(verbose=verbose) if rc != 0: return rc args = [self._clang_tidy_path, '-list-checks', '-checks=%s' % self._get_checks()] - return self._run_command_in_objdir(args=args, pass_thru=True) + rc = self._run_command_in_objdir(args=args, pass_thru=True) + if rc != 0: + return rc + checkers, _ = self._get_infer_config() + print('Infer checks:') + for checker in checkers: + print(' '*4 + checker) + return 0 @Command('clang-format', category='misc', description='Run clang-format on current changes') @CommandArgument('--show', '-s', action='store_true', default=False, help='Show diff output on instead of applying changes') @CommandArgument('--path', '-p', nargs='+', default=None, help='Specify the path(s) to reformat') def clang_format(self, show, path, verbose=False): # Run clang-format or clang-format-diff on the local changes @@ -1934,17 +2086,18 @@ class StaticAnalysis(MachCommandBase): return self._run_clang_format_path(self._clang_format_path, show, path) def _verify_checker(self, item): check = item['name'] test_file_path = mozpath.join(self._clang_tidy_base_path, "test", check) test_file_path_cpp = test_file_path + '.cpp' test_file_path_json = test_file_path + '.json' - self.log(logging.INFO, 'static-analysis', {},"RUNNING: clang-tidy checker {}.".format(check)) + self.log(logging.INFO, 'static-analysis', {}, + "RUNNING: clang-tidy checker {}.".format(check)) # Verify if this checker actually exists if not check in self._clang_tidy_checks: self.log(logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy checker {} doesn't exist in this clang-tidy version.".format(check)) return self.TOOLS_CHECKER_NOT_FOUND # Verify if the test file exists for this checker if not os.path.exists(test_file_path_cpp): @@ -2226,16 +2379,57 @@ class StaticAnalysis(MachCommandBase): continue # empty or comment magics = ['exclude'] if pattern.startswith('^'): magics += ['top'] pattern = pattern[1:] args += [':({0}){1}'.format(','.join(magics), pattern)] return args + def _get_infer(self, force=False, skip_cache=False, + download_if_needed=True, verbose=False): + rc, config, _ = self._get_config_environment() + if rc != 0: + return rc + infer_path = mozpath.join(self._mach_context.state_dir, 'infer') + self._infer_path = mozpath.join(infer_path, 'infer', 'bin', + 'infer' + + config.substs.get('BIN_SUFFIX', '')) + if os.path.exists(self._infer_path) and not force: + return 0 + else: + if os.path.isdir(infer_path) and download_if_needed: + # The directory exists, perhaps it's corrupted? Delete it + # and start from scratch. + import shutil + shutil.rmtree(infer_path) + return self._get_infer(force=force, skip_cache=skip_cache, + verbose=verbose, + download_if_needed=download_if_needed) + os.mkdir(infer_path) + self._artifact_manager = PackageFrontend(self._mach_context) + if not download_if_needed: + return 0 + job, _ = self.platform + if job != 'linux64': + return -1 + else: + job += '-infer' + # We want to unpack data in the infer mozbuild folder + currentWorkingDir = os.getcwd() + os.chdir(infer_path) + rc = self._artifact_manager.artifact_toolchain(verbose=verbose, + skip_cache=skip_cache, + from_build=[job], + no_unpack=False, + retry=0) + # Change back the cwd + os.chdir(currentWorkingDir) + return rc + def _run_clang_format_diff(self, clang_format_diff, clang_format, show): # Run clang-format on the diff # Note that this will potentially miss a lot things from subprocess import Popen, PIPE, check_output, CalledProcessError diff_process = Popen(self._get_clang_format_diff_command(), stdout=PIPE) args = [sys.executable, clang_format_diff, "-p1", "-binary=%s" % clang_format] @@ -2527,9 +2721,8 @@ class Analyze(MachCommandBase): if os.path.isfile(path): g = Graph(path) g.file_summaries(files) g.close() else: res = 'Please make sure you have a local tup db *or* specify the location with --path.' print ('Could not find a valid tup db in ' + path, res, sep='\n') return 1 -
new file mode 100644 --- /dev/null +++ b/tools/infer/config.yaml @@ -0,0 +1,23 @@ +--- +target: obj-x86_64-pc-linux-gnu +# It is used by 'mach static-analysis' and 'mozreview static-analysis bot' +# in order to have consistency across the used checkers. +platforms: + - linux64 +infer_checkers: + - name: check-nullable + publish: !!bool yes + - name: eradicate + publish: !!bool no + - name: quandary + publish: !!bool yes + - name: starvation + publish: !!bool yes + - name: litho + publish: !!bool yes + - name: racerd + publish: !!bool yes + - name: liveness + publish: !!bool yes +# Third party files from mozilla-central +third_party: tools/rewriting/ThirdPartyPaths.txt