Bug 1473278: Add infer to java-check, install, clear-cache, and print-checks subcommands. r=gps
authorRobert Bartlensky <rbartlensky@mozilla.com>
Thu, 26 Jul 2018 14:45:44 +0100
changeset 430537 9fb094b1e5f6d7aa6e3976a842fbe6a788f19327
parent 430536 28d1ff679d50a7840db50c23b165efccec9e7138
child 430538 1f426c672aabc1b58a4aceeaca499309afd9b66f
push id106196
push userncsoregi@mozilla.com
push dateWed, 08 Aug 2018 11:11:21 +0000
treeherdermozilla-inbound@9fb094b1e5f6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1473278
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1473278: Add infer to java-check, install, clear-cache, and print-checks subcommands. r=gps MozReview-Commit-ID: 5ngZu6lh1wU
.inferconfig
python/mozbuild/mozbuild/mach_commands.py
tools/infer/config.yaml
deleted file mode 100644
--- a/.inferconfig
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-    "infer-blacklist-path-regex": [
-        // This is full of issues, and is a dependency we need to discard
-        // sooner rather than later anyway:
-        "mobile/android/thirdparty/ch/boye/httpclientandroidlib"
-    ]
-}
\ No newline at end of file
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1625,17 +1625,17 @@ class StaticAnalysisMonitor(object):
         return (warning, True)
 
 
 @CommandProvider
 class StaticAnalysis(MachCommandBase):
     """Utilities for running C++ static analysis checks and format."""
 
     # List of file extension to consider (should start with dot)
-    _format_include_extensions = ('.cpp', '.c', '.h')
+    _format_include_extensions = ('.cpp', '.c', '.h', '.java')
     # File contaning all paths to exclude from formatting
     _format_ignore_file = '.clang-format-ignore'
 
     @Command('static-analysis', category='testing',
              description='Run C++ static analysis checks')
     def static_analysis(self):
         # If not arguments are provided, just print a help message.
         mach = Mach(os.getcwd())
@@ -1707,17 +1707,151 @@ class StaticAnalysis(MachCommandBase):
 
         footer = StaticAnalysisFooter(self.log_manager.terminal, monitor)
         with StaticAnalysisOutputManager(self.log_manager, monitor, footer) as output:
             rc = self.run_process(args=args, line_handler=output.on_line, cwd=cwd)
 
             self.log(logging.WARNING, 'warning_summary',
                      {'count': len(monitor.warnings_db)},
                      '{count} warnings present.')
+        if rc != 0:
             return rc
+        # if we are building firefox for android it might be nice to
+        # also analyze the java code base
+        if self.substs['MOZ_BUILD_APP'] == 'mobile/android':
+            rc = self.check_java(source, jobs, strip, verbose, skip_export=True)
+        return rc
+
+    @StaticAnalysisSubCommand('static-analysis', 'check-java',
+                              'Run infer on the java codebase.')
+    @CommandArgument('source', nargs='*', default=['mobile'],
+                     help='Source files to be analyzed. '
+                          'Can be omitted, in which case the entire code base '
+                          'is analyzed.  The source argument is ignored if '
+                          'there is anything fed through stdin, in which case '
+                          'the analysis is only performed on the files changed '
+                          'in the patch streamed through stdin.  This is called '
+                          'the diff mode.')
+    @CommandArgument('--checks', '-c', default=[], metavar='checks', nargs='*',
+                     help='Static analysis checks to enable.')
+    @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,
+                     help='Number of concurrent jobs to run.'
+                     ' Default is the number of CPUs.')
+    @CommandArgument('--task', '-t', type=str,
+                     default='compileLocalWithGeckoBinariesNoMinApiDebugSources',
+                     help='Which gradle tasks to use to compile the java codebase.')
+    def check_java(self, source=['mobile'], jobs=2, strip=1, verbose=False, checks=[],
+                   task='compileLocalWithGeckoBinariesNoMinApiDebugSources',
+                   skip_export=False):
+        self._set_log_level(verbose)
+        self.log_manager.enable_all_structured_loggers()
+        if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
+            self.log(logging.WARNING, 'static-analysis', {},
+                     'Cannot check java source code unless you are building for android!')
+            return 1
+        # if source contains the whole mobile folder, then we just have to
+        # analyze everything
+        check_all = any(i.rstrip(os.sep).split(os.sep)[-1] == 'mobile' for i in source)
+        # gather all java sources from the source variable
+        java_sources = []
+        if not check_all:
+            java_sources = self._get_java_files(source)
+            if not java_sources:
+                return 0
+        if not skip_export:
+            rc = self._build_export(jobs=jobs, verbose=verbose)
+            if rc != 0:
+                return rc
+        rc = self._get_infer(verbose=verbose)
+        if rc != 0:
+            self.log(logging.WARNING, 'static-analysis', {},
+                     'This command is only available for linux64!')
+            return rc
+        # which checkers to use, and which folders to exclude
+        all_checkers, third_party_path = self._get_infer_config()
+        checkers, excludes = self._get_infer_args(
+            checks=checks or all_checkers,
+            third_party_path=third_party_path
+        )
+        gradlew = mozpath.join(self.topsrcdir, 'gradlew')
+        # infer capture command
+        capture_cmd = [self._infer_path, 'capture'] + excludes + \
+            ['--', gradlew, task]
+        tmp_file, args = self._get_infer_source_args(java_sources)
+        # infer analyze command
+        analysis_cmd = [self._infer_path, 'analyze', '--keep-going'] +  \
+            checkers + args
+        # capture, then analyze the sources
+        for args in [[gradlew, 'clean'], capture_cmd, analysis_cmd]:
+            rc = self.run_process(args=args, cwd=self.topsrcdir,
+                                  pass_thru=True)
+            # if a command fails, break and close the tmp file before returning
+            if rc != 0:
+                break
+        if tmp_file:
+            tmp_file.close()
+        return rc
+
+    def _get_java_files(self, sources):
+        java_sources = []
+        for i in sources:
+            f = mozpath.join(self.topsrcdir, i)
+            if os.path.isdir(f):
+                for root, dirs, files in os.walk(f):
+                    dirs.sort()
+                    for file in sorted(files):
+                        if file.endswith('.java'):
+                            java_sources.append(mozpath.join(root, file))
+            elif f.endswith('.java'):
+                java_sources.append(f)
+        return java_sources
+
+    def _get_infer_source_args(self, sources):
+        '''Return the arguments to only analyze <sources>'''
+        if not sources:
+            return (None, [])
+        # create a temporary file in which we place all sources
+        # this is used by the analysis command to only analyze certain files
+        f = tempfile.NamedTemporaryFile()
+        for source in sources:
+            f.write(source+'\n')
+        f.flush()
+        return (f, ['--changed-files-index', f.name])
+
+    def _get_infer_config(self):
+        '''Load the infer config file.'''
+        import yaml
+        checkers = []
+        tp_path = ''
+        with open(mozpath.join(self.topsrcdir, 'tools',
+                               'infer', 'config.yaml')) as f:
+            try:
+                config = yaml.safe_load(f)
+                for item in config['infer_checkers']:
+                    if item['publish']:
+                        checkers.append(item['name'])
+                tp_path = mozpath.join(self.topsrcdir, config['third_party'])
+            except Exception as e:
+                print('Looks like config.yaml is not valid, so we are unable '
+                      'to determine default checkers, and which folder to '
+                      'exclude, using defaults provided by infer')
+        return checkers, tp_path
+
+    def _get_infer_args(self, checks, third_party_path):
+        '''Return the arguments which include the checkers <checks>, and
+        excludes all folder in <third_party_path>.'''
+        checkers = ['-a', 'checkers']
+        excludes = []
+        for checker in checks:
+            checkers.append('--' + checker)
+        with open(third_party_path) as f:
+            for line in f:
+                excludes.append('--skip-analysis-in-path')
+                excludes.append(line.strip('\n'))
+        return checkers, excludes
 
     def _get_clang_tidy_command(self, checks, header_filter, sources, jobs, fix):
 
         if checks == '-*':
             checks = self._get_checks()
 
         common_args = ['-clang-tidy-binary', self._clang_tidy_path,
                        '-clang-apply-replacements-binary', self._clang_apply_replacements,
@@ -1741,17 +1875,18 @@ class StaticAnalysis(MachCommandBase):
     @StaticAnalysisSubCommand('static-analysis', 'autotest',
                               'Run the auto-test suite in order to determine that'
                               ' the analysis did not regress.')
     @CommandArgument('--dump-results', '-d', default=False, action='store_true',
                      help='Generate the baseline for the regression test. Based on'
                      ' this baseline we will test future results.')
     @CommandArgument('--intree-tool', '-i', default=False, action='store_true',
                      help='Use a pre-aquired in-tree clang-tidy package.')
-    @CommandArgument('checker_names', nargs='*', default=[], help='Checkers that are going to be auto-tested.')
+    @CommandArgument('checker_names', nargs='*', default=[],
+                     help='Checkers that are going to be auto-tested.')
     def autotest(self, verbose=False, dump_results=False, intree_tool=False, checker_names=[]):
         # If 'dump_results' is True than we just want to generate the issues files for each
         # checker in particulat and thus 'force_download' becomes 'False' since we want to
         # do this on a local trusted clang-tidy package.
         self._set_log_level(verbose)
         self._dump_results = dump_results
 
         force_download = True
@@ -1788,29 +1923,31 @@ class StaticAnalysis(MachCommandBase):
                                                    "clang", "clang-format-diff.py")
 
             # Ensure that clang-tidy is present
             rc = not os.path.exists(self._clang_tidy_path)
         else:
             rc = self._get_clang_tools(force=force_download, verbose=verbose)
 
         if rc != 0:
-            self.log(logging.ERROR, 'ERROR: static-analysis', {}, 'clang-tidy unable to locate package.')
+            self.log(logging.ERROR, 'ERROR: static-analysis', {},
+                     'clang-tidy unable to locate package.')
             return self.TOOLS_FAILED_DOWNLOAD
 
         self._clang_tidy_base_path = mozpath.join(self.topsrcdir, "tools", "clang-tidy")
 
         # For each checker run it
         f = open(mozpath.join(self._clang_tidy_base_path, "config.yaml"))
         import yaml
         config = yaml.safe_load(f)
         platform, _ = self.platform
 
         if platform not in config['platforms']:
-            self.log(logging.ERROR, 'static-analysis', {},"RUNNING: clang-tidy autotest for platform {} not supported.".format(platform))
+            self.log(logging.ERROR, 'static-analysis', {},
+                     "RUNNING: clang-tidy autotest for platform {} not supported.".format(platform))
             return TOOLS_UNSUPORTED_PLATFORM
 
         import concurrent.futures
         import multiprocessing
         import shutil
 
         max_workers = multiprocessing.cpu_count()
 
@@ -1870,50 +2007,65 @@ class StaticAnalysis(MachCommandBase):
             json.dump(compile_commands, file_handler)
             file_handler.flush()
 
             return directory
 
     @StaticAnalysisSubCommand('static-analysis', 'install',
                               'Install the static analysis helper tool')
     @CommandArgument('source', nargs='?', type=str,
-                     help='Where to fetch a local archive containing the static-analysis and format helper tool.'
-                          'It will be installed in ~/.mozbuild/clang-tools/.'
-                          'Can be omitted, in which case the latest clang-tools '
-                          ' helper for the platform would be automatically '
-                          'detected and installed.')
+                     help='Where to fetch a local archive containing the static-analysis and '
+                     'format helper tool.'
+                          'It will be installed in ~/.mozbuild/clang-tools and ~/.mozbuild/infer.'
+                          'Can be omitted, in which case the latest clang-tools and infer '
+                          'helper for the platform would be automatically detected and installed.')
     @CommandArgument('--skip-cache', action='store_true',
                      help='Skip all local caches to force re-fetching the helper tool.',
                      default=False)
     def install(self, source=None, skip_cache=False, verbose=False):
         self._set_log_level(verbose)
         rc = self._get_clang_tools(force=True, skip_cache=skip_cache,
                                    source=source, verbose=verbose)
+        if rc == 0:
+            # XXX ignore the return code because if it fails or not, infer is
+            # not mandatory, but clang-tidy is
+            self._get_infer(force=True, skip_cache=skip_cache, verbose=verbose)
         return rc
 
     @StaticAnalysisSubCommand('static-analysis', 'clear-cache',
                               'Delete local helpers and reset static analysis helper tool cache')
     def clear_cache(self, verbose=False):
         self._set_log_level(verbose)
         rc = self._get_clang_tools(force=True, download_if_needed=True, skip_cache=True,
                                    verbose=verbose)
+        if rc == 0:
+            self._get_infer(force=True, download_if_needed=True, skip_cache=True,
+                            verbose=verbose)
         if rc != 0:
             return rc
-
-        self._artifact_manager.artifact_clear_cache()
+        return self._artifact_manager.artifact_clear_cache()
 
     @StaticAnalysisSubCommand('static-analysis', 'print-checks',
                               'Print a list of the static analysis checks performed by default')
     def print_checks(self, verbose=False):
         self._set_log_level(verbose)
         rc = self._get_clang_tools(verbose=verbose)
+        if rc == 0:
+            rc = self._get_infer(verbose=verbose)
         if rc != 0:
             return rc
         args = [self._clang_tidy_path, '-list-checks', '-checks=%s' % self._get_checks()]
-        return self._run_command_in_objdir(args=args, pass_thru=True)
+        rc = self._run_command_in_objdir(args=args, pass_thru=True)
+        if rc != 0:
+            return rc
+        checkers, _ = self._get_infer_config()
+        print('Infer checks:')
+        for checker in checkers:
+            print(' '*4 + checker)
+        return 0
 
     @Command('clang-format',  category='misc', description='Run clang-format on current changes')
     @CommandArgument('--show', '-s', action='store_true', default=False,
                      help='Show diff output on instead of applying changes')
     @CommandArgument('--path', '-p', nargs='+', default=None,
                      help='Specify the path(s) to reformat')
     def clang_format(self, show, path, verbose=False):
         # Run clang-format or clang-format-diff on the local changes
@@ -1934,17 +2086,18 @@ class StaticAnalysis(MachCommandBase):
             return self._run_clang_format_path(self._clang_format_path, show, path)
 
     def _verify_checker(self, item):
         check = item['name']
         test_file_path = mozpath.join(self._clang_tidy_base_path, "test", check)
         test_file_path_cpp = test_file_path + '.cpp'
         test_file_path_json = test_file_path + '.json'
 
-        self.log(logging.INFO, 'static-analysis', {},"RUNNING: clang-tidy checker {}.".format(check))
+        self.log(logging.INFO, 'static-analysis', {},
+                 "RUNNING: clang-tidy checker {}.".format(check))
 
         # Verify if this checker actually exists
         if not check in self._clang_tidy_checks:
             self.log(logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy checker {} doesn't exist in this clang-tidy version.".format(check))
             return self.TOOLS_CHECKER_NOT_FOUND
 
         # Verify if the test file exists for this checker
         if not os.path.exists(test_file_path_cpp):
@@ -2226,16 +2379,57 @@ class StaticAnalysis(MachCommandBase):
                         continue  # empty or comment
                     magics = ['exclude']
                     if pattern.startswith('^'):
                         magics += ['top']
                         pattern = pattern[1:]
                     args += [':({0}){1}'.format(','.join(magics), pattern)]
         return args
 
+    def _get_infer(self, force=False, skip_cache=False,
+                   download_if_needed=True, verbose=False):
+        rc, config, _ = self._get_config_environment()
+        if rc != 0:
+            return rc
+        infer_path = mozpath.join(self._mach_context.state_dir, 'infer')
+        self._infer_path = mozpath.join(infer_path, 'infer', 'bin',
+                                        'infer' +
+                                        config.substs.get('BIN_SUFFIX', ''))
+        if os.path.exists(self._infer_path) and not force:
+            return 0
+        else:
+            if os.path.isdir(infer_path) and download_if_needed:
+                # The directory exists, perhaps it's corrupted?  Delete it
+                # and start from scratch.
+                import shutil
+                shutil.rmtree(infer_path)
+                return self._get_infer(force=force, skip_cache=skip_cache,
+                                       verbose=verbose,
+                                       download_if_needed=download_if_needed)
+            os.mkdir(infer_path)
+            self._artifact_manager = PackageFrontend(self._mach_context)
+            if not download_if_needed:
+                return 0
+            job, _ = self.platform
+            if job != 'linux64':
+                return -1
+            else:
+                job += '-infer'
+            # We want to unpack data in the infer mozbuild folder
+            currentWorkingDir = os.getcwd()
+            os.chdir(infer_path)
+            rc = self._artifact_manager.artifact_toolchain(verbose=verbose,
+                                                           skip_cache=skip_cache,
+                                                           from_build=[job],
+                                                           no_unpack=False,
+                                                           retry=0)
+            # Change back the cwd
+            os.chdir(currentWorkingDir)
+            return rc
+
     def _run_clang_format_diff(self, clang_format_diff, clang_format, show):
         # Run clang-format on the diff
         # Note that this will potentially miss a lot things
         from subprocess import Popen, PIPE, check_output, CalledProcessError
 
         diff_process = Popen(self._get_clang_format_diff_command(), stdout=PIPE)
         args = [sys.executable, clang_format_diff, "-p1", "-binary=%s" % clang_format]
 
@@ -2527,9 +2721,8 @@ class Analyze(MachCommandBase):
         if os.path.isfile(path):
             g = Graph(path)
             g.file_summaries(files)
             g.close()
         else:
             res = 'Please make sure you have a local tup db *or* specify the location with --path.'
             print ('Could not find a valid tup db in ' + path, res, sep='\n')
             return 1
-
new file mode 100644
--- /dev/null
+++ b/tools/infer/config.yaml
@@ -0,0 +1,23 @@
+---
+target: obj-x86_64-pc-linux-gnu
+# It is used by 'mach static-analysis' and 'mozreview static-analysis bot'
+# in order to have consistency across the used checkers.
+platforms:
+  - linux64
+infer_checkers:
+  - name: check-nullable
+    publish: !!bool yes
+  - name: eradicate
+    publish: !!bool no
+  - name: quandary
+    publish: !!bool yes
+  - name: starvation
+    publish: !!bool yes
+  - name: litho
+    publish: !!bool yes
+  - name: racerd
+    publish: !!bool yes
+  - name: liveness
+    publish: !!bool yes
+# Third party files from mozilla-central
+third_party: tools/rewriting/ThirdPartyPaths.txt