Bug 1473278: Add infer to java-check, install, clear-cache, and print-checks subcommands. draft
authorRobert Bartlensky <rbartlensky@mozilla.com>
Thu, 26 Jul 2018 14:45:44 +0100
changeset 827544 c200c3710b6c51dc58c709c95b06e476876c8b50
parent 827540 17116905bc072c37d74226ccc46c93f0bd45d516
push id118543
push userbmo:rbartlensky@mozilla.com
push dateWed, 08 Aug 2018 11:00:28 +0000
bugs1473278
milestone63.0a1
Bug 1473278: Add infer to java-check, install, clear-cache, and print-checks subcommands. MozReview-Commit-ID: 5ngZu6lh1wU
.inferconfig
python/mozbuild/mozbuild/mach_commands.py
tools/infer/config.yaml
deleted file mode 100644
--- a/.inferconfig
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-    "infer-blacklist-path-regex": [
-        // This is full of issues, and is a dependency we need to discard
-        // sooner rather than later anyway:
-        "mobile/android/thirdparty/ch/boye/httpclientandroidlib"
-    ]
-}
\ No newline at end of file
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1625,17 +1625,17 @@ class StaticAnalysisMonitor(object):
         return (warning, True)
 
 
 @CommandProvider
 class StaticAnalysis(MachCommandBase):
     """Utilities for running C++ static analysis checks and format."""
 
     # List of file extension to consider (should start with dot)
-    _format_include_extensions = ('.cpp', '.c', '.h')
+    _format_include_extensions = ('.cpp', '.c', '.h', '.java')
     # File contaning all paths to exclude from formatting
     _format_ignore_file = '.clang-format-ignore'
 
     @Command('static-analysis', category='testing',
              description='Run C++ static analysis checks')
     def static_analysis(self):
         # If not arguments are provided, just print a help message.
         mach = Mach(os.getcwd())
@@ -1707,17 +1707,151 @@ class StaticAnalysis(MachCommandBase):
 
         footer = StaticAnalysisFooter(self.log_manager.terminal, monitor)
         with StaticAnalysisOutputManager(self.log_manager, monitor, footer) as output:
             rc = self.run_process(args=args, line_handler=output.on_line, cwd=cwd)
 
             self.log(logging.WARNING, 'warning_summary',
                      {'count': len(monitor.warnings_db)},
                      '{count} warnings present.')
+        if rc != 0:
             return rc
+        # if we are building firefox for android it might be nice to
+        # also analyze the java code base
+        if self.substs['MOZ_BUILD_APP'] == 'mobile/android':
+            rc = self.check_java(source, jobs, strip, verbose, skip_export=True)
+        return rc
+
+    @StaticAnalysisSubCommand('static-analysis', 'check-java',
+                              'Run infer on the java codebase.')
+    @CommandArgument('source', nargs='*', default=['mobile'],
+                     help='Source files to be analyzed. '
+                          'Can be omitted, in which case the entire code base '
+                          'is analyzed.  The source argument is ignored if '
+                          'there is anything fed through stdin, in which case '
+                          'the analysis is only performed on the files changed '
+                          'in the patch streamed through stdin.  This is called '
+                          'the diff mode.')
+    @CommandArgument('--checks', '-c', default=[], metavar='checks', nargs='*',
+                     help='Static analysis checks to enable.')
+    @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,
+                     help='Number of concurrent jobs to run.'
+                     ' Default is the number of CPUs.')
+    @CommandArgument('--task', '-t', type=str,
+                     default='compileLocalWithGeckoBinariesNoMinApiDebugSources',
+                     help='Which gradle tasks to use to compile the java codebase.')
+    def check_java(self, source=['mobile'], jobs=2, strip=1, verbose=False, checks=[],
+                   task='compileLocalWithGeckoBinariesNoMinApiDebugSources',
+                   skip_export=False):
+        self._set_log_level(verbose)
+        self.log_manager.enable_all_structured_loggers()
+        if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
+            self.log(logging.WARNING, 'static-analysis', {},
+                     'Cannot check java source code unless you are building for android!')
+            return 1
+        # if source contains the whole mobile folder, then we just have to
+        # analyze everything
+        check_all = any(i.rstrip(os.sep).split(os.sep)[-1] == 'mobile' for i in source)
+        # gather all java sources from the source variable
+        java_sources = []
+        if not check_all:
+            java_sources = self._get_java_files(source)
+            if not java_sources:
+                return 0
+        if not skip_export:
+            rc = self._build_export(jobs=jobs, verbose=verbose)
+            if rc != 0:
+                return rc
+        rc = self._get_infer(verbose=verbose)
+        if rc != 0:
+            self.log(logging.WARNING, 'static-analysis', {},
+                     'This command is only available for linux64!')
+            return rc
+        # which checkers to use, and which folders to exclude
+        all_checkers, third_party_path = self._get_infer_config()
+        checkers, excludes = self._get_infer_args(
+            checks=checks or all_checkers,
+            third_party_path=third_party_path
+        )
+        gradlew = mozpath.join(self.topsrcdir, 'gradlew')
+        # infer capture command
+        capture_cmd = [self._infer_path, 'capture'] + excludes + \
+            ['--', gradlew, task]
+        tmp_file, args = self._get_infer_source_args(java_sources)
+        # infer analyze command
+        analysis_cmd = [self._infer_path, 'analyze', '--keep-going'] +  \
+            checkers + args
+        # capture, then analyze the sources
+        for args in [[gradlew, 'clean'], capture_cmd, analysis_cmd]:
+            rc = self.run_process(args=args, cwd=self.topsrcdir,
+                                  pass_thru=True)
+            # if a command fails, break and close the tmp file before returning
+            if rc != 0:
+                break
+        if tmp_file:
+            tmp_file.close()
+        return rc
+
+    def _get_java_files(self, sources):
+        java_sources = []
+        for i in sources:
+            f = mozpath.join(self.topsrcdir, i)
+            if os.path.isdir(f):
+                for root, dirs, files in os.walk(f):
+                    dirs.sort()
+                    for file in sorted(files):
+                        if file.endswith('.java'):
+                            java_sources.append(mozpath.join(root, file))
+            elif f.endswith('.java'):
+                java_sources.append(f)
+        return java_sources
+
+    def _get_infer_source_args(self, sources):
+        '''Return the arguments to only analyze <sources>'''
+        if not sources:
+            return (None, [])
+        # create a temporary file in which we place all sources
+        # this is used by the analysis command to only analyze certain files
+        f = tempfile.NamedTemporaryFile()
+        for source in sources:
+            f.write(source+'\n')
+        f.flush()
+        return (f, ['--changed-files-index', f.name])
+
+    def _get_infer_config(self):
+        '''Load the infer config file.'''
+        import yaml
+        checkers = []
+        tp_path = ''
+        with open(mozpath.join(self.topsrcdir, 'tools',
+                               'infer', 'config.yaml')) as f:
+            try:
+                config = yaml.safe_load(f)
+                for item in config['infer_checkers']:
+                    if item['publish']:
+                        checkers.append(item['name'])
+                tp_path = mozpath.join(self.topsrcdir, config['third_party'])
+            except Exception as e:
+                print('Looks like config.yaml is not valid, so we are unable '
+                      'to determine default checkers, and which folder to '
+                      'exclude, using defaults provided by infer')
+        return checkers, tp_path
+
+    def _get_infer_args(self, checks, third_party_path):
+        '''Return the arguments which include the checkers <checks>, and
+        excludes all folder in <third_party_path>.'''
+        checkers = ['-a', 'checkers']
+        excludes = []
+        for checker in checks:
+            checkers.append('--' + checker)
+        with open(third_party_path) as f:
+            for line in f:
+                excludes.append('--skip-analysis-in-path')
+                excludes.append(line.strip('\n'))
+        return checkers, excludes
 
     def _get_clang_tidy_command(self, checks, header_filter, sources, jobs, fix):
 
         if checks == '-*':
             checks = self._get_checks()
 
         common_args = ['-clang-tidy-binary', self._clang_tidy_path,
                        '-clang-apply-replacements-binary', self._clang_apply_replacements,
@@ -1741,17 +1875,18 @@ class StaticAnalysis(MachCommandBase):
     @StaticAnalysisSubCommand('static-analysis', 'autotest',
                               'Run the auto-test suite in order to determine that'
                               ' the analysis did not regress.')
     @CommandArgument('--dump-results', '-d', default=False, action='store_true',
                      help='Generate the baseline for the regression test. Based on'
                      ' this baseline we will test future results.')
     @CommandArgument('--intree-tool', '-i', default=False, action='store_true',
                      help='Use a pre-aquired in-tree clang-tidy package.')
-    @CommandArgument('checker_names', nargs='*', default=[], help='Checkers that are going to be auto-tested.')
+    @CommandArgument('checker_names', nargs='*', default=[],
+                     help='Checkers that are going to be auto-tested.')
     def autotest(self, verbose=False, dump_results=False, intree_tool=False, checker_names=[]):
         # If 'dump_results' is True than we just want to generate the issues files for each
         # checker in particulat and thus 'force_download' becomes 'False' since we want to
         # do this on a local trusted clang-tidy package.
         self._set_log_level(verbose)
         self._dump_results = dump_results
 
         force_download = True
@@ -1788,29 +1923,31 @@ class StaticAnalysis(MachCommandBase):
                                                    "clang", "clang-format-diff.py")
 
             # Ensure that clang-tidy is present
             rc = not os.path.exists(self._clang_tidy_path)
         else:
             rc = self._get_clang_tools(force=force_download, verbose=verbose)
 
         if rc != 0:
-            self.log(logging.ERROR, 'ERROR: static-analysis', {}, 'clang-tidy unable to locate package.')
+            self.log(logging.ERROR, 'ERROR: static-analysis', {},
+                     'clang-tidy unable to locate package.')
             return self.TOOLS_FAILED_DOWNLOAD
 
         self._clang_tidy_base_path = mozpath.join(self.topsrcdir, "tools", "clang-tidy")
 
         # For each checker run it
         f = open(mozpath.join(self._clang_tidy_base_path, "config.yaml"))
         import yaml
         config = yaml.safe_load(f)
         platform, _ = self.platform
 
         if platform not in config['platforms']:
-            self.log(logging.ERROR, 'static-analysis', {},"RUNNING: clang-tidy autotest for platform {} not supported.".format(platform))
+            self.log(logging.ERROR, 'static-analysis', {},
+                     "RUNNING: clang-tidy autotest for platform {} not supported.".format(platform))
             return TOOLS_UNSUPORTED_PLATFORM
 
         import concurrent.futures
         import multiprocessing
         import shutil
 
         max_workers = multiprocessing.cpu_count()
 
@@ -1870,50 +2007,65 @@ class StaticAnalysis(MachCommandBase):
             json.dump(compile_commands, file_handler)
             file_handler.flush()
 
             return directory
 
     @StaticAnalysisSubCommand('static-analysis', 'install',
                               'Install the static analysis helper tool')
     @CommandArgument('source', nargs='?', type=str,
-                     help='Where to fetch a local archive containing the static-analysis and format helper tool.'
-                          'It will be installed in ~/.mozbuild/clang-tools/.'
-                          'Can be omitted, in which case the latest clang-tools '
-                          ' helper for the platform would be automatically '
-                          'detected and installed.')
+                     help='Where to fetch a local archive containing the static-analysis and '
+                     'format helper tool.'
+                          'It will be installed in ~/.mozbuild/clang-tools and ~/.mozbuild/infer.'
+                          'Can be omitted, in which case the latest clang-tools and infer '
+                          'helper for the platform would be automatically detected and installed.')
     @CommandArgument('--skip-cache', action='store_true',
                      help='Skip all local caches to force re-fetching the helper tool.',
                      default=False)
     def install(self, source=None, skip_cache=False, verbose=False):
         self._set_log_level(verbose)
         rc = self._get_clang_tools(force=True, skip_cache=skip_cache,
                                    source=source, verbose=verbose)
+        if rc == 0:
+            # XXX ignore the return code because if it fails or not, infer is
+            # not mandatory, but clang-tidy is
+            self._get_infer(force=True, skip_cache=skip_cache, verbose=verbose)
         return rc
 
     @StaticAnalysisSubCommand('static-analysis', 'clear-cache',
                               'Delete local helpers and reset static analysis helper tool cache')
     def clear_cache(self, verbose=False):
         self._set_log_level(verbose)
         rc = self._get_clang_tools(force=True, download_if_needed=True, skip_cache=True,
                                    verbose=verbose)
+        if rc == 0:
+            self._get_infer(force=True, download_if_needed=True, skip_cache=True,
+                            verbose=verbose)
         if rc != 0:
             return rc
-
-        self._artifact_manager.artifact_clear_cache()
+        return self._artifact_manager.artifact_clear_cache()
 
     @StaticAnalysisSubCommand('static-analysis', 'print-checks',
                               'Print a list of the static analysis checks performed by default')
     def print_checks(self, verbose=False):
         self._set_log_level(verbose)
         rc = self._get_clang_tools(verbose=verbose)
+        if rc == 0:
+            rc = self._get_infer(verbose=verbose)
         if rc != 0:
             return rc
         args = [self._clang_tidy_path, '-list-checks', '-checks=%s' % self._get_checks()]
-        return self._run_command_in_objdir(args=args, pass_thru=True)
+        rc = self._run_command_in_objdir(args=args, pass_thru=True)
+        if rc != 0:
+            return rc
+        checkers, _ = self._get_infer_config()
+        print('Infer checks:')
+        for checker in checkers:
+            print(' '*4 + checker)
+        return 0
 
     @Command('clang-format',  category='misc', description='Run clang-format on current changes')
     @CommandArgument('--show', '-s', action='store_true', default=False,
                      help='Show diff output on instead of applying changes')
     @CommandArgument('--path', '-p', nargs='+', default=None,
                      help='Specify the path(s) to reformat')
     def clang_format(self, show, path, verbose=False):
         # Run clang-format or clang-format-diff on the local changes
@@ -1934,17 +2086,18 @@ class StaticAnalysis(MachCommandBase):
             return self._run_clang_format_path(self._clang_format_path, show, path)
 
     def _verify_checker(self, item):
         check = item['name']
         test_file_path = mozpath.join(self._clang_tidy_base_path, "test", check)
         test_file_path_cpp = test_file_path + '.cpp'
         test_file_path_json = test_file_path + '.json'
 
-        self.log(logging.INFO, 'static-analysis', {},"RUNNING: clang-tidy checker {}.".format(check))
+        self.log(logging.INFO, 'static-analysis', {},
+                 "RUNNING: clang-tidy checker {}.".format(check))
 
         # Verify if this checker actually exists
         if not check in self._clang_tidy_checks:
             self.log(logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy checker {} doesn't exist in this clang-tidy version.".format(check))
             return self.TOOLS_CHECKER_NOT_FOUND
 
         # Verify if the test file exists for this checker
         if not os.path.exists(test_file_path_cpp):
@@ -2226,16 +2379,57 @@ class StaticAnalysis(MachCommandBase):
                         continue  # empty or comment
                     magics = ['exclude']
                     if pattern.startswith('^'):
                         magics += ['top']
                         pattern = pattern[1:]
                     args += [':({0}){1}'.format(','.join(magics), pattern)]
         return args
 
+    def _get_infer(self, force=False, skip_cache=False,
+                   download_if_needed=True, verbose=False):
+        rc, config, _ = self._get_config_environment()
+        if rc != 0:
+            return rc
+        infer_path = mozpath.join(self._mach_context.state_dir, 'infer')
+        self._infer_path = mozpath.join(infer_path, 'infer', 'bin',
+                                        'infer' +
+                                        config.substs.get('BIN_SUFFIX', ''))
+        if os.path.exists(self._infer_path) and not force:
+            return 0
+        else:
+            if os.path.isdir(infer_path) and download_if_needed:
+                # The directory exists, perhaps it's corrupted?  Delete it
+                # and start from scratch.
+                import shutil
+                shutil.rmtree(infer_path)
+                return self._get_infer(force=force, skip_cache=skip_cache,
+                                       verbose=verbose,
+                                       download_if_needed=download_if_needed)
+            os.mkdir(infer_path)
+            self._artifact_manager = PackageFrontend(self._mach_context)
+            if not download_if_needed:
+                return 0
+            job, _ = self.platform
+            if job != 'linux64':
+                return -1
+            else:
+                job += '-infer'
+            # We want to unpack data in the infer mozbuild folder
+            currentWorkingDir = os.getcwd()
+            os.chdir(infer_path)
+            rc = self._artifact_manager.artifact_toolchain(verbose=verbose,
+                                                           skip_cache=skip_cache,
+                                                           from_build=[job],
+                                                           no_unpack=False,
+                                                           retry=0)
+            # Change back the cwd
+            os.chdir(currentWorkingDir)
+            return rc
+
     def _run_clang_format_diff(self, clang_format_diff, clang_format, show):
         # Run clang-format on the diff
         # Note that this will potentially miss a lot things
         from subprocess import Popen, PIPE, check_output, CalledProcessError
 
         diff_process = Popen(self._get_clang_format_diff_command(), stdout=PIPE)
         args = [sys.executable, clang_format_diff, "-p1", "-binary=%s" % clang_format]
 
@@ -2527,9 +2721,8 @@ class Analyze(MachCommandBase):
         if os.path.isfile(path):
             g = Graph(path)
             g.file_summaries(files)
             g.close()
         else:
             res = 'Please make sure you have a local tup db *or* specify the location with --path.'
             print ('Could not find a valid tup db in ' + path, res, sep='\n')
             return 1
-
new file mode 100644
--- /dev/null
+++ b/tools/infer/config.yaml
@@ -0,0 +1,23 @@
+---
+target: obj-x86_64-pc-linux-gnu
+# It is used by 'mach static-analysis' and 'mozreview static-analysis bot'
+# in order to have consistency across the used checkers.
+platforms:
+  - linux64
+infer_checkers:
+  - name: check-nullable
+    publish: !!bool yes
+  - name: eradicate
+    publish: !!bool no
+  - name: quandary
+    publish: !!bool yes
+  - name: starvation
+    publish: !!bool yes
+  - name: litho
+    publish: !!bool yes
+  - name: racerd
+    publish: !!bool yes
+  - name: liveness
+    publish: !!bool yes
+# Third party files from mozilla-central
+third_party: tools/rewriting/ThirdPartyPaths.txt