Bug 1541412 - Add static-analysis check-java to source-test tasks, r=ahal,andi
authorBastien Abadie <bastien@mozilla.com>
Mon, 08 Apr 2019 15:41:30 +0000
changeset 468772 323eef8fd5ff8d4968c77fb6c77d65b010b31ad6
parent 468771 68df166b4d9e53aaa1ff1e39ea6f3e534814f891
child 468773 e0edc59f9158d65e91a45cb6b1d96d75b5998da3
push id35850
push userdvarga@mozilla.com
push dateWed, 10 Apr 2019 21:52:56 +0000
treeherdermozilla-central@9d3dbe3fef26 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal, andi
bugs1541412
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1541412 - Add static-analysis check-java to source-test tasks, r=ahal,andi Differential Revision: https://phabricator.services.mozilla.com/D25929
python/mozbuild/mozbuild/mach_commands.py
taskcluster/ci/source-test/infer.yml
taskcluster/ci/source-test/kind.yml
taskcluster/scripts/misc/source-test-clang-setup.sh
taskcluster/scripts/misc/source-test-common.sh
taskcluster/scripts/misc/source-test-infer-setup.sh
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -2048,34 +2048,51 @@ class StaticAnalysis(MachCommandBase):
     @CommandArgument('--checks', '-c', default=[], metavar='checks', nargs='*',
                      help='Static analysis checks to enable.')
     @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int,
                      help='Number of concurrent jobs to run.'
                      ' Default is the number of CPUs.')
     @CommandArgument('--task', '-t', type=str,
                      default='compileWithGeckoBinariesDebugSources',
                      help='Which gradle tasks to use to compile the java codebase.')
+    @CommandArgument('--outgoing', default=False, action='store_true',
+                     help='Run infer checks on outgoing files from repository')
+    @CommandArgument('--output', default=None,
+                     help='Write infer json output in a file')
     def check_java(self, source=['mobile'], jobs=2, strip=1, verbose=False, checks=[],
                    task='compileWithGeckoBinariesDebugSources',
-                   skip_export=False):
+                   skip_export=False, outgoing=False, output=None):
         self._set_log_level(verbose)
         self.log_manager.enable_all_structured_loggers()
         if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
             self.log(logging.WARNING, 'static-analysis', {},
                      'Cannot check java source code unless you are building for android!')
             return 1
         rc = self._check_for_java()
         if rc != 0:
             return 1
+        if output is not None:
+            output = os.path.abspath(output)
+            if not os.path.isdir(os.path.dirname(output)):
+                self.log(logging.WARNING, 'static-analysis', {},
+                         'Missing report destination folder for {}'.format(output))
+
         # if source contains the whole mobile folder, then we just have to
         # analyze everything
         check_all = any(i.rstrip(os.sep).split(os.sep)[-1] == 'mobile' for i in source)
         # gather all java sources from the source variable
         java_sources = []
-        if not check_all:
+        if outgoing:
+            repo = get_repository_object(self.topsrcdir)
+            java_sources = self._get_java_files(repo.get_outgoing_files())
+            if not java_sources:
+                self.log(logging.WARNING, 'static-analysis', {},
+                         'No outgoing Java files to check')
+                return 0
+        elif not check_all:
             java_sources = self._get_java_files(source)
             if not java_sources:
                 return 0
         if not skip_export:
             rc = self._build_export(jobs=jobs, verbose=verbose)
             if rc != 0:
                 return rc
         rc = self._get_infer(verbose=verbose)
@@ -2095,16 +2112,24 @@ class StaticAnalysis(MachCommandBase):
         rc = rc or self._gradle([task], infer_args=capture_cmd, verbose=verbose)
         tmp_file, args = self._get_infer_source_args(java_sources)
         # infer analyze command
         analysis_cmd = [self._infer_path, 'analyze', '--keep-going'] +  \
             checkers + args
         rc = rc or self.run_process(args=analysis_cmd, cwd=self.topsrcdir, pass_thru=True)
         if tmp_file:
             tmp_file.close()
+
+        # Copy the infer report
+        report_path = os.path.join(self.topsrcdir, 'infer-out', 'report.json')
+        if output is not None and os.path.exists(report_path):
+            shutil.copy(report_path, output)
+            self.log(logging.INFO, 'static-analysis', {},
+                     'Report available in {}'.format(output))
+
         return rc
 
     def _get_java_files(self, sources):
         java_sources = []
         for i in sources:
             f = mozpath.join(self.topsrcdir, i)
             if os.path.isdir(f):
                 for root, dirs, files in os.walk(f):
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/source-test/infer.yml
@@ -0,0 +1,50 @@
+job-defaults:
+    # Run only on try and code-review tasks
+    # to avoid running infer on the whole codebase
+    run-on-projects:
+        - try
+
+    platform: linux64/opt
+    attributes:
+        code-review: true
+    worker-type:
+        by-platform:
+            linux64.*: aws-provisioner-v1/gecko-t-linux-xlarge
+    worker:
+        docker-image: {in-tree: android-build}
+        max-run-time: 5400
+    treeherder:
+        kind: other
+        tier: 2
+    run:
+        using: run-task
+        tooltool-downloads: public
+    toolchains:
+        - linux64-infer
+        - linux64-android-sdk-linux-repack
+        - linux64-android-ndk-linux-repack
+        - linux64-rust-android
+        - linux64-clang
+        - linux64-cbindgen
+        - linux64-nasm
+        - linux64-node
+    when:
+        files-changed:
+            - 'mobile/**/*.java'
+
+infer:
+    description: Run static-analysis (infer) on Java patches
+    treeherder:
+        symbol: infer
+    run:
+        command: >-
+            source $HOME/checkouts/gecko/taskcluster/scripts/misc/source-test-infer-setup.sh &&
+            cd $HOME/checkouts/gecko &&
+            ./mach --log-no-times configure &&
+            ./mach --log-no-times static-analysis check-java --outgoing --output $HOME/infer.json
+
+    worker:
+        artifacts:
+            - type: file
+              name: public/code-review/infer.json
+              path: /builds/worker/infer.json
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -16,16 +16,17 @@ kind-dependencies:
    - toolchain
 
 jobs-from:
    - clang.yml
    - coverity.yml
    - cram.yml
    - doc.yml
    - file-metadata.yml
+   - infer.yml
    - jsshell.yml
    - mozlint.yml
    - node.yml
    - python.yml
    - webidl.yml
    - wpt-manifest.yml
 
 # This is used by run-task based tasks to lookup which build task it
--- a/taskcluster/scripts/misc/source-test-clang-setup.sh
+++ b/taskcluster/scripts/misc/source-test-clang-setup.sh
@@ -1,24 +1,12 @@
-#! /bin/bash -vex
-
-set -x -e
-
-export MOZBUILD_STATE_PATH=$HOME/workspace
+#!/bin/bash
+source $HOME/checkouts/gecko/taskcluster/scripts/misc/source-test-common.sh
 
-# Setup toolchains
-cd $MOZBUILD_STATE_PATH
-$HOME/checkouts/gecko/mach artifact toolchain -v $MOZ_TOOLCHAINS
-
-# Add toolchain binaries to PATH
-export PATH=$MOZBUILD_STATE_PATH/nasm:$PATH
-export PATH=$MOZBUILD_STATE_PATH/clang/bin:$PATH
+# Add clang-tidy to PATH
 export PATH=$MOZBUILD_STATE_PATH/clang-tidy/bin:$PATH
-export PATH=$MOZBUILD_STATE_PATH/rustc/bin:$PATH
-export PATH=$MOZBUILD_STATE_PATH/cbindgen:$PATH
-export PATH=$MOZBUILD_STATE_PATH/node/bin:$PATH
 
 # Use toolchain clang
 export LD_LIBRARY_PATH=$MOZBUILD_STATE_PATH/clang/lib
 
 # Mach lookup clang-tidy in clang-tools
 mkdir -p $MOZBUILD_STATE_PATH/clang-tools
 ln -s $MOZBUILD_STATE_PATH/clang-tidy $MOZBUILD_STATE_PATH/clang-tools/clang-tidy
copy from taskcluster/scripts/misc/source-test-clang-setup.sh
copy to taskcluster/scripts/misc/source-test-common.sh
--- a/taskcluster/scripts/misc/source-test-clang-setup.sh
+++ b/taskcluster/scripts/misc/source-test-common.sh
@@ -3,22 +3,18 @@
 set -x -e
 
 export MOZBUILD_STATE_PATH=$HOME/workspace
 
 # Setup toolchains
 cd $MOZBUILD_STATE_PATH
 $HOME/checkouts/gecko/mach artifact toolchain -v $MOZ_TOOLCHAINS
 
-# Add toolchain binaries to PATH
-export PATH=$MOZBUILD_STATE_PATH/nasm:$PATH
+# Add toolchain binaries to PATH to run ./mach configure
 export PATH=$MOZBUILD_STATE_PATH/clang/bin:$PATH
-export PATH=$MOZBUILD_STATE_PATH/clang-tidy/bin:$PATH
 export PATH=$MOZBUILD_STATE_PATH/rustc/bin:$PATH
 export PATH=$MOZBUILD_STATE_PATH/cbindgen:$PATH
+export PATH=$MOZBUILD_STATE_PATH/nasm:$PATH
 export PATH=$MOZBUILD_STATE_PATH/node/bin:$PATH
 
-# Use toolchain clang
-export LD_LIBRARY_PATH=$MOZBUILD_STATE_PATH/clang/lib
-
-# Mach lookup clang-tidy in clang-tools
-mkdir -p $MOZBUILD_STATE_PATH/clang-tools
-ln -s $MOZBUILD_STATE_PATH/clang-tidy $MOZBUILD_STATE_PATH/clang-tools/clang-tidy
+# Use clang as host compiler
+export CC=$MOZBUILD_STATE_PATH/clang/bin/clang
+export CXX=$MOZBUILD_STATE_PATH/clang/bin/clang++
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/source-test-infer-setup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+source $HOME/checkouts/gecko/taskcluster/scripts/misc/source-test-common.sh
+
+# Write custom mozconfig
+MOZCONFIG=$HOME/checkouts/gecko/mozconfig
+echo "ac_add_options --enable-application=mobile/android" > $MOZCONFIG
+echo "ac_add_options --target=arm-linux-androideabi" >> $MOZCONFIG
+echo "ac_add_options --with-android-sdk=${MOZBUILD_STATE_PATH}/android-sdk-linux" >> $MOZCONFIG
+echo "ac_add_options --with-android-ndk=${MOZBUILD_STATE_PATH}/android-ndk" >> $MOZCONFIG
+
+# Write custom grade properties
+export GRADLE_USER_HOME=$HOME/workspace/gradle
+mkdir -p $GRADLE_USER_HOME
+echo "org.gradle.daemon=false" >> ${GRADLE_USER_HOME}/gradle.properties
+
+# Mach lookup infer in infer...
+mkdir -p $MOZBUILD_STATE_PATH/infer/infer
+mv $MOZBUILD_STATE_PATH/infer/{bin,lib} $MOZBUILD_STATE_PATH/infer/infer