Bug 1473951: Add infer to taskcluster and build. r=gps
authorRobert Bartlensky <rbartlensky@mozilla.com>
Fri, 06 Jul 2018 17:37:16 +0100
changeset 428779 2e7b746dfdcf28d93f32cb4cede3124e2a0f486c
parent 428758 d353b80fd66cd8519b1637fe8b071e6e05fbe95e
child 428780 98a2048f6d50a1a3f1ee7f76011d78622bb3e9b7
push id105780
push usercsabou@mozilla.com
push dateFri, 27 Jul 2018 22:57:42 +0000
treeherdermozilla-inbound@b1713dde630a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1473951
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1473951: Add infer to taskcluster and build. r=gps MozReview-Commit-ID: BHi3E6J3nuH
.flake8
build/build-infer/README
build/build-infer/build-infer.py
build/build-infer/infer-linux64.json
taskcluster/ci/docker-image/kind.yml
taskcluster/ci/static-analysis-autotest/kind.yml
taskcluster/ci/toolchain/linux.yml
taskcluster/docker/infer-build/Dockerfile
taskcluster/scripts/misc/build-infer-linux.sh
--- a/.flake8
+++ b/.flake8
@@ -16,8 +16,9 @@ exclude =
     glx/skia/,
     js/*.configure,
     memory/moz.configure,
     mobile/android/*.configure,
     node_modules,
     security/nss/,
     testing/mochitest/pywebsocket,
     tools/lint/test/files,
+    build/build-infer/build-infer.py,
new file mode 100644
--- /dev/null
+++ b/build/build-infer/README
@@ -0,0 +1,36 @@
+build-infer.py
+==============
+
+A script to build infer from source.
+
+```
+usage: build-infer.py [-h] -c CONFIG [--clean]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -c CONFIG, --config CONFIG
+                        infer configuration file
+  --clean               Clean the build directory
+```
+
+Pre-requisites
+--------------
+* Working build toolchain.
+* ocam
+* git
+* autoconf
+* libsqlite-dev
+* CMake
+* Ninja
+* Python 2.7
+
+Please use the latest available CMake for your platform to avoid surprises.
+
+Config file format
+------------------
+
+build-clang.py accepts a JSON config format with the following fields:
+
+* infer_revision: The infer revision to build.
+* infer_repo: git repository for infer.
+* patches: Optional list of patches to apply.
\ No newline at end of file
new file mode 100755
--- /dev/null
+++ b/build/build-infer/build-infer.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+import json
+import argparse
+import sys
+import shutil
+from functools import reduce
+
+
+def check_run(args, path):
+    print(' '.join(args) + ' in ' + path, file=sys.stderr)
+    subprocess.run(args, cwd=path, check=True)
+
+
+def run_in(path, args, extra_env=None):
+    '''
+    Runs the given commands in the directory specified by <path>.
+    '''
+    env = dict(os.environ)
+    env.update(extra_env or {})
+    check_run(args, path)
+    subprocess.run(args, cwd=path)
+
+
+def build_tar_package(tar, name, base, directory):
+    name = os.path.realpath(name)
+    run_in(base, [tar,
+                  '-c',
+                  '-%s' % ('J' if '.xz' in name else 'j'),
+                  '-f',
+                  name, directory])
+
+
+def is_git_repo(dir):
+    '''Check whether the given directory is a git repository.'''
+    from subprocess import CalledProcessError
+    try:
+        check_run(['git', 'rev-parse'], dir)
+        return True
+    except CalledProcessError:
+        return False
+
+
+def git_clone(main_dir, url, clone_dir, commit):
+    '''
+    Clones the repository from <url> into <clone_dir>, and brings the
+    repository to the state of <commit>.
+    '''
+    run_in(main_dir, ['git', 'clone', url, clone_dir])
+    run_in(clone_dir, ['git', 'checkout', commit])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config', required=True,
+                        type=argparse.FileType('r'),
+                        help='Infer configuration file')
+    parser.add_argument('-b', '--base-dir',
+                        help="Base directory for code and build artifacts")
+    parser.add_argument('--clean', action='store_true',
+                        help='Clean the build directory')
+    parser.add_argument('--skip-tar', action='store_true',
+                        help='Skip tar packaging stage')
+
+    args = parser.parse_args()
+
+    # The directories end up in the debug info, so the easy way of getting
+    # a reproducible build is to run it in a know absolute directory.
+    # We use a directory that is registered as a volume in the Docker image.
+    if args.base_dir:
+        base_dir = args.base_dir
+    else:
+        base_dir = reduce(os.path.join,
+                          [os.sep + 'builds', 'worker',
+                           'workspace', 'moz-toolchain'])
+    infer_dir = os.path.join(base_dir, 'infer')
+    source_dir = os.path.join(infer_dir, 'src')
+    build_dir = os.path.join(infer_dir, 'build')
+
+    if args.clean:
+        shutil.rmtree(build_dir)
+        os.sys.exit(0)
+
+    config = json.load(args.config)
+    infer_revision = config['infer_revision']
+    infer_repo = config['infer_repo']
+
+    for folder in [infer_dir, source_dir, build_dir]:
+        os.makedirs(folder, exist_ok=True)
+
+    # clone infer
+    if not is_git_repo(source_dir):
+        # git doesn't like cloning into a non-empty folder. If src is not a git
+        # repo then just remove it in order to reclone
+        shutil.rmtree(source_dir)
+        git_clone(infer_dir, infer_repo, source_dir, infer_revision)
+    # apply a few patches
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    # clean the git directory by reseting all changes
+    git_commands = [['clean', '-f'], ['reset', '--hard']]
+    for command in git_commands:
+        run_in(source_dir, ['git']+command)
+    for p in config.get('patches', []):
+        run_in(source_dir, ['git', 'apply', os.path.join(dir_path, p)])
+    # build infer
+    run_in(source_dir, ['./build-infer.sh', 'java'],
+           extra_env={'NO_CMAKE_STRIP': '1'})
+    package_name = 'infer'
+    if not args.skip_tar:
+        build_tar_package('tar', '%s.tar.xz' % (package_name),
+                          source_dir, os.path.join('infer', 'bin'))
new file mode 100644
--- /dev/null
+++ b/build/build-infer/infer-linux64.json
@@ -0,0 +1,5 @@
+{
+    "infer_repo": "https://github.com/facebook/infer",
+    "infer_revision": "14aa1edbf532675f48ba0a35a2e55d07b79b1f3c",
+    "patches": []
+}
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -81,16 +81,19 @@ jobs:
       - deb9-mercurial
       - deb9-python-zstandard
   android-build:
     symbol: I(agb)
     parent: debian9-base
   fetch:
     symbol: I(fetch)
     parent: debian9-base
+  infer-build:
+    symbol: I(infer)
+    parent: debian9-base
   mingw32-build:
     symbol: I(mingw)
     parent: debian9-base
   index-task:
     symbol: I(idx)
   funsize-update-generator:
     symbol: I(pg)
   google-play-strings:
--- a/taskcluster/ci/static-analysis-autotest/kind.yml
+++ b/taskcluster/ci/static-analysis-autotest/kind.yml
@@ -43,16 +43,17 @@ jobs:
             config:
                 - builds/releng_base_firefox.py
                 - builds/releng_sub_linux_configs/64_stat_and_debug.py
             tooltool-downloads: public
             keep-artifacts: false
         toolchains:
             - linux64-clang
             - linux64-clang-tidy
+            - linux64-infer
             - linux64-rust
             - linux64-sccache
 
     win64-st-autotest/debug:
         description: "Win64 Debug Static Analysis Autotest"
         index:
             job-name: win64-st-autotest-debug
         treeherder:
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -120,16 +120,38 @@ linux64-clang-tidy:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clang-tidy.tar.xz
     run-on-projects:
         - trunk
         - try
     toolchains:
         - linux64-gcc-4.9
 
+linux64-infer:
+    description: "infer build"
+    index:
+        product: static-analysis
+        job-name: linux64-infer
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(infer)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: infer-build}
+        max-run-time: 3600
+    run:
+        using: toolchain-script
+        script: build-infer-linux.sh
+        resources:
+            - 'build/build-infer/build-infer.py'
+            - 'build/build-infer/infer-linux64.json'
+        toolchain-artifact: public/build/infer.tar.xz
+
 linux64-gcc-4.9:
     description: "GCC 4.9 toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(gcc4.9)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/infer-build/Dockerfile
@@ -0,0 +1,30 @@
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/workspace
+VOLUME /builds/worker/tooltool-cache
+
+ENV XZ_OPT=-T0
+
+RUN apt-get update && \
+    apt-get install \
+      autoconf \
+      bison \
+      bzip2 \
+      flex \
+      curl \
+      git \
+      opam \
+      libsqlite3-dev \
+      autoconf \
+      automake \
+      cmake \
+      libc6-dev \
+      openjdk-8-jdk-headless \
+      pkg-config \
+      patch \
+      tar \
+      unzip \
+      zlib1g-dev
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-infer-linux.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building infer for Linux.
+
+WORKSPACE=$HOME/workspace
+HOME_DIR=$WORKSPACE/build
+UPLOAD_DIR=$HOME/artifacts
+
+cd $HOME_DIR/src
+
+# gets a bit too verbose here
+set +x
+
+cd build/build-infer
+./build-infer.py -c infer-linux64.json
+
+set -x
+
+# Put a tarball in the artifacts dir
+mkdir -p $UPLOAD_DIR
+cp infer.tar.* $UPLOAD_DIR