Bug 1372127 - mach command to run clang-format, build, and tests, r=mt
authorFranziskus Kiefer <franziskuskiefer@gmail.com>
Thu, 27 Apr 2017 12:15:26 +0200
changeset 13438 8cd50ae99b5067c65611089843802173d5cf4050
parent 13437 ea7097106da78852117bfdc2e6a5bdf64ddca0cf
child 13439 70ff6d097ae93ede47f3e10992473fe6272f0619
push id2251
push userfranziskuskiefer@gmail.com
push dateMon, 12 Jun 2017 11:36:38 +0000
reviewersmt
bugs1372127
Bug 1372127 - mach command to run clang-format, build, and tests, r=mt Differential Revision: https://nss-review.dev.mozaws.net/D300
automation/clang-format/Dockerfile
automation/clang-format/run_clang_format.sh
automation/clang-format/setup.sh
automation/taskcluster/graph/src/extend.js
automation/taskcluster/scripts/run_clang_format.sh
mach
new file mode 100644
--- /dev/null
+++ b/automation/clang-format/Dockerfile
@@ -0,0 +1,26 @@
+FROM ubuntu:16.04
+MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com>
+
+RUN useradd -d /home/worker -s /bin/bash -m worker
+WORKDIR /home/worker
+
+# Install dependencies.
+ADD setup.sh /tmp/setup.sh
+RUN bash /tmp/setup.sh
+
+# Change user.
+USER worker
+
+# Env variables.
+ENV HOME /home/worker
+ENV SHELL /bin/bash
+ENV USER worker
+ENV LOGNAME worker
+ENV HOSTNAME taskcluster-worker
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV HOST localhost
+ENV DOMSUF localdomain
+
+# Entrypoint.
+ENTRYPOINT ["/home/worker/nss/automation/clang-format/run_clang_format.sh"]
new file mode 100755
--- /dev/null
+++ b/automation/clang-format/run_clang_format.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+if [[ $(id -u) -eq 0 ]]; then
+    # Drop privileges by re-running this script.
+    # Note: this mangles arguments, better to avoid running scripts as root.
+    exec su worker -c "$0 $*"
+fi
+
+# Apply clang-format on the provided folder and verify that this doesn't change any file.
+# If any file differs after formatting, the script eventually exits with 1.
+# Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong.
+
+# Includes a default set of directories NOT to clang-format on.
+blacklist=(
+     "./automation" \
+     "./coreconf" \
+     "./doc" \
+     "./pkg" \
+     "./tests" \
+     "./lib/libpkix" \
+     "./lib/zlib" \
+     "./lib/sqlite" \
+     "./gtests/google_test" \
+     "./.hg" \
+)
+
+top="$(dirname $0)/../.."
+cd "$top"
+
+if [ $# -gt 0 ]; then
+    dirs=("$@")
+else
+    dirs=($(find . -maxdepth 2 -mindepth 1 -type d ! -path . \( ! -regex '.*/' \)))
+fi
+
+format_folder()
+{
+    for black in "${blacklist[@]}"; do
+        if [[ "$1" == "$black"* ]]; then
+            echo "skip $1"
+            return 1
+        fi
+    done
+    return 0
+}
+
+for dir in "${dirs[@]}"; do
+    if format_folder "$dir" ; then
+        c="${dir//[^\/]}"
+        echo "formatting $dir ..."
+        depth=""
+        if [ "${#c}" == "1" ]; then
+            depth="-maxdepth 1"
+        fi
+        find "$dir" $depth -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+
+    fi
+done
+
+TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXX)
+trap 'rm $TMPFILE' exit
+if (cd $(dirname $0); hg root >/dev/null 2>&1); then
+    hg diff --git "$top" | tee $TMPFILE
+else
+    git -C "$top" diff | tee $TMPFILE
+fi
+[[ ! -s $TMPFILE ]]
new file mode 100644
--- /dev/null
+++ b/automation/clang-format/setup.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Update packages.
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y update && apt-get -y upgrade
+
+# Install packages.
+apt_packages=()
+apt_packages+=('ca-certificates')
+apt_packages+=('curl')
+apt_packages+=('xz-utils')
+apt_packages+=('mercurial')
+apt_packages+=('git')
+apt-get install -y --no-install-recommends ${apt_packages[@]}
+
+# Download clang.
+curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -o clang.tar.xz
+curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig -o clang.tar.xz.sig
+# Verify the signature.
+gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
+gpg --verify clang.tar.xz.sig
+# Install into /usr/local/.
+tar xJvf *.tar.xz -C /usr/local --strip-components=1
+
+# Cleanup.
+function cleanup() {
+  rm -f clang.tar.xz clang.tar.xz.sig
+}
+trap cleanup ERR EXIT
+
+locale-gen en_US.UTF-8
+dpkg-reconfigure locales
+
+# Cleanup.
+rm -rf ~/.ccache ~/.cache
+apt-get autoremove -y
+apt-get clean
+apt-get autoclean
+
+# We're done. Remove this script.
+rm $0
--- a/automation/taskcluster/graph/src/extend.js
+++ b/automation/taskcluster/graph/src/extend.js
@@ -741,17 +741,17 @@ async function scheduleTools() {
 
   queue.scheduleTask(merge(base, {
     symbol: "clang-format-3.9",
     name: "clang-format-3.9",
     image: LINUX_CLANG39_IMAGE,
     command: [
       "/bin/bash",
       "-c",
-      "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh"
+      "bin/checkout.sh && nss/automation/clang-format/run_clang_format.sh"
     ]
   }));
 
   queue.scheduleTask(merge(base, {
     symbol: "scan-build-4.0",
     name: "scan-build-4.0",
     image: LINUX_IMAGE,
     env: {
deleted file mode 100755
--- a/automation/taskcluster/scripts/run_clang_format.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-
-source $(dirname "$0")/tools.sh
-
-set +x
-
-# Apply clang-format on the provided folder and verify that this doesn't change any file.
-# If any file differs after formatting, the script eventually exits with 1.
-# Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong.
-
-# Includes a default set of directories NOT to clang-format on.
-blacklist=(
-     "./automation" \
-     "./coreconf" \
-     "./doc" \
-     "./pkg" \
-     "./tests" \
-     "./lib/libpkix" \
-     "./lib/zlib" \
-     "./lib/sqlite" \
-     "./gtests/google_test" \
-     "./.hg" \
-)
-
-top="$PWD/$(dirname $0)/../../.."
-cd "$top"
-
-if [ $# -gt 0 ]; then
-    dirs=("$@")
-else
-    dirs=($(find . ! -path . \( ! -regex '.*/' \) -maxdepth 2 -mindepth 1 -type d))
-fi
-
-format_folder()
-{
-    for black in "${blacklist[@]}"; do
-        if [[ "$1" == "$black"* ]]; then
-            echo "skip $1"
-            return 1
-        fi
-    done
-    return 0
-}
-
-for dir in "${dirs[@]}"; do
-    if format_folder "$dir" ; then
-        c="${dir//[^\/]}"
-        echo "formatting $dir ..."
-        depth=""
-        if [ "${#c}" == "1" ]; then
-            depth="-maxdepth 1"
-        fi
-        find "$dir" $depth -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+
-    fi
-done
-
-TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXX)
-trap 'rm $TMPFILE' exit
-if (cd $(dirname $0); hg root >/dev/null 2>&1); then
-    hg diff --git "$top" | tee $TMPFILE
-else
-    git -C "$top" diff | tee $TMPFILE
-fi
-[[ ! -s $TMPFILE ]]
new file mode 100755
--- /dev/null
+++ b/mach
@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+################################################################################
+#
+# This is a collection of helper tools to get stuff done in NSS.
+#
+
+import sys
+import argparse
+import subprocess
+import os
+import platform
+from hashlib import sha256
+
+cwd = os.path.dirname(os.path.abspath(__file__))
+
+
+class cfAction(argparse.Action):
+    docker_command = ["docker"]
+
+    def __call__(self, parser, args, values, option_string=None):
+        if "noroot" not in values:
+            self.setDockerCommand()
+        else:
+            values.remove("noroot")
+
+        # First check if we can run docker.
+        try:
+            with open(os.devnull, "w") as f:
+                subprocess.check_call(
+                    self.docker_command + ["images"], stdout=f)
+        except:
+            print("Please install docker and start the docker daemon.")
+            sys.exit(1)
+
+        docker_image = 'clang-format-service:latest'
+        cf_docker_folder = cwd + "/automation/clang-format"
+
+        # Build the image if necessary.
+        if self.filesChanged(cf_docker_folder):
+            self.buildImage(docker_image, cf_docker_folder)
+
+        # Check if we have the docker image.
+        try:
+            command = self.docker_command + [
+                "image", "inspect", "clang-format-service:latest"
+            ]
+            with open(os.devnull, "w") as f:
+                subprocess.check_call(command, stdout=f)
+        except:
+            print("I have to build the docker image first.")
+            self.buildImage(docker_image, cf_docker_folder)
+
+        command = self.docker_command + [
+            'run', '-v', cwd + ':/home/worker/nss', '--rm', '-ti', docker_image
+        ]
+        # The clang format script returns 1 if something's to do. We don't care.
+        subprocess.call(command + values)
+
+    def filesChanged(self, path):
+        hash = sha256()
+        for dirname, dirnames, files in os.walk(path):
+            for file in files:
+                with open(os.path.join(dirname, file), "rb") as f:
+                    hash.update(f.read())
+        chk_file = cwd + "/out/.chk"
+        old_chk = ""
+        new_chk = hash.hexdigest()
+        if os.path.exists(chk_file):
+            with open(chk_file) as f:
+                old_chk = f.readline()
+        if old_chk != new_chk:
+            with open(chk_file, "w+") as f:
+                f.write(new_chk)
+            return True
+        return False
+
+    def buildImage(self, docker_image, cf_docker_folder):
+        command = self.docker_command + [
+            "build", "-t", docker_image, cf_docker_folder
+        ]
+        subprocess.check_call(command)
+        return
+
+    def setDockerCommand(self):
+        if platform.system() == "Linux":
+            self.docker_command = ["sudo"] + self.docker_command
+
+
+class buildAction(argparse.Action):
+    def __call__(self, parser, args, values, option_string=None):
+        cwd = os.path.dirname(os.path.abspath(__file__))
+        subprocess.check_call([cwd + "/build.sh"] + values)
+
+
+class testAction(argparse.Action):
+    def runTest(self, test, cycles="standard"):
+        cwd = os.path.dirname(os.path.abspath(__file__))
+        domsuf = os.getenv('DOMSUF', "localdomain")
+        env = {"NSS_TESTS": test, "NSS_CYCLES": cycles, "DOMSUF": domsuf}
+        command = cwd + "/tests/all.sh"
+        subprocess.check_call(command, env=env)
+
+    def __call__(self, parser, args, values, option_string=None):
+        self.runTest(values)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(
+        description='NSS helper script. ' +
+        'Make sure to separate sub-command arguments with --.')
+    subparsers = parser.add_subparsers()
+
+    parser_build = subparsers.add_parser(
+        'build', help='All arguments are passed to build.sh')
+    parser_build.add_argument(
+        'build_args', nargs='*', help="build arguments", action=buildAction)
+
+    parser_cf = subparsers.add_parser(
+        'clang-format',
+        help='Run clang-format on all folders or provide a folder to format.')
+    parser_cf.add_argument(
+        'cf_args',
+        nargs='*',
+        help="clang-format folders and noroot if you don't want to use sudo",
+        action=cfAction)
+
+    parser_test = subparsers.add_parser(
+        'tests', help='Run tests through tests/all.sh.')
+    tests = [
+        "cipher", "lowhash", "libpkix", "cert", "dbtests", "tools", "fips",
+        "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec",
+        "gtests", "ssl_gtests"
+    ]
+    parser_test.add_argument(
+        'test', choices=tests, help="Available tests", action=testAction)
+    return parser.parse_args()
+
+
+def main():
+    parse_arguments()
+
+
+if __name__ == '__main__':
+    main()