Bug 1372127 - mach command to run clang-format, build, and tests, r=mt
Differential Revision: https://nss-review.dev.mozaws.net/D300
new file mode 100644
--- /dev/null
+++ b/automation/clang-format/Dockerfile
@@ -0,0 +1,26 @@
+FROM ubuntu:16.04
+MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com>
+
+RUN useradd -d /home/worker -s /bin/bash -m worker
+WORKDIR /home/worker
+
+# Install dependencies.
+ADD setup.sh /tmp/setup.sh
+RUN bash /tmp/setup.sh
+
+# Change user.
+USER worker
+
+# Env variables.
+ENV HOME /home/worker
+ENV SHELL /bin/bash
+ENV USER worker
+ENV LOGNAME worker
+ENV HOSTNAME taskcluster-worker
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV HOST localhost
+ENV DOMSUF localdomain
+
+# Entrypoint.
+ENTRYPOINT ["/home/worker/nss/automation/clang-format/run_clang_format.sh"]
new file mode 100755
--- /dev/null
+++ b/automation/clang-format/run_clang_format.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+if [[ $(id -u) -eq 0 ]]; then
+ # Drop privileges by re-running this script.
+ # Note: this mangles arguments, better to avoid running scripts as root.
+ exec su worker -c "$0 $*"
+fi
+
+# Apply clang-format on the provided folder and verify that this doesn't change any file.
+# If any file differs after formatting, the script eventually exits with 1.
+# Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong.
+
+# Includes a default set of directories NOT to clang-format on.
+blacklist=(
+ "./automation" \
+ "./coreconf" \
+ "./doc" \
+ "./pkg" \
+ "./tests" \
+ "./lib/libpkix" \
+ "./lib/zlib" \
+ "./lib/sqlite" \
+ "./gtests/google_test" \
+ "./.hg" \
+)
+
+top="$(dirname $0)/../.."
+cd "$top"
+
+if [ $# -gt 0 ]; then
+ dirs=("$@")
+else
+ dirs=($(find . -maxdepth 2 -mindepth 1 -type d ! -path . \( ! -regex '.*/' \)))
+fi
+
+format_folder()
+{
+ for black in "${blacklist[@]}"; do
+ if [[ "$1" == "$black"* ]]; then
+ echo "skip $1"
+ return 1
+ fi
+ done
+ return 0
+}
+
+for dir in "${dirs[@]}"; do
+ if format_folder "$dir" ; then
+ c="${dir//[^\/]}"
+ echo "formatting $dir ..."
+ depth=""
+ if [ "${#c}" == "1" ]; then
+ depth="-maxdepth 1"
+ fi
+ find "$dir" $depth -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+
+ fi
+done
+
+TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXX)
+trap 'rm $TMPFILE' exit
+if (cd $(dirname $0); hg root >/dev/null 2>&1); then
+ hg diff --git "$top" | tee $TMPFILE
+else
+ git -C "$top" diff | tee $TMPFILE
+fi
+[[ ! -s $TMPFILE ]]
new file mode 100644
--- /dev/null
+++ b/automation/clang-format/setup.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Update packages.
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y update && apt-get -y upgrade
+
+# Install packages.
+apt_packages=()
+apt_packages+=('ca-certificates')
+apt_packages+=('curl')
+apt_packages+=('xz-utils')
+apt_packages+=('mercurial')
+apt_packages+=('git')
+apt-get install -y --no-install-recommends ${apt_packages[@]}
+
+# Download clang.
+curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -o clang.tar.xz
+curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig -o clang.tar.xz.sig
+# Verify the signature.
+gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
+gpg --verify clang.tar.xz.sig
+# Install into /usr/local/.
+tar xJvf *.tar.xz -C /usr/local --strip-components=1
+
+# Cleanup.
+function cleanup() {
+ rm -f clang.tar.xz clang.tar.xz.sig
+}
+trap cleanup ERR EXIT
+
+locale-gen en_US.UTF-8
+dpkg-reconfigure locales
+
+# Cleanup.
+rm -rf ~/.ccache ~/.cache
+apt-get autoremove -y
+apt-get clean
+apt-get autoclean
+
+# We're done. Remove this script.
+rm $0
--- a/automation/taskcluster/graph/src/extend.js
+++ b/automation/taskcluster/graph/src/extend.js
@@ -741,17 +741,17 @@ async function scheduleTools() {
queue.scheduleTask(merge(base, {
symbol: "clang-format-3.9",
name: "clang-format-3.9",
image: LINUX_CLANG39_IMAGE,
command: [
"/bin/bash",
"-c",
- "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh"
+ "bin/checkout.sh && nss/automation/clang-format/run_clang_format.sh"
]
}));
queue.scheduleTask(merge(base, {
symbol: "scan-build-4.0",
name: "scan-build-4.0",
image: LINUX_IMAGE,
env: {
deleted file mode 100755
--- a/automation/taskcluster/scripts/run_clang_format.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-
-source $(dirname "$0")/tools.sh
-
-set +x
-
-# Apply clang-format on the provided folder and verify that this doesn't change any file.
-# If any file differs after formatting, the script eventually exits with 1.
-# Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong.
-
-# Includes a default set of directories NOT to clang-format on.
-blacklist=(
- "./automation" \
- "./coreconf" \
- "./doc" \
- "./pkg" \
- "./tests" \
- "./lib/libpkix" \
- "./lib/zlib" \
- "./lib/sqlite" \
- "./gtests/google_test" \
- "./.hg" \
-)
-
-top="$PWD/$(dirname $0)/../../.."
-cd "$top"
-
-if [ $# -gt 0 ]; then
- dirs=("$@")
-else
- dirs=($(find . ! -path . \( ! -regex '.*/' \) -maxdepth 2 -mindepth 1 -type d))
-fi
-
-format_folder()
-{
- for black in "${blacklist[@]}"; do
- if [[ "$1" == "$black"* ]]; then
- echo "skip $1"
- return 1
- fi
- done
- return 0
-}
-
-for dir in "${dirs[@]}"; do
- if format_folder "$dir" ; then
- c="${dir//[^\/]}"
- echo "formatting $dir ..."
- depth=""
- if [ "${#c}" == "1" ]; then
- depth="-maxdepth 1"
- fi
- find "$dir" $depth -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+
- fi
-done
-
-TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXX)
-trap 'rm $TMPFILE' exit
-if (cd $(dirname $0); hg root >/dev/null 2>&1); then
- hg diff --git "$top" | tee $TMPFILE
-else
- git -C "$top" diff | tee $TMPFILE
-fi
-[[ ! -s $TMPFILE ]]
new file mode 100755
--- /dev/null
+++ b/mach
@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+################################################################################
+#
+# This is a collection of helper tools to get stuff done in NSS.
+#
+
+import sys
+import argparse
+import subprocess
+import os
+import platform
+from hashlib import sha256
+
+cwd = os.path.dirname(os.path.abspath(__file__))
+
+
+class cfAction(argparse.Action):
+ docker_command = ["docker"]
+
+ def __call__(self, parser, args, values, option_string=None):
+ if "noroot" not in values:
+ self.setDockerCommand()
+ else:
+ values.remove("noroot")
+
+ # First check if we can run docker.
+ try:
+ with open(os.devnull, "w") as f:
+ subprocess.check_call(
+ self.docker_command + ["images"], stdout=f)
+ except:
+ print("Please install docker and start the docker daemon.")
+ sys.exit(1)
+
+ docker_image = 'clang-format-service:latest'
+ cf_docker_folder = cwd + "/automation/clang-format"
+
+ # Build the image if necessary.
+ if self.filesChanged(cf_docker_folder):
+ self.buildImage(docker_image, cf_docker_folder)
+
+ # Check if we have the docker image.
+ try:
+ command = self.docker_command + [
+ "image", "inspect", "clang-format-service:latest"
+ ]
+ with open(os.devnull, "w") as f:
+ subprocess.check_call(command, stdout=f)
+ except:
+ print("I have to build the docker image first.")
+ self.buildImage(docker_image, cf_docker_folder)
+
+ command = self.docker_command + [
+ 'run', '-v', cwd + ':/home/worker/nss', '--rm', '-ti', docker_image
+ ]
+ # The clang format script returns 1 if something's to do. We don't care.
+ subprocess.call(command + values)
+
+ def filesChanged(self, path):
+ hash = sha256()
+ for dirname, dirnames, files in os.walk(path):
+ for file in files:
+ with open(os.path.join(dirname, file), "rb") as f:
+ hash.update(f.read())
+ chk_file = cwd + "/out/.chk"
+ old_chk = ""
+ new_chk = hash.hexdigest()
+ if os.path.exists(chk_file):
+ with open(chk_file) as f:
+ old_chk = f.readline()
+ if old_chk != new_chk:
+ with open(chk_file, "w+") as f:
+ f.write(new_chk)
+ return True
+ return False
+
+ def buildImage(self, docker_image, cf_docker_folder):
+ command = self.docker_command + [
+ "build", "-t", docker_image, cf_docker_folder
+ ]
+ subprocess.check_call(command)
+ return
+
+ def setDockerCommand(self):
+ if platform.system() == "Linux":
+ self.docker_command = ["sudo"] + self.docker_command
+
+
+class buildAction(argparse.Action):
+ def __call__(self, parser, args, values, option_string=None):
+ cwd = os.path.dirname(os.path.abspath(__file__))
+ subprocess.check_call([cwd + "/build.sh"] + values)
+
+
+class testAction(argparse.Action):
+ def runTest(self, test, cycles="standard"):
+ cwd = os.path.dirname(os.path.abspath(__file__))
+ domsuf = os.getenv('DOMSUF', "localdomain")
+ env = {"NSS_TESTS": test, "NSS_CYCLES": cycles, "DOMSUF": domsuf}
+ command = cwd + "/tests/all.sh"
+ subprocess.check_call(command, env=env)
+
+ def __call__(self, parser, args, values, option_string=None):
+ self.runTest(values)
+
+
+def parse_arguments():
+ parser = argparse.ArgumentParser(
+ description='NSS helper script. ' +
+ 'Make sure to separate sub-command arguments with --.')
+ subparsers = parser.add_subparsers()
+
+ parser_build = subparsers.add_parser(
+ 'build', help='All arguments are passed to build.sh')
+ parser_build.add_argument(
+ 'build_args', nargs='*', help="build arguments", action=buildAction)
+
+ parser_cf = subparsers.add_parser(
+ 'clang-format',
+ help='Run clang-format on all folders or provide a folder to format.')
+ parser_cf.add_argument(
+ 'cf_args',
+ nargs='*',
+ help="clang-format folders and noroot if you don't want to use sudo",
+ action=cfAction)
+
+ parser_test = subparsers.add_parser(
+ 'tests', help='Run tests through tests/all.sh.')
+ tests = [
+ "cipher", "lowhash", "libpkix", "cert", "dbtests", "tools", "fips",
+ "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec",
+ "gtests", "ssl_gtests"
+ ]
+ parser_test.add_argument(
+ 'test', choices=tests, help="Available tests", action=testAction)
+ return parser.parse_args()
+
+
+def main():
+ parse_arguments()
+
+
+if __name__ == '__main__':
+ main()