Bug 1246801 - Use docker image v0.0.17, use clang-3.8 for ASan/LSan runs r=me
authorTim Taubert <ttaubert@mozilla.com>
Thu, 30 Jun 2016 07:28:12 +0200
changeset 12346 b2730e41497ef665264ee77346681ec3bc8c8b4e
parent 12345 9e0b4530a2ea66eb69d633643be9be3108f97f26
child 12347 98a35207af80f780de09b5e67570add9ca4b6e7e
push id1360
push userttaubert@mozilla.com
push dateThu, 30 Jun 2016 05:29:08 +0000
reviewersme
bugs1246801
Bug 1246801 - Use docker image v0.0.17, use clang-3.8 for ASan/LSan runs r=me
.taskcluster.yml
automation/taskcluster/docker/Dockerfile
automation/taskcluster/docker/setup.sh
automation/taskcluster/graph/linux/_build_base.yml
automation/taskcluster/graph/linux/_test_base.yml
automation/taskcluster/graph/linux/build32-debug.yml
automation/taskcluster/graph/linux/build32-opt.yml
automation/taskcluster/graph/linux/build64-asan.yml
automation/taskcluster/graph/linux/build64-debug.yml
automation/taskcluster/graph/linux/build64-lsan.yml
automation/taskcluster/graph/linux/build64-opt.yml
automation/taskcluster/graph/tools/clang-format.yml
automation/taskcluster/scripts/build.sh
automation/taskcluster/scripts/run_clang_format.sh
automation/taskcluster/scripts/run_tests.sh
automation/taskcluster/scripts/tools.sh
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -52,17 +52,17 @@ tasks:
       tags:
         createdForUser: {{owner}}
 
       routes:
         - "tc-treeherder-stage.v2.nss.{{revision}}.{{pushlog_id}}"
         - "tc-treeherder.v2.nss.{{revision}}.{{pushlog_id}}"
 
       payload:
-        image: "ttaubert/nss-ci:0.0.16"
+        image: "ttaubert/nss-ci:0.0.17"
 
         env:
           TC_OWNER: {{owner}}
           TC_SOURCE: {{{source}}}
           NSS_PUSHLOG_ID: '{{pushlog_id}}'
           NSS_HEAD_REPOSITORY: '{{{url}}}'
           NSS_HEAD_REVISION: '{{revision}}'
 
--- a/automation/taskcluster/docker/Dockerfile
+++ b/automation/taskcluster/docker/Dockerfile
@@ -1,27 +1,27 @@
 FROM ubuntu:16.04
 MAINTAINER Tim Taubert <ttaubert@mozilla.com>
 
 RUN useradd -d /home/worker -s /bin/bash -m worker
 WORKDIR /home/worker
 
-# Install non-build specific dependencies.
-ADD setup.sh /tmp/setup.sh
-RUN bash /tmp/setup.sh
-
 # Add build and test scripts.
 ADD bin /home/worker/bin
 RUN chmod +x /home/worker/bin/*
 
-# Set variables usually configured at login.
+# Install dependencies.
+ADD setup.sh /tmp/setup.sh
+RUN bash /tmp/setup.sh
+
+# Env variables.
 ENV HOME /home/worker
 ENV SHELL /bin/bash
 ENV USER worker
 ENV LOGNAME worker
 ENV HOSTNAME taskcluster-worker
 ENV LANG en_US.UTF-8
 ENV LC_ALL en_US.UTF-8
-env HOST localhost
-env DOMSUF localdomain
+ENV HOST localhost
+ENV DOMSUF localdomain
 
 # Set a default command for debugging.
 CMD ["/bin/bash", "--login"]
--- a/automation/taskcluster/docker/setup.sh
+++ b/automation/taskcluster/docker/setup.sh
@@ -17,22 +17,16 @@ apt_packages+=('lib32z1-dev')
 apt_packages+=('gcc-multilib')
 apt_packages+=('g++-multilib')
 
 # Install prerequisites.
 apt-get -y update
 export DEBIAN_FRONTEND=noninteractive
 apt-get install -y --no-install-recommends curl apt-utils
 
-# clang(-format)-3.8
-apt_packages+=('clang-3.8')
-apt_packages+=('clang-format-3.8')
-curl http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add -
-echo "deb http://llvm.org/apt/xenial/ llvm-toolchain-xenial-3.8 main" > /etc/apt/sources.list.d/docker.list
-
 # Install the first round of packages.
 apt-get -y update
 apt-get install -y --no-install-recommends ${apt_packages[@]}
 
 # gcc 6
 apt_packages=()
 apt_packages+=('g++-6')
 apt_packages+=('g++-4.8')
@@ -43,19 +37,22 @@ echo "deb http://ppa.launchpad.net/ubunt
 
 # Install the second round of packages.
 apt-get -y update
 apt-get install -y --no-install-recommends ${apt_packages[@]}
 
 # 32-bit builds
 ln -s /usr/include/x86_64-linux-gnu/zconf.h /usr/include
 
+# Install clang-3.8 into /usr/local/.
+curl http://llvm.org/releases/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz | tar xJv -C /usr/local --strip-components=1
+
 # Compiler options.
-update-alternatives --install /usr/bin/gcc gcc /usr/bin/clang-3.8 5
-update-alternatives --install /usr/bin/g++ g++ /usr/bin/clang++-3.8 5
+update-alternatives --install /usr/bin/gcc gcc /usr/local/bin/clang 5
+update-alternatives --install /usr/bin/g++ g++ /usr/local/bin/clang++ 5
 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 10
 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 10
 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-6 20
 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-6 20
 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 30
 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 30
 
 locale-gen en_US.UTF-8
--- a/automation/taskcluster/graph/linux/_build_base.yml
+++ b/automation/taskcluster/graph/linux/_build_base.yml
@@ -9,17 +9,17 @@ task:
   schedulerId: task-graph-scheduler
 
   metadata:
     owner: !env TC_OWNER
     source: !env TC_SOURCE
 
   payload:
     maxRunTime: 3600
-    image: ttaubert/nss-ci:0.0.16
+    image: ttaubert/nss-ci:0.0.17
 
     artifacts:
       public:
         type: directory
         path: /home/worker/artifacts
         expires: !from_now 24
 
     command:
--- a/automation/taskcluster/graph/linux/_test_base.yml
+++ b/automation/taskcluster/graph/linux/_test_base.yml
@@ -9,14 +9,14 @@ task:
   schedulerId: task-graph-scheduler
 
   metadata:
     owner: !env TC_OWNER
     source: !env TC_SOURCE
 
   payload:
     maxRunTime: 3600
-    image: ttaubert/nss-ci:0.0.16
+    image: ttaubert/nss-ci:0.0.17
 
     command:
       - "/bin/bash"
       - "-c"
       - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
--- a/automation/taskcluster/graph/linux/build32-debug.yml
+++ b/automation/taskcluster/graph/linux/build32-debug.yml
@@ -55,18 +55,18 @@
 - task:
     metadata:
       name: "Linux 32 (debug, clang-3.8)"
       description: "Linux 32 (debug, clang-3.8)"
 
     payload:
       env:
         NSS_ENABLE_TLS_1_3: 1
-        GCC_VERSION: clang-3.8
-        GXX_VERSION: clang++-3.8
+        GCC_VERSION: clang
+        GXX_VERSION: clang++
 
     extra:
       treeherder:
         build:
           platform: linux32
         machine:
           platform: linux32
         collection:
--- a/automation/taskcluster/graph/linux/build32-opt.yml
+++ b/automation/taskcluster/graph/linux/build32-opt.yml
@@ -60,18 +60,18 @@
 - task:
     metadata:
       name: "Linux 32 (opt, clang-3.8)"
       description: "Linux 32 (opt, clang-3.8)"
 
     payload:
       env:
         NSS_ENABLE_TLS_1_3: 1
-        GCC_VERSION: clang-3.8
-        GXX_VERSION: clang++-3.8
+        GCC_VERSION: clang
+        GXX_VERSION: clang++
         BUILD_OPT: 1
 
     extra:
       treeherder:
         build:
           platform: linux32
         machine:
           platform: linux32
--- a/automation/taskcluster/graph/linux/build64-asan.yml
+++ b/automation/taskcluster/graph/linux/build64-asan.yml
@@ -1,16 +1,18 @@
 ---
 - task:
     metadata:
       name: "Linux 64 (ASan, debug)"
       description: "Linux 64 (ASan, debug)"
 
     payload:
       env:
+        GCC_VERSION: clang
+        GXX_VERSION: clang++
         NSS_ENABLE_TLS_1_3: 1
         USE_ASAN: 1
         USE_64: 1
 
     extra:
       treeherder:
         build:
           platform: linux64
--- a/automation/taskcluster/graph/linux/build64-debug.yml
+++ b/automation/taskcluster/graph/linux/build64-debug.yml
@@ -61,18 +61,18 @@
 - task:
     metadata:
       name: "Linux 64 (debug, clang-3.8)"
       description: "Linux 64 (debug, clang-3.8)"
 
     payload:
       env:
         NSS_ENABLE_TLS_1_3: 1
-        GCC_VERSION: clang-3.8
-        GXX_VERSION: clang++-3.8
+        GCC_VERSION: clang
+        GXX_VERSION: clang++
         USE_64: 1
 
     extra:
       treeherder:
         build:
           platform: linux64
         machine:
           platform: linux64
--- a/automation/taskcluster/graph/linux/build64-lsan.yml
+++ b/automation/taskcluster/graph/linux/build64-lsan.yml
@@ -1,16 +1,18 @@
 ---
 - task:
     metadata:
       name: "Linux 64 (LSan, debug)"
       description: "Linux 64 (LSan, debug)"
 
     payload:
       env:
+        GCC_VERSION: clang
+        GXX_VERSION: clang++
         NSS_DISABLE_ARENA_FREE_LIST: 1
         NSS_DISABLE_UNLOAD: 1
         NSS_ENABLE_TLS_1_3: 1
         NSS_ENABLE_LSAN: 1
         USE_ASAN: 1
         USE_64: 1
 
     extra:
--- a/automation/taskcluster/graph/linux/build64-opt.yml
+++ b/automation/taskcluster/graph/linux/build64-opt.yml
@@ -62,18 +62,18 @@
 - task:
     metadata:
       name: "Linux 64 (opt, clang-3.8)"
       description: "Linux 64 (opt, clang-3.8)"
 
     payload:
       env:
         NSS_ENABLE_TLS_1_3: 1
-        GCC_VERSION: clang-3.8
-        GXX_VERSION: clang++-3.8
+        GCC_VERSION: clang
+        GXX_VERSION: clang++
         BUILD_OPT: 1
         USE_64: 1
 
     extra:
       treeherder:
         build:
           platform: linux64
         machine:
--- a/automation/taskcluster/graph/tools/clang-format.yml
+++ b/automation/taskcluster/graph/tools/clang-format.yml
@@ -10,17 +10,17 @@
     metadata:
       owner: !env TC_OWNER
       source: !env TC_SOURCE
       name: clang-format-3.8
       description: clang-format-3.8
 
     payload:
       maxRunTime: 3600
-      image: ttaubert/nss-ci:0.0.16
+      image: ttaubert/nss-ci:0.0.17
 
       command:
         - "/bin/bash"
         - "-c"
         - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh nss/lib/ssl"
 
       env:
         NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY
--- a/automation/taskcluster/scripts/build.sh
+++ b/automation/taskcluster/scripts/build.sh
@@ -1,33 +1,31 @@
 #!/usr/bin/env bash
 
 set -v -e -x
 
 if [ $(id -u) = 0 ]; then
-    # Switch compilers.
-    GCC=${GCC_VERSION:-gcc-5}
-    GXX=${GXX_VERSION:-g++-5}
+    source $(dirname $0)/tools.sh
 
-    update-alternatives --set gcc "/usr/bin/$GCC"
-    update-alternatives --set g++ "/usr/bin/$GXX"
+    # Set compiler.
+    switch_compilers
 
     # Drop privileges by re-running this script.
     exec su worker $0
 fi
 
 # Clone NSPR if needed.
 if [ ! -d "nspr" ]; then
     hg clone https://hg.mozilla.org/projects/nspr
 fi
 
 # Build.
-cd nss && make nss_build_all
+cd nss && make nss_build_all && cd ..
 
 # Generate certificates.
-cd tests && NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" ./all.sh
+NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" $(dirname $0)/run_tests.sh
 
 # Reset test counter so that test runs pick up our certificates.
-cd && echo 1 > tests_results/security/localhost
+echo 1 > tests_results/security/localhost
 
 # Package.
 mkdir artifacts
 tar cvfjh artifacts/dist.tar.bz2 dist tests_results
--- a/automation/taskcluster/scripts/run_clang_format.sh
+++ b/automation/taskcluster/scripts/run_clang_format.sh
@@ -8,14 +8,14 @@ if [ $(id -u) = 0 ]; then
 fi
 
 # Apply clang-format 3.8 on the provided folder and verify that this doesn't change any file.
 # If any file differs after formatting, the script eventually exits with 1.
 # Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong.
 
 STATUS=0
 for i in $(find $1 -type f -name '*.[ch]' -print); do
-    if ! clang-format-3.8 $i | diff -Naur $i -; then
+    if ! clang-format $i | diff -Naur $i -; then
         echo "Sorry, $i is not formatted properly. Please use clang-format 3.8 on your patch before landing."
         STATUS=1
     fi
 done
 exit $STATUS
--- a/automation/taskcluster/scripts/run_tests.sh
+++ b/automation/taskcluster/scripts/run_tests.sh
@@ -1,19 +1,17 @@
 #!/usr/bin/env bash
 
 set -v -e -x
 
 if [ $(id -u) = 0 ]; then
-    # Switch compilers.
-    GCC=${GCC_VERSION:-gcc-5}
-    GXX=${GXX_VERSION:-g++-5}
+    source $(dirname $0)/tools.sh
 
-    update-alternatives --set gcc "/usr/bin/$GCC"
-    update-alternatives --set g++ "/usr/bin/$GXX"
+    # Set compiler.
+    switch_compilers
 
     # Stupid Docker.
     echo "127.0.0.1 localhost.localdomain" >> /etc/hosts
 
     # Drop privileges by re-running this script.
     exec su worker $0
 fi
 
new file mode 100644
--- /dev/null
+++ b/automation/taskcluster/scripts/tools.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+switch_compilers() {
+    GCC=`which ${GCC_VERSION:-gcc-5}`
+    GXX=`which ${GXX_VERSION:-g++-5}`
+
+    if [ -e "$GCC" ] && [ -e "$GXX" ]; then
+        update-alternatives --set gcc $GCC
+        update-alternatives --set g++ $GXX
+    else
+        echo "Unknown compiler $GCC_VERSION/$GXX_VERSION."
+        exit 1
+    fi
+}