Bug 1329307 - Only package the clang-tidy binaries for the clang-tidy builds; r=mystor
authorEhsan Akhgari <ehsan@mozilla.com>
Mon, 02 Jan 2017 22:23:08 -0500
changeset 356439 c05634d71a73b5b5f58d90b6dc0241e373d88d02
parent 356438 d6dd74d3c794b9dd56d1890bab199b496b6dbf16
child 356440 e11584ada439d4a9525b75f01e6135d1ccf83d2a
push id10621
push userjlund@mozilla.com
push dateMon, 23 Jan 2017 16:02:43 +0000
treeherdermozilla-aurora@dca7b42e6c67 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmystor
bugs1329307
milestone53.0a1
Bug 1329307 - Only package the clang-tidy binaries for the clang-tidy builds; r=mystor
build/build-clang/README
build/build-clang/build-clang.py
build/build-clang/clang-tidy-linux64.json
build/build-clang/clang-tidy-macosx64.json
build/build-clang/clang-tidy-win32.json
build/build-clang/clang-tidy-win64.json
taskcluster/scripts/misc/build-clang-tidy-linux.sh
taskcluster/scripts/misc/build-clang-tidy-macosx.sh
taskcluster/scripts/misc/build-clang-windows-helper32.sh
taskcluster/scripts/misc/build-clang-windows-helper64.sh
--- a/build/build-clang/README
+++ b/build/build-clang/README
@@ -40,17 +40,17 @@ build-clang.py accepts a JSON config for
 * cc: Path to the bootsraping C Compiler.
 * cxx: Path to the bootsraping C++ Compiler.
 * ar: Path to the library archiver tool.
 * ranlib: Path to the ranlib tool.
 * ld: Path to the linker.
 * patches: Optional list of patches to apply per platform.  Supported platforms: macosx64, linux32, linux64.  The default is Release.
 * build_type: The type of build to make.  Supported types: Release, Debug, RelWithDebInfo or MinSizeRel.
 * build_libcxx: Whether to build with libcxx.  The default is false.
-* import_clang_tidy: Whether to import Mozilla checks into clang-tidy before building.  The default is false.
+* build_clang_tidy: Whether to build clang-tidy with the Mozilla checks imported.  The default is false.
 * osx_cross_compile: Whether to invoke CMake for OS X cross compile builds.
 * assertions: Whether to enable LLVM assertions.  The default is false.
 
 Environment Variables
 ---------------------
 
 The following environment variables are used for cross-compile builds targeting OS X on Linux.
 
--- a/build/build-clang/build-clang.py
+++ b/build/build-clang/build-clang.py
@@ -52,17 +52,17 @@ def run_in(path, args):
 
 
 def patch(patch, srcdir):
     patch = os.path.realpath(patch)
     check_run(['patch', '-d', srcdir, '-p1', '-i', patch, '--fuzz=0',
                '-s'])
 
 
-def do_import_clang_tidy(source_dir):
+def import_clang_tidy(source_dir):
     clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]),
                                      '..', 'clang-plugin')
     clang_tidy_path = os.path.join(source_dir,
                                    'tools/clang/tools/extra/clang-tidy')
     sys.path.append(clang_plugin_path)
     from import_mozilla_checks import do_import
     do_import(clang_plugin_path, clang_tidy_path)
 
@@ -123,16 +123,26 @@ def copy_dir_contents(src, dest):
 def mkdir_p(path):
     try:
         os.makedirs(path)
     except OSError as e:
         if e.errno != errno.EEXIST or not os.path.isdir(path):
             raise
 
 
+def delete(path):
+    if os.path.isdir(path):
+        shutil.rmtree(path)
+    else:
+        try:
+            os.unlink(path)
+        except:
+            pass
+
+
 def install_libgcc(gcc_dir, clang_dir):
     out = subprocess.check_output([os.path.join(gcc_dir, "bin", "gcc"),
                                    '-print-libgcc-file-name'])
 
     libgcc_dir = os.path.dirname(out.rstrip())
     clang_lib_dir = os.path.join(clang_dir, "lib", "gcc",
                                  "x86_64-unknown-linux-gnu",
                                  os.path.basename(libgcc_dir))
@@ -276,16 +286,82 @@ def get_tool(config, key):
             return f
 
     # Assume that we have the name of some program that should be on PATH.
     try:
         return which.which(f) if f else which.which(key)
     except which.WhichError:
         raise ValueError("%s not found on PATH" % f)
 
+
+# This function is intended to be called on the final build directory when
+# building clang-tidy.  Its job is to remove all of the files which won't
+# be used for clang-tidy to reduce the download size.  Currently when this
+# function finishes its job, it will leave final_dir with a layout like this:
+#
+# clang/
+#   bin/
+#     clang-tidy
+#   include/
+#     * (nothing will be deleted here)
+#   lib/
+#     clang/
+#       4.0.0/
+#         include/
+#           * (nothing will be deleted here)
+#   share/
+#     clang/
+#       clang-tidy-diff.py
+#       run-clang-tidy.py
+def prune_final_dir_for_clang_tidy(final_dir):
+    # Make sure we only have what we expect.
+    dirs = ("bin", "include", "lib", "libexec", "msbuild-bin", "share", "tools")
+    for f in glob.glob("%s/*" % final_dir):
+        if os.path.basename(f) not in dirs:
+            raise Exception("Found unknown file %s in the final directory" % f)
+        if not os.path.isdir(f):
+            raise Exception("Expected %s to be a directory" %f)
+
+    # In bin/, only keep clang-tidy.
+    re_clang_tidy = re.compile(r"^clang-tidy(\.exe)?$", re.I)
+    for f in glob.glob("%s/bin/*" % final_dir):
+        if re_clang_tidy.search(os.path.basename(f)) is None:
+            delete(f)
+
+    # Keep include/ intact.
+
+    # In lib/, only keep lib/clang/N.M.O/include.
+    re_ver_num = re.compile(r"^\d+\.\d+\.\d+$", re.I)
+    for f in glob.glob("%s/lib/*" % final_dir):
+        if os.path.basename(f) != "clang":
+            delete(f)
+    for f in glob.glob("%s/lib/clang/*" % final_dir):
+        if re_ver_num.search(os.path.basename(f)) is None:
+            delete(f)
+    for f in glob.glob("%s/lib/clang/*/*" % final_dir):
+        if os.path.basename(f) != "include":
+            delete(f)
+
+    # Completely remove libexec/, msbuilld-bin and tools, if it exists.
+    shutil.rmtree(os.path.join(final_dir, "libexec"))
+    for d in ("msbuild-bin", "tools"):
+        d = os.path.join(final_dir, d)
+        if os.path.exists(d):
+            shutil.rmtree(d)
+
+    # In share/, only keep share/clang/*tidy*
+    re_clang_tidy = re.compile(r"tidy", re.I)
+    for f in glob.glob("%s/share/*" % final_dir):
+        if os.path.basename(f) != "clang":
+            delete(f)
+    for f in glob.glob("%s/share/clang/*" % final_dir):
+        if re_clang_tidy.search(os.path.basename(f)) is None:
+            delete(f)
+
+
 if __name__ == "__main__":
     # The directories end up in the debug info, so the easy way of getting
     # a reproducible build is to run it in a know absolute directory.
     # We use a directory in /builds/slave because the mozilla infrastructure
     # cleans it up automatically.
     base_dir = "/builds/slave/moz-toolchain"
     if is_windows():
         # TODO: Because Windows taskcluster builds are run with distinct
@@ -353,21 +429,21 @@ if __name__ == "__main__":
         build_type = config["build_type"]
         if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
             raise ValueError("We only know how to do Release, Debug, RelWithDebInfo or MinSizeRel builds")
     build_libcxx = False
     if "build_libcxx" in config:
         build_libcxx = config["build_libcxx"]
         if build_libcxx not in (True, False):
             raise ValueError("Only boolean values are accepted for build_libcxx.")
-    import_clang_tidy = False
-    if "import_clang_tidy" in config:
-        import_clang_tidy = config["import_clang_tidy"]
-        if import_clang_tidy not in (True, False):
-            raise ValueError("Only boolean values are accepted for import_clang_tidy.")
+    build_clang_tidy = False
+    if "build_clang_tidy" in config:
+        build_clang_tidy = config["build_clang_tidy"]
+        if build_clang_tidy not in (True, False):
+            raise ValueError("Only boolean values are accepted for build_clang_tidy.")
     osx_cross_compile = False
     if "osx_cross_compile" in config:
         osx_cross_compile = config["osx_cross_compile"]
         if osx_cross_compile not in (True, False):
             raise ValueError("Only boolean values are accepted for osx_cross_compile.")
         if osx_cross_compile and not is_linux():
             raise ValueError("osx_cross_compile can only be used on Linux.")
     assertions = False
@@ -432,25 +508,22 @@ if __name__ == "__main__":
                 (source_dir + "/libcxx",
                  llvm_source_dir + "/projects/libcxx"),
                 (source_dir + "/libcxxabi",
                  llvm_source_dir + "/projects/libcxxabi")]
     for l in symlinks:
         # On Windows, we have to re-copy the whole directory every time.
         if not is_windows() and os.path.islink(l[1]):
             continue
-        if os.path.isdir(l[1]):
-            shutil.rmtree(l[1])
-        elif os.path.exists(l[1]):
-            os.unlink(l[1])
+        delete(l[1]);
         if os.path.exists(l[0]):
             symlink(l[0], l[1])
 
-    if import_clang_tidy:
-        do_import_clang_tidy(llvm_source_dir)
+    if build_clang_tidy:
+        import_clang_tidy(llvm_source_dir)
 
     if not os.path.exists(build_dir):
         os.makedirs(build_dir)
 
     stage1_dir = build_dir + '/stage1'
     stage1_inst_dir = stage1_dir + '/clang'
 
     final_stage_dir = stage1_dir
@@ -533,12 +606,17 @@ if __name__ == "__main__":
                 (cc_name, exe_ext)] + extra_cflags2,
             [stage2_inst_dir + "/bin/%s%s" %
                 (cxx_name, exe_ext)] + extra_cxxflags2,
             [ld] + extra_ldflags,
             ar, ranlib,
             llvm_source_dir, stage3_dir, build_libcxx, osx_cross_compile,
             build_type, assertions, python_path, gcc_dir)
 
+    package_name = "clang"
+    if build_clang_tidy:
+        prune_final_dir_for_clang_tidy(os.path.join(final_stage_dir, "clang"))
+        package_name = "clang-tidy"
+
     if is_darwin() or is_windows():
-        build_tar_package("tar", "clang.tar.bz2", final_stage_dir, "clang")
+        build_tar_package("tar", package_name + ".tar.bz2", final_stage_dir, "clang")
     else:
-        build_tar_package("tar", "clang.tar.xz", final_stage_dir, "clang")
+        build_tar_package("tar", package_name + ".tar.xz", final_stage_dir, "clang")
--- a/build/build-clang/clang-tidy-linux64.json
+++ b/build/build-clang/clang-tidy-linux64.json
@@ -1,15 +1,15 @@
 {
     "llvm_revision": "290055",
     "stages": "1",
     "build_libcxx": true,
     "build_type": "Release",
     "assertions": false,
-    "import_clang_tidy": true,
+    "build_clang_tidy": true,
     "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
     "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
     "extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/trunk",
     "python_path": "/usr/bin/python2.7",
     "gcc_dir": "/home/worker/workspace/build/src/gcc",
--- a/build/build-clang/clang-tidy-macosx64.json
+++ b/build/build-clang/clang-tidy-macosx64.json
@@ -1,15 +1,15 @@
 {
     "llvm_revision": "290055",
     "stages": "1",
     "build_libcxx": true,
     "build_type": "Release",
     "assertions": false,
-    "import_clang_tidy": true,
+    "build_clang_tidy": true,
     "osx_cross_compile": true,
     "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
     "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
     "extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/trunk",
     "python_path": "/usr/bin/python2.7",
--- a/build/build-clang/clang-tidy-win32.json
+++ b/build/build-clang/clang-tidy-win32.json
@@ -1,15 +1,15 @@
 {
     "llvm_revision": "290055",
     "stages": "1",
     "build_libcxx": false,
     "build_type": "Release",
     "assertions": false,
-    "import_clang_tidy": true,
+    "build_clang_tidy": true,
     "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
     "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
     "extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "python_path": "c:/mozilla-build/python/python.exe",
     "cc": "cl.exe",
     "cxx": "cl.exe"
--- a/build/build-clang/clang-tidy-win64.json
+++ b/build/build-clang/clang-tidy-win64.json
@@ -1,15 +1,15 @@
 {
     "llvm_revision": "290055",
     "stages": "1",
     "build_libcxx": false,
     "build_type": "Release",
     "assertions": false,
-    "import_clang_tidy": true,
+    "build_clang_tidy": true,
     "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
     "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
     "extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "python_path": "c:/mozilla-build/python/python.exe",
     "cc": "cl.exe",
     "cxx": "cl.exe"
--- a/taskcluster/scripts/misc/build-clang-tidy-linux.sh
+++ b/taskcluster/scripts/misc/build-clang-tidy-linux.sh
@@ -18,9 +18,9 @@ set +x
 cd build/build-clang
 # |mach python| sets up a virtualenv for us!
 ../../mach python ./build-clang.py -c clang-tidy-linux64.json
 
 set -x
 
 # Put a tarball in the artifacts dir
 mkdir -p $UPLOAD_DIR
-cp clang.tar.* $UPLOAD_DIR
+cp clang-tidy.tar.* $UPLOAD_DIR
--- a/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
+++ b/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
@@ -27,9 +27,9 @@ set +x
 cd build/build-clang
 # |mach python| sets up a virtualenv for us!
 ../../mach python ./build-clang.py -c clang-tidy-macosx64.json
 
 set -x
 
 # Put a tarball in the artifacts dir
 mkdir -p $UPLOAD_DIR
-cp clang.tar.* $UPLOAD_DIR
+cp clang-tidy.tar.* $UPLOAD_DIR
--- a/taskcluster/scripts/misc/build-clang-windows-helper32.sh
+++ b/taskcluster/scripts/misc/build-clang-windows-helper32.sh
@@ -51,9 +51,9 @@ set +x
 BUILD_CLANG_DIR=build/src/build/build-clang
 MOZCONFIG=${MOZCONFIG} build/src/mach python ${BUILD_CLANG_DIR}/build-clang.py -c ${BUILD_CLANG_DIR}/${1}
 
 set -x
 
 # Put a tarball in the artifacts dir
 UPLOAD_PATH=public/build
 mkdir -p ${UPLOAD_PATH}
-cp clang.tar.* ${UPLOAD_PATH}
+cp clang*.tar.* ${UPLOAD_PATH}
--- a/taskcluster/scripts/misc/build-clang-windows-helper64.sh
+++ b/taskcluster/scripts/misc/build-clang-windows-helper64.sh
@@ -50,9 +50,9 @@ set +x
 BUILD_CLANG_DIR=build/src/build/build-clang
 MOZCONFIG=${MOZCONFIG} build/src/mach python ${BUILD_CLANG_DIR}/build-clang.py -c ${BUILD_CLANG_DIR}/${1}
 
 set -x
 
 # Put a tarball in the artifacts dir
 UPLOAD_PATH=public/build
 mkdir -p ${UPLOAD_PATH}
-cp clang.tar.* ${UPLOAD_PATH}
+cp clang*.tar.* ${UPLOAD_PATH}