Bug 1329307 - Only package the clang-tidy binaries for the clang-tidy builds; r=mystor
authorEhsan Akhgari <ehsan@mozilla.com>
Mon, 02 Jan 2017 22:23:08 -0500
changeset 373415 c05634d71a73b5b5f58d90b6dc0241e373d88d02
parent 373414 d6dd74d3c794b9dd56d1890bab199b496b6dbf16
child 373416 e11584ada439d4a9525b75f01e6135d1ccf83d2a
push id6996
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 20:48:21 +0000
treeherdermozilla-beta@d89512dab048 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmystor
bugs1329307
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1329307 - Only package the clang-tidy binaries for the clang-tidy builds; r=mystor
build/build-clang/README
build/build-clang/build-clang.py
build/build-clang/clang-tidy-linux64.json
build/build-clang/clang-tidy-macosx64.json
build/build-clang/clang-tidy-win32.json
build/build-clang/clang-tidy-win64.json
taskcluster/scripts/misc/build-clang-tidy-linux.sh
taskcluster/scripts/misc/build-clang-tidy-macosx.sh
taskcluster/scripts/misc/build-clang-windows-helper32.sh
taskcluster/scripts/misc/build-clang-windows-helper64.sh
--- a/build/build-clang/README
+++ b/build/build-clang/README
@@ -40,17 +40,17 @@ build-clang.py accepts a JSON config for
 * cc: Path to the bootsraping C Compiler.
 * cxx: Path to the bootsraping C++ Compiler.
 * ar: Path to the library archiver tool.
 * ranlib: Path to the ranlib tool.
 * ld: Path to the linker.
 * patches: Optional list of patches to apply per platform.  Supported platforms: macosx64, linux32, linux64.  The default is Release.
 * build_type: The type of build to make.  Supported types: Release, Debug, RelWithDebInfo or MinSizeRel.
 * build_libcxx: Whether to build with libcxx.  The default is false.
-* import_clang_tidy: Whether to import Mozilla checks into clang-tidy before building.  The default is false.
+* build_clang_tidy: Whether to build clang-tidy with the Mozilla checks imported.  The default is false.
 * osx_cross_compile: Whether to invoke CMake for OS X cross compile builds.
 * assertions: Whether to enable LLVM assertions.  The default is false.
 
 Environment Variables
 ---------------------
 
 The following environment variables are used for cross-compile builds targeting OS X on Linux.
 
--- a/build/build-clang/build-clang.py
+++ b/build/build-clang/build-clang.py
@@ -52,17 +52,17 @@ def run_in(path, args):
 
 
 def patch(patch, srcdir):
     patch = os.path.realpath(patch)
     check_run(['patch', '-d', srcdir, '-p1', '-i', patch, '--fuzz=0',
                '-s'])
 
 
-def do_import_clang_tidy(source_dir):
+def import_clang_tidy(source_dir):
     clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]),
                                      '..', 'clang-plugin')
     clang_tidy_path = os.path.join(source_dir,
                                    'tools/clang/tools/extra/clang-tidy')
     sys.path.append(clang_plugin_path)
     from import_mozilla_checks import do_import
     do_import(clang_plugin_path, clang_tidy_path)
 
@@ -123,16 +123,26 @@ def copy_dir_contents(src, dest):
 def mkdir_p(path):
     try:
         os.makedirs(path)
     except OSError as e:
         if e.errno != errno.EEXIST or not os.path.isdir(path):
             raise
 
 
+def delete(path):
+    if os.path.isdir(path):
+        shutil.rmtree(path)
+    else:
+        try:
+            os.unlink(path)
+        except:
+            pass
+
+
 def install_libgcc(gcc_dir, clang_dir):
     out = subprocess.check_output([os.path.join(gcc_dir, "bin", "gcc"),
                                    '-print-libgcc-file-name'])
 
     libgcc_dir = os.path.dirname(out.rstrip())
     clang_lib_dir = os.path.join(clang_dir, "lib", "gcc",
                                  "x86_64-unknown-linux-gnu",
                                  os.path.basename(libgcc_dir))
@@ -276,16 +286,82 @@ def get_tool(config, key):
             return f
 
     # Assume that we have the name of some program that should be on PATH.
     try:
         return which.which(f) if f else which.which(key)
     except which.WhichError:
         raise ValueError("%s not found on PATH" % f)
 
+
+# This function is intended to be called on the final build directory when
+# building clang-tidy.  Its job is to remove all of the files which won't
+# be used for clang-tidy to reduce the download size.  Currently when this
+# function finishes its job, it will leave final_dir with a layout like this:
+#
+# clang/
+#   bin/
+#     clang-tidy
+#   include/
+#     * (nothing will be deleted here)
+#   lib/
+#     clang/
+#       4.0.0/
+#         include/
+#           * (nothing will be deleted here)
+#   share/
+#     clang/
+#       clang-tidy-diff.py
+#       run-clang-tidy.py
+def prune_final_dir_for_clang_tidy(final_dir):
+    # Make sure we only have what we expect.
+    dirs = ("bin", "include", "lib", "libexec", "msbuild-bin", "share", "tools")
+    for f in glob.glob("%s/*" % final_dir):
+        if os.path.basename(f) not in dirs:
+            raise Exception("Found unknown file %s in the final directory" % f)
+        if not os.path.isdir(f):
+            raise Exception("Expected %s to be a directory" %f)
+
+    # In bin/, only keep clang-tidy.
+    re_clang_tidy = re.compile(r"^clang-tidy(\.exe)?$", re.I)
+    for f in glob.glob("%s/bin/*" % final_dir):
+        if re_clang_tidy.search(os.path.basename(f)) is None:
+            delete(f)
+
+    # Keep include/ intact.
+
+    # In lib/, only keep lib/clang/N.M.O/include.
+    re_ver_num = re.compile(r"^\d+\.\d+\.\d+$", re.I)
+    for f in glob.glob("%s/lib/*" % final_dir):
+        if os.path.basename(f) != "clang":
+            delete(f)
+    for f in glob.glob("%s/lib/clang/*" % final_dir):
+        if re_ver_num.search(os.path.basename(f)) is None:
+            delete(f)
+    for f in glob.glob("%s/lib/clang/*/*" % final_dir):
+        if os.path.basename(f) != "include":
+            delete(f)
+
+    # Completely remove libexec/, msbuilld-bin and tools, if it exists.
+    shutil.rmtree(os.path.join(final_dir, "libexec"))
+    for d in ("msbuild-bin", "tools"):
+        d = os.path.join(final_dir, d)
+        if os.path.exists(d):
+            shutil.rmtree(d)
+
+    # In share/, only keep share/clang/*tidy*
+    re_clang_tidy = re.compile(r"tidy", re.I)
+    for f in glob.glob("%s/share/*" % final_dir):
+        if os.path.basename(f) != "clang":
+            delete(f)
+    for f in glob.glob("%s/share/clang/*" % final_dir):
+        if re_clang_tidy.search(os.path.basename(f)) is None:
+            delete(f)
+
+
 if __name__ == "__main__":
     # The directories end up in the debug info, so the easy way of getting
     # a reproducible build is to run it in a know absolute directory.
     # We use a directory in /builds/slave because the mozilla infrastructure
     # cleans it up automatically.
     base_dir = "/builds/slave/moz-toolchain"
     if is_windows():
         # TODO: Because Windows taskcluster builds are run with distinct
@@ -353,21 +429,21 @@ if __name__ == "__main__":
         build_type = config["build_type"]
         if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
             raise ValueError("We only know how to do Release, Debug, RelWithDebInfo or MinSizeRel builds")
     build_libcxx = False
     if "build_libcxx" in config:
         build_libcxx = config["build_libcxx"]
         if build_libcxx not in (True, False):
             raise ValueError("Only boolean values are accepted for build_libcxx.")
-    import_clang_tidy = False
-    if "import_clang_tidy" in config:
-        import_clang_tidy = config["import_clang_tidy"]
-        if import_clang_tidy not in (True, False):
-            raise ValueError("Only boolean values are accepted for import_clang_tidy.")
+    build_clang_tidy = False
+    if "build_clang_tidy" in config:
+        build_clang_tidy = config["build_clang_tidy"]
+        if build_clang_tidy not in (True, False):
+            raise ValueError("Only boolean values are accepted for build_clang_tidy.")
     osx_cross_compile = False
     if "osx_cross_compile" in config:
         osx_cross_compile = config["osx_cross_compile"]
         if osx_cross_compile not in (True, False):
             raise ValueError("Only boolean values are accepted for osx_cross_compile.")
         if osx_cross_compile and not is_linux():
             raise ValueError("osx_cross_compile can only be used on Linux.")
     assertions = False
@@ -432,25 +508,22 @@ if __name__ == "__main__":
                 (source_dir + "/libcxx",
                  llvm_source_dir + "/projects/libcxx"),
                 (source_dir + "/libcxxabi",
                  llvm_source_dir + "/projects/libcxxabi")]
     for l in symlinks:
         # On Windows, we have to re-copy the whole directory every time.
         if not is_windows() and os.path.islink(l[1]):
             continue
-        if os.path.isdir(l[1]):
-            shutil.rmtree(l[1])
-        elif os.path.exists(l[1]):
-            os.unlink(l[1])
+        delete(l[1]);
         if os.path.exists(l[0]):
             symlink(l[0], l[1])
 
-    if import_clang_tidy:
-        do_import_clang_tidy(llvm_source_dir)
+    if build_clang_tidy:
+        import_clang_tidy(llvm_source_dir)
 
     if not os.path.exists(build_dir):
         os.makedirs(build_dir)
 
     stage1_dir = build_dir + '/stage1'
     stage1_inst_dir = stage1_dir + '/clang'
 
     final_stage_dir = stage1_dir
@@ -533,12 +606,17 @@ if __name__ == "__main__":
                 (cc_name, exe_ext)] + extra_cflags2,
             [stage2_inst_dir + "/bin/%s%s" %
                 (cxx_name, exe_ext)] + extra_cxxflags2,
             [ld] + extra_ldflags,
             ar, ranlib,
             llvm_source_dir, stage3_dir, build_libcxx, osx_cross_compile,
             build_type, assertions, python_path, gcc_dir)
 
+    package_name = "clang"
+    if build_clang_tidy:
+        prune_final_dir_for_clang_tidy(os.path.join(final_stage_dir, "clang"))
+        package_name = "clang-tidy"
+
     if is_darwin() or is_windows():
-        build_tar_package("tar", "clang.tar.bz2", final_stage_dir, "clang")
+        build_tar_package("tar", package_name + ".tar.bz2", final_stage_dir, "clang")
     else:
-        build_tar_package("tar", "clang.tar.xz", final_stage_dir, "clang")
+        build_tar_package("tar", package_name + ".tar.xz", final_stage_dir, "clang")
--- a/build/build-clang/clang-tidy-linux64.json
+++ b/build/build-clang/clang-tidy-linux64.json
@@ -1,15 +1,15 @@
 {
     "llvm_revision": "290055",
     "stages": "1",
     "build_libcxx": true,
     "build_type": "Release",
     "assertions": false,
-    "import_clang_tidy": true,
+    "build_clang_tidy": true,
     "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
     "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
     "extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/trunk",
     "python_path": "/usr/bin/python2.7",
     "gcc_dir": "/home/worker/workspace/build/src/gcc",
--- a/build/build-clang/clang-tidy-macosx64.json
+++ b/build/build-clang/clang-tidy-macosx64.json
@@ -1,15 +1,15 @@
 {
     "llvm_revision": "290055",
     "stages": "1",
     "build_libcxx": true,
     "build_type": "Release",
     "assertions": false,
-    "import_clang_tidy": true,
+    "build_clang_tidy": true,
     "osx_cross_compile": true,
     "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
     "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
     "extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/trunk",
     "python_path": "/usr/bin/python2.7",
--- a/build/build-clang/clang-tidy-win32.json
+++ b/build/build-clang/clang-tidy-win32.json
@@ -1,15 +1,15 @@
 {
     "llvm_revision": "290055",
     "stages": "1",
     "build_libcxx": false,
     "build_type": "Release",
     "assertions": false,
-    "import_clang_tidy": true,
+    "build_clang_tidy": true,
     "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
     "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
     "extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "python_path": "c:/mozilla-build/python/python.exe",
     "cc": "cl.exe",
     "cxx": "cl.exe"
--- a/build/build-clang/clang-tidy-win64.json
+++ b/build/build-clang/clang-tidy-win64.json
@@ -1,15 +1,15 @@
 {
     "llvm_revision": "290055",
     "stages": "1",
     "build_libcxx": false,
     "build_type": "Release",
     "assertions": false,
-    "import_clang_tidy": true,
+    "build_clang_tidy": true,
     "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
     "clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
     "extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "python_path": "c:/mozilla-build/python/python.exe",
     "cc": "cl.exe",
     "cxx": "cl.exe"
--- a/taskcluster/scripts/misc/build-clang-tidy-linux.sh
+++ b/taskcluster/scripts/misc/build-clang-tidy-linux.sh
@@ -18,9 +18,9 @@ set +x
 cd build/build-clang
 # |mach python| sets up a virtualenv for us!
 ../../mach python ./build-clang.py -c clang-tidy-linux64.json
 
 set -x
 
 # Put a tarball in the artifacts dir
 mkdir -p $UPLOAD_DIR
-cp clang.tar.* $UPLOAD_DIR
+cp clang-tidy.tar.* $UPLOAD_DIR
--- a/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
+++ b/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
@@ -27,9 +27,9 @@ set +x
 cd build/build-clang
 # |mach python| sets up a virtualenv for us!
 ../../mach python ./build-clang.py -c clang-tidy-macosx64.json
 
 set -x
 
 # Put a tarball in the artifacts dir
 mkdir -p $UPLOAD_DIR
-cp clang.tar.* $UPLOAD_DIR
+cp clang-tidy.tar.* $UPLOAD_DIR
--- a/taskcluster/scripts/misc/build-clang-windows-helper32.sh
+++ b/taskcluster/scripts/misc/build-clang-windows-helper32.sh
@@ -51,9 +51,9 @@ set +x
 BUILD_CLANG_DIR=build/src/build/build-clang
 MOZCONFIG=${MOZCONFIG} build/src/mach python ${BUILD_CLANG_DIR}/build-clang.py -c ${BUILD_CLANG_DIR}/${1}
 
 set -x
 
 # Put a tarball in the artifacts dir
 UPLOAD_PATH=public/build
 mkdir -p ${UPLOAD_PATH}
-cp clang.tar.* ${UPLOAD_PATH}
+cp clang*.tar.* ${UPLOAD_PATH}
--- a/taskcluster/scripts/misc/build-clang-windows-helper64.sh
+++ b/taskcluster/scripts/misc/build-clang-windows-helper64.sh
@@ -50,9 +50,9 @@ set +x
 BUILD_CLANG_DIR=build/src/build/build-clang
 MOZCONFIG=${MOZCONFIG} build/src/mach python ${BUILD_CLANG_DIR}/build-clang.py -c ${BUILD_CLANG_DIR}/${1}
 
 set -x
 
 # Put a tarball in the artifacts dir
 UPLOAD_PATH=public/build
 mkdir -p ${UPLOAD_PATH}
-cp clang.tar.* ${UPLOAD_PATH}
+cp clang*.tar.* ${UPLOAD_PATH}