Bug 1329307 - Only package the clang-tidy binaries for the clang-tidy builds; r=mystor
--- a/build/build-clang/README
+++ b/build/build-clang/README
@@ -40,17 +40,17 @@ build-clang.py accepts a JSON config for
* cc: Path to the bootsraping C Compiler.
* cxx: Path to the bootsraping C++ Compiler.
* ar: Path to the library archiver tool.
* ranlib: Path to the ranlib tool.
* ld: Path to the linker.
* patches: Optional list of patches to apply per platform. Supported platforms: macosx64, linux32, linux64. The default is Release.
* build_type: The type of build to make. Supported types: Release, Debug, RelWithDebInfo or MinSizeRel.
* build_libcxx: Whether to build with libcxx. The default is false.
-* import_clang_tidy: Whether to import Mozilla checks into clang-tidy before building. The default is false.
+* build_clang_tidy: Whether to build clang-tidy with the Mozilla checks imported. The default is false.
* osx_cross_compile: Whether to invoke CMake for OS X cross compile builds.
* assertions: Whether to enable LLVM assertions. The default is false.
Environment Variables
---------------------
The following environment variables are used for cross-compile builds targeting OS X on Linux.
--- a/build/build-clang/build-clang.py
+++ b/build/build-clang/build-clang.py
@@ -52,17 +52,17 @@ def run_in(path, args):
def patch(patch, srcdir):
patch = os.path.realpath(patch)
check_run(['patch', '-d', srcdir, '-p1', '-i', patch, '--fuzz=0',
'-s'])
-def do_import_clang_tidy(source_dir):
+def import_clang_tidy(source_dir):
clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]),
'..', 'clang-plugin')
clang_tidy_path = os.path.join(source_dir,
'tools/clang/tools/extra/clang-tidy')
sys.path.append(clang_plugin_path)
from import_mozilla_checks import do_import
do_import(clang_plugin_path, clang_tidy_path)
@@ -123,16 +123,26 @@ def copy_dir_contents(src, dest):
def mkdir_p(path):
try:
os.makedirs(path)
except OSError as e:
if e.errno != errno.EEXIST or not os.path.isdir(path):
raise
+def delete(path):
+ if os.path.isdir(path):
+ shutil.rmtree(path)
+ else:
+ try:
+ os.unlink(path)
+ except:
+ pass
+
+
def install_libgcc(gcc_dir, clang_dir):
out = subprocess.check_output([os.path.join(gcc_dir, "bin", "gcc"),
'-print-libgcc-file-name'])
libgcc_dir = os.path.dirname(out.rstrip())
clang_lib_dir = os.path.join(clang_dir, "lib", "gcc",
"x86_64-unknown-linux-gnu",
os.path.basename(libgcc_dir))
@@ -276,16 +286,82 @@ def get_tool(config, key):
return f
# Assume that we have the name of some program that should be on PATH.
try:
return which.which(f) if f else which.which(key)
except which.WhichError:
raise ValueError("%s not found on PATH" % f)
+
+# This function is intended to be called on the final build directory when
+# building clang-tidy. Its job is to remove all of the files which won't
+# be used for clang-tidy to reduce the download size. Currently when this
+# function finishes its job, it will leave final_dir with a layout like this:
+#
+# clang/
+# bin/
+# clang-tidy
+# include/
+# * (nothing will be deleted here)
+# lib/
+# clang/
+# 4.0.0/
+# include/
+# * (nothing will be deleted here)
+# share/
+# clang/
+# clang-tidy-diff.py
+# run-clang-tidy.py
+def prune_final_dir_for_clang_tidy(final_dir):
+ # Make sure we only have what we expect.
+ dirs = ("bin", "include", "lib", "libexec", "msbuild-bin", "share", "tools")
+ for f in glob.glob("%s/*" % final_dir):
+ if os.path.basename(f) not in dirs:
+ raise Exception("Found unknown file %s in the final directory" % f)
+ if not os.path.isdir(f):
+ raise Exception("Expected %s to be a directory" %f)
+
+ # In bin/, only keep clang-tidy.
+ re_clang_tidy = re.compile(r"^clang-tidy(\.exe)?$", re.I)
+ for f in glob.glob("%s/bin/*" % final_dir):
+ if re_clang_tidy.search(os.path.basename(f)) is None:
+ delete(f)
+
+ # Keep include/ intact.
+
+ # In lib/, only keep lib/clang/N.M.O/include.
+ re_ver_num = re.compile(r"^\d+\.\d+\.\d+$", re.I)
+ for f in glob.glob("%s/lib/*" % final_dir):
+ if os.path.basename(f) != "clang":
+ delete(f)
+ for f in glob.glob("%s/lib/clang/*" % final_dir):
+ if re_ver_num.search(os.path.basename(f)) is None:
+ delete(f)
+ for f in glob.glob("%s/lib/clang/*/*" % final_dir):
+ if os.path.basename(f) != "include":
+ delete(f)
+
+ # Completely remove libexec/, msbuilld-bin and tools, if it exists.
+ shutil.rmtree(os.path.join(final_dir, "libexec"))
+ for d in ("msbuild-bin", "tools"):
+ d = os.path.join(final_dir, d)
+ if os.path.exists(d):
+ shutil.rmtree(d)
+
+ # In share/, only keep share/clang/*tidy*
+ re_clang_tidy = re.compile(r"tidy", re.I)
+ for f in glob.glob("%s/share/*" % final_dir):
+ if os.path.basename(f) != "clang":
+ delete(f)
+ for f in glob.glob("%s/share/clang/*" % final_dir):
+ if re_clang_tidy.search(os.path.basename(f)) is None:
+ delete(f)
+
+
if __name__ == "__main__":
# The directories end up in the debug info, so the easy way of getting
# a reproducible build is to run it in a know absolute directory.
# We use a directory in /builds/slave because the mozilla infrastructure
# cleans it up automatically.
base_dir = "/builds/slave/moz-toolchain"
if is_windows():
# TODO: Because Windows taskcluster builds are run with distinct
@@ -353,21 +429,21 @@ if __name__ == "__main__":
build_type = config["build_type"]
if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
raise ValueError("We only know how to do Release, Debug, RelWithDebInfo or MinSizeRel builds")
build_libcxx = False
if "build_libcxx" in config:
build_libcxx = config["build_libcxx"]
if build_libcxx not in (True, False):
raise ValueError("Only boolean values are accepted for build_libcxx.")
- import_clang_tidy = False
- if "import_clang_tidy" in config:
- import_clang_tidy = config["import_clang_tidy"]
- if import_clang_tidy not in (True, False):
- raise ValueError("Only boolean values are accepted for import_clang_tidy.")
+ build_clang_tidy = False
+ if "build_clang_tidy" in config:
+ build_clang_tidy = config["build_clang_tidy"]
+ if build_clang_tidy not in (True, False):
+ raise ValueError("Only boolean values are accepted for build_clang_tidy.")
osx_cross_compile = False
if "osx_cross_compile" in config:
osx_cross_compile = config["osx_cross_compile"]
if osx_cross_compile not in (True, False):
raise ValueError("Only boolean values are accepted for osx_cross_compile.")
if osx_cross_compile and not is_linux():
raise ValueError("osx_cross_compile can only be used on Linux.")
assertions = False
@@ -432,25 +508,22 @@ if __name__ == "__main__":
(source_dir + "/libcxx",
llvm_source_dir + "/projects/libcxx"),
(source_dir + "/libcxxabi",
llvm_source_dir + "/projects/libcxxabi")]
for l in symlinks:
# On Windows, we have to re-copy the whole directory every time.
if not is_windows() and os.path.islink(l[1]):
continue
- if os.path.isdir(l[1]):
- shutil.rmtree(l[1])
- elif os.path.exists(l[1]):
- os.unlink(l[1])
+ delete(l[1]);
if os.path.exists(l[0]):
symlink(l[0], l[1])
- if import_clang_tidy:
- do_import_clang_tidy(llvm_source_dir)
+ if build_clang_tidy:
+ import_clang_tidy(llvm_source_dir)
if not os.path.exists(build_dir):
os.makedirs(build_dir)
stage1_dir = build_dir + '/stage1'
stage1_inst_dir = stage1_dir + '/clang'
final_stage_dir = stage1_dir
@@ -533,12 +606,17 @@ if __name__ == "__main__":
(cc_name, exe_ext)] + extra_cflags2,
[stage2_inst_dir + "/bin/%s%s" %
(cxx_name, exe_ext)] + extra_cxxflags2,
[ld] + extra_ldflags,
ar, ranlib,
llvm_source_dir, stage3_dir, build_libcxx, osx_cross_compile,
build_type, assertions, python_path, gcc_dir)
+ package_name = "clang"
+ if build_clang_tidy:
+ prune_final_dir_for_clang_tidy(os.path.join(final_stage_dir, "clang"))
+ package_name = "clang-tidy"
+
if is_darwin() or is_windows():
- build_tar_package("tar", "clang.tar.bz2", final_stage_dir, "clang")
+ build_tar_package("tar", package_name + ".tar.bz2", final_stage_dir, "clang")
else:
- build_tar_package("tar", "clang.tar.xz", final_stage_dir, "clang")
+ build_tar_package("tar", package_name + ".tar.xz", final_stage_dir, "clang")
--- a/build/build-clang/clang-tidy-linux64.json
+++ b/build/build-clang/clang-tidy-linux64.json
@@ -1,15 +1,15 @@
{
"llvm_revision": "290055",
"stages": "1",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
- "import_clang_tidy": true,
+ "build_clang_tidy": true,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
"extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/trunk",
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/home/worker/workspace/build/src/gcc",
--- a/build/build-clang/clang-tidy-macosx64.json
+++ b/build/build-clang/clang-tidy-macosx64.json
@@ -1,15 +1,15 @@
{
"llvm_revision": "290055",
"stages": "1",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
- "import_clang_tidy": true,
+ "build_clang_tidy": true,
"osx_cross_compile": true,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
"extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
"libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/trunk",
"python_path": "/usr/bin/python2.7",
--- a/build/build-clang/clang-tidy-win32.json
+++ b/build/build-clang/clang-tidy-win32.json
@@ -1,15 +1,15 @@
{
"llvm_revision": "290055",
"stages": "1",
"build_libcxx": false,
"build_type": "Release",
"assertions": false,
- "import_clang_tidy": true,
+ "build_clang_tidy": true,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
"extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
"python_path": "c:/mozilla-build/python/python.exe",
"cc": "cl.exe",
"cxx": "cl.exe"
--- a/build/build-clang/clang-tidy-win64.json
+++ b/build/build-clang/clang-tidy-win64.json
@@ -1,15 +1,15 @@
{
"llvm_revision": "290055",
"stages": "1",
"build_libcxx": false,
"build_type": "Release",
"assertions": false,
- "import_clang_tidy": true,
+ "build_clang_tidy": true,
"llvm_repo": "https://llvm.org/svn/llvm-project/llvm/trunk",
"clang_repo": "https://llvm.org/svn/llvm-project/cfe/trunk",
"extra_repo": "https://llvm.org/svn/llvm-project/clang-tools-extra/trunk",
"compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
"libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
"python_path": "c:/mozilla-build/python/python.exe",
"cc": "cl.exe",
"cxx": "cl.exe"
--- a/taskcluster/scripts/misc/build-clang-tidy-linux.sh
+++ b/taskcluster/scripts/misc/build-clang-tidy-linux.sh
@@ -18,9 +18,9 @@ set +x
cd build/build-clang
# |mach python| sets up a virtualenv for us!
../../mach python ./build-clang.py -c clang-tidy-linux64.json
set -x
# Put a tarball in the artifacts dir
mkdir -p $UPLOAD_DIR
-cp clang.tar.* $UPLOAD_DIR
+cp clang-tidy.tar.* $UPLOAD_DIR
--- a/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
+++ b/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
@@ -27,9 +27,9 @@ set +x
cd build/build-clang
# |mach python| sets up a virtualenv for us!
../../mach python ./build-clang.py -c clang-tidy-macosx64.json
set -x
# Put a tarball in the artifacts dir
mkdir -p $UPLOAD_DIR
-cp clang.tar.* $UPLOAD_DIR
+cp clang-tidy.tar.* $UPLOAD_DIR
--- a/taskcluster/scripts/misc/build-clang-windows-helper32.sh
+++ b/taskcluster/scripts/misc/build-clang-windows-helper32.sh
@@ -51,9 +51,9 @@ set +x
BUILD_CLANG_DIR=build/src/build/build-clang
MOZCONFIG=${MOZCONFIG} build/src/mach python ${BUILD_CLANG_DIR}/build-clang.py -c ${BUILD_CLANG_DIR}/${1}
set -x
# Put a tarball in the artifacts dir
UPLOAD_PATH=public/build
mkdir -p ${UPLOAD_PATH}
-cp clang.tar.* ${UPLOAD_PATH}
+cp clang*.tar.* ${UPLOAD_PATH}
--- a/taskcluster/scripts/misc/build-clang-windows-helper64.sh
+++ b/taskcluster/scripts/misc/build-clang-windows-helper64.sh
@@ -50,9 +50,9 @@ set +x
BUILD_CLANG_DIR=build/src/build/build-clang
MOZCONFIG=${MOZCONFIG} build/src/mach python ${BUILD_CLANG_DIR}/build-clang.py -c ${BUILD_CLANG_DIR}/${1}
set -x
# Put a tarball in the artifacts dir
UPLOAD_PATH=public/build
mkdir -p ${UPLOAD_PATH}
-cp clang.tar.* ${UPLOAD_PATH}
+cp clang*.tar.* ${UPLOAD_PATH}