Bug 518136: Use -showIncludes to generate dependencies for MSVC for a substantial build performance win. r=ted
authorKyle Huey <me@kylehuey.com>
Wed, 04 Aug 2010 16:59:23 -0700
changeset 48881 6001758d1f47a8336e2a0257eda10781996a7859
parent 48880 d59ad0384708fcb57463e32fc878f6576dd8cb7f
child 48883 e103582a72cafc862cd29a0da20020b0499ae544
child 49344 029bd66ece21db3630ce4081750baeeebb458fa7
push idunknown
push userunknown
push dateunknown
reviewersted
bugs518136
milestone2.0b4pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 518136: Use -showIncludes to generate dependencies for MSVC for a substantial build performance win. r=ted
build/cl.py
config/autoconf.mk.in
config/config.mk
configure.in
js/src/build/cl.py
js/src/config/autoconf.mk.in
js/src/config/config.mk
js/src/configure.in
security/manager/Makefile.in
new file mode 100644
--- /dev/null
+++ b/build/cl.py
@@ -0,0 +1,94 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is cl.py: a python wrapper for cl to automatically generate
+# dependency information.
+#
+# The Initial Developer of the Original Code is
+#   Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Kyle Huey <me@kylehuey.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os, os.path
+import subprocess
+import sys
+
+def InvokeClWithDependencyGeneration(cmdline):
+    target = ""
+    # Figure out what the target is
+    for arg in cmdline:
+        if arg.startswith("-Fo"):
+            target = arg[3:]
+            break
+
+    if target == None:
+        print >>sys.stderr, "No target set" and sys.exit(1)
+
+    # The deps target lives here
+    depstarget = os.path.basename(target) + ".pp"
+
+    cmdline += ['-showIncludes']
+    cl = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
+
+    deps = set()
+    for line in cl.stdout:
+        # cl -showIncludes prefixes every header with "Note: including file:"
+        # and an indentation corresponding to the depth (which we don't need)
+        if line.startswith("Note: including file:"):
+            dep = line[21:].strip()
+            # We can't handle pathes with spaces properly in mddepend.pl, but
+            # we can assume that anything in a path with spaces is a system
+            # header and throw it away.
+            if dep.find(' ') == -1:
+                deps.add(dep)
+        else:
+            sys.stdout.write(line) # Make sure we preserve the relevant output
+                                   # from cl
+
+    ret = cl.wait()
+    if ret != 0 or target == "":
+        sys.exit(ret)
+
+    depsdir = os.path.normpath(os.path.join(os.path.dirname(target), ".deps"))
+    depstarget = os.path.join(depsdir, depstarget)
+    if not os.path.isdir(depsdir):
+        try:
+            os.makedirs(depsdir)
+        except OSError:
+            pass # This suppresses the error we get when the dir exists, at the
+                 # cost of masking failure to create the directory.  We'll just
+                 # die on the next line though, so it's not that much of a loss.
+
+    f = open(depstarget, "w")
+    for dep in sorted(deps):
+        print >>f, "%s: %s" % (target, dep)
+
+if __name__ == "__main__":
+    InvokeClWithDependencyGeneration(sys.argv[1:])
--- a/config/autoconf.mk.in
+++ b/config/autoconf.mk.in
@@ -565,16 +565,18 @@ LIBIDL_CFLAGS = @LIBIDL_CFLAGS@
 LIBIDL_LIBS = @LIBIDL_LIBS@
 STATIC_LIBIDL = @STATIC_LIBIDL@
 
 MOZ_NATIVE_MAKEDEPEND	= @SYSTEM_MAKEDEPEND@
 
 MOZ_AUTO_DEPS	= @MOZ_AUTO_DEPS@
 COMPILER_DEPEND = @COMPILER_DEPEND@
 MDDEPDIR        := @MDDEPDIR@
+CC_WRAPPER = @CC_WRAPPER@
+CXX_WRAPPER = @CXX_WRAPPER@
 
 MOZ_DEMANGLE_SYMBOLS = @MOZ_DEMANGLE_SYMBOLS@
 
 # XXX - these need to be cleaned up and have real checks added -cls
 CM_BLDTYPE=dbg
 AWT_11=1
 OS_TARGET=@OS_TARGET@
 OS_ARCH=@OS_ARCH@
--- a/config/config.mk
+++ b/config/config.mk
@@ -157,16 +157,19 @@ MOZ_UNICHARUTIL_LIBS = $(LIBXUL_DIST)/li
 MOZ_WIDGET_SUPPORT_LIBS    = $(DIST)/lib/$(LIB_PREFIX)widgetsupport_s.$(LIB_SUFFIX)
 
 ifdef MOZ_MEMORY
 ifneq (,$(filter-out WINNT WINCE,$(OS_ARCH)))
 JEMALLOC_LIBS = $(MKSHLIB_FORCE_ALL) $(call EXPAND_MOZLIBNAME,jemalloc) $(MKSHLIB_UNFORCE_ALL)
 endif
 endif
 
+CC := $(CC_WRAPPER) $(CC)
+CXX := $(CXX_WRAPPER) $(CXX)
+
 # determine debug-related options
 _DEBUG_CFLAGS :=
 _DEBUG_LDFLAGS :=
 
 ifdef MOZ_DEBUG
   _DEBUG_CFLAGS += $(MOZ_DEBUG_ENABLE_DEFS) $(MOZ_DEBUG_FLAGS)
   _DEBUG_LDFLAGS += $(MOZ_DEBUG_LDFLAGS)
   XULPPFLAGS += $(MOZ_DEBUG_ENABLE_DEFS)
--- a/configure.in
+++ b/configure.in
@@ -7985,23 +7985,31 @@ if test "$_cpp_md_flag"; then
   fi
 else
   COMPILER_DEPEND=
   dnl Don't override this for MSVC
   if test -z "$_WIN32_MSVC"; then
     _USE_CPP_INCLUDE_FLAG=
     _DEFINES_CFLAGS='$(ACDEFINES) -D_MOZILLA_CONFIG_H_ -DMOZILLA_CLIENT'
     _DEFINES_CXXFLAGS='$(ACDEFINES) -D_MOZILLA_CONFIG_H_ -DMOZILLA_CLIENT'
+  else
+    _topsrcdirwin=`cd \`dirname $0\`; pwd -W`
+    dnl cl.py provides dependency generation for MSVC
+    CC_WRAPPER="$PYTHON -O $_topsrcdirwin/build/cl.py"
+    CXX_WRAPPER="$PYTHON -O $_topsrcdirwin/build/cl.py"
+    COMPILER_DEPEND=1
   fi
 fi
 fi # MOZ_AUTO_DEPS
 MDDEPDIR='.deps'
 AC_SUBST(MOZ_AUTO_DEPS)
 AC_SUBST(COMPILER_DEPEND)
 AC_SUBST(MDDEPDIR)
+AC_SUBST(CC_WRAPPER)
+AC_SUBST(CXX_WRAPPER)
 
 
 dnl ========================================================
 dnl =
 dnl = Static Build Options
 dnl =
 dnl ========================================================
 MOZ_ARG_HEADER(Static build options)
new file mode 100644
--- /dev/null
+++ b/js/src/build/cl.py
@@ -0,0 +1,94 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is cl.py: a python wrapper for cl to automatically generate
+# dependency information.
+#
+# The Initial Developer of the Original Code is
+#   Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Kyle Huey <me@kylehuey.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import os, os.path
+import subprocess
+import sys
+
+def InvokeClWithDependencyGeneration(cmdline):
+    target = ""
+    # Figure out what the target is
+    for arg in cmdline:
+        if arg.startswith("-Fo"):
+            target = arg[3:]
+            break
+
+    if target == None:
+        print >>sys.stderr, "No target set" and sys.exit(1)
+
+    # The deps target lives here
+    depstarget = os.path.basename(target) + ".pp"
+
+    cmdline += ['-showIncludes']
+    cl = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
+
+    deps = set()
+    for line in cl.stdout:
+        # cl -showIncludes prefixes every header with "Note: including file:"
+        # and an indentation corresponding to the depth (which we don't need)
+        if line.startswith("Note: including file:"):
+            dep = line[21:].strip()
+            # We can't handle pathes with spaces properly in mddepend.pl, but
+            # we can assume that anything in a path with spaces is a system
+            # header and throw it away.
+            if dep.find(' ') == -1:
+                deps.add(dep)
+        else:
+            sys.stdout.write(line) # Make sure we preserve the relevant output
+                                   # from cl
+
+    ret = cl.wait()
+    if ret != 0 or target == "":
+        sys.exit(ret)
+
+    depsdir = os.path.normpath(os.path.join(os.path.dirname(target), ".deps"))
+    depstarget = os.path.join(depsdir, depstarget)
+    if not os.path.isdir(depsdir):
+        try:
+            os.makedirs(depsdir)
+        except OSError:
+            pass # This suppresses the error we get when the dir exists, at the
+                 # cost of masking failure to create the directory.  We'll just
+                 # die on the next line though, so it's not that much of a loss.
+
+    f = open(depstarget, "w")
+    for dep in sorted(deps):
+        print >>f, "%s: %s" % (target, dep)
+
+if __name__ == "__main__":
+    InvokeClWithDependencyGeneration(sys.argv[1:])
--- a/js/src/config/autoconf.mk.in
+++ b/js/src/config/autoconf.mk.in
@@ -278,16 +278,18 @@ NO_LD_ARCHIVE_FLAGS     = @NO_LD_ARCHIVE
 MOZ_TOOLKIT_REGISTRY_CFLAGS = \
 	$(TK_CFLAGS)
 
 MOZ_NATIVE_MAKEDEPEND	= @SYSTEM_MAKEDEPEND@
 
 MOZ_AUTO_DEPS	= @MOZ_AUTO_DEPS@
 COMPILER_DEPEND = @COMPILER_DEPEND@
 MDDEPDIR        := @MDDEPDIR@
+CC_WRAPPER = @CC_WRAPPER@
+CXX_WRAPPER = @CXX_WRAPPER@
 
 MOZ_DEMANGLE_SYMBOLS = @MOZ_DEMANGLE_SYMBOLS@
 
 # XXX - these need to be cleaned up and have real checks added -cls
 CM_BLDTYPE=dbg
 AWT_11=1
 OS_TARGET=@OS_TARGET@
 OS_ARCH=@OS_ARCH@
--- a/js/src/config/config.mk
+++ b/js/src/config/config.mk
@@ -157,16 +157,19 @@ MOZ_UNICHARUTIL_LIBS = $(LIBXUL_DIST)/li
 MOZ_WIDGET_SUPPORT_LIBS    = $(DIST)/lib/$(LIB_PREFIX)widgetsupport_s.$(LIB_SUFFIX)
 
 ifdef MOZ_MEMORY
 ifneq (,$(filter-out WINNT WINCE,$(OS_ARCH)))
 JEMALLOC_LIBS = $(MKSHLIB_FORCE_ALL) $(call EXPAND_MOZLIBNAME,jemalloc) $(MKSHLIB_UNFORCE_ALL)
 endif
 endif
 
+CC := $(CC_WRAPPER) $(CC)
+CXX := $(CXX_WRAPPER) $(CXX)
+
 # determine debug-related options
 _DEBUG_CFLAGS :=
 _DEBUG_LDFLAGS :=
 
 ifdef MOZ_DEBUG
   _DEBUG_CFLAGS += $(MOZ_DEBUG_ENABLE_DEFS) $(MOZ_DEBUG_FLAGS)
   _DEBUG_LDFLAGS += $(MOZ_DEBUG_LDFLAGS)
   XULPPFLAGS += $(MOZ_DEBUG_ENABLE_DEFS)
--- a/js/src/configure.in
+++ b/js/src/configure.in
@@ -4878,23 +4878,31 @@ if test "$_cpp_md_flag"; then
   fi
 else
   COMPILER_DEPEND=
   dnl Don't override this for MSVC
   if test -z "$_WIN32_MSVC"; then
     _USE_CPP_INCLUDE_FLAG=
     _DEFINES_CFLAGS='$(ACDEFINES) -D_JS_CONFDEFS_H_ -DMOZILLA_CLIENT'
     _DEFINES_CXXFLAGS='$(ACDEFINES) -D_JS_CONFDEFS_H_ -DMOZILLA_CLIENT'
+  else
+    _topsrcdirwin=`cd \`dirname $0\`; pwd -W`
+    dnl cl.py provides dependency generation for MSVC
+    CC_WRAPPER="$PYTHON -O $_topsrcdirwin/build/cl.py"
+    CXX_WRAPPER="$PYTHON -O $_topsrcdirwin/build/cl.py"
+    COMPILER_DEPEND=1
   fi
 fi
 fi # MOZ_AUTO_DEPS
 MDDEPDIR='.deps'
 AC_SUBST(MOZ_AUTO_DEPS)
 AC_SUBST(COMPILER_DEPEND)
 AC_SUBST(MDDEPDIR)
+AC_SUBST(CC_WRAPPER)
+AC_SUBST(CXX_WRAPPER)
 
 
 dnl ========================================================
 dnl =
 dnl = Static Build Options
 dnl =
 dnl ========================================================
 MOZ_ARG_HEADER(Static build options)
--- a/security/manager/Makefile.in
+++ b/security/manager/Makefile.in
@@ -37,16 +37,18 @@
 # ***** END LICENSE BLOCK *****
 
 DEPTH		= ../..
 topsrcdir	= @top_srcdir@
 srcdir		= @srcdir@
 VPATH		= @srcdir@
 
 include $(DEPTH)/config/autoconf.mk
+CC_WRAPPER =
+CXX_WRAPPER =
 include $(topsrcdir)/config/config.mk
 
 MODULE = psm
 
 ifndef MOZ_NATIVE_NSS
 LOADABLE_ROOT_MODULE = $(DLL_PREFIX)nssckbi$(DLL_SUFFIX)
 endif