Bug 462463 - Stop using mddepend.pl. r=ted
authorMike Hommey <mh+mozilla@glandium.org>
Sun, 17 Mar 2013 07:43:35 +0100
changeset 135508 fc11223a77451464f54852d1cc307d4dda15237e
parent 135507 d926c77d4cd79879dbd29735569666e5aa0ff0f8
child 135509 267c636bd241616fb5e83b84feba2a5c1dd3b23e
push id2452
push userlsblakk@mozilla.com
push dateMon, 13 May 2013 16:59:38 +0000
treeherdermozilla-beta@d4b152d29d8d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs462463
milestone22.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 462463 - Stop using mddepend.pl. r=ted
build/cl.py
build/unix/add_phony_targets.py
config/expandlibs_exec.py
config/expandlibs_gen.py
config/rules.mk
configure.in
js/src/build/cl.py
js/src/build/unix/add_phony_targets.py
js/src/build/unix/mddepend.pl
js/src/config/expandlibs_exec.py
js/src/config/expandlibs_gen.py
js/src/config/rules.mk
js/src/configure.in
python/codegen/makeutils.py
xpcom/idl-parser/header.py
--- a/build/cl.py
+++ b/build/cl.py
@@ -52,11 +52,12 @@ def InvokeClWithDependencyGeneration(cmd
         except OSError:
             pass # This suppresses the error we get when the dir exists, at the
                  # cost of masking failure to create the directory.  We'll just
                  # die on the next line though, so it's not that much of a loss.
 
     f = open(depstarget, "w")
     for dep in sorted(deps):
         print >>f, "%s: %s" % (target, dep)
+        print >>f, "%s:" % dep
 
 if __name__ == "__main__":
     InvokeClWithDependencyGeneration(sys.argv[1:])
new file mode 100644
--- /dev/null
+++ b/build/unix/add_phony_targets.py
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import pymake.data
+import pymake.parser
+import pymake.parserdata
+import sys
+
+'''
+Modifies the output of Sun Studio's -xM to look more like the output
+of gcc's -MD -MP, adding phony targets for dependencies.
+'''
+
+
+def add_phony_targets(path):
+    print path
+    deps = set()
+    targets = set()
+    for stmt in pymake.parser.parsefile(path):
+        if isinstance(stmt, pymake.parserdata.Rule):
+            assert isinstance(stmt.depexp, pymake.data.StringExpansion)
+            assert isinstance(stmt.targetexp, pymake.data.StringExpansion)
+            for d in stmt.depexp.s.split():
+                deps.add(d)
+            for t in stmt.targetexp.s.split():
+                targets.add(t)
+    phony_targets = deps - targets
+    if not phony_targets:
+        return
+    with open(path, 'a') as f:
+        f.writelines('%s:\n' % d for d in phony_targets)
+
+
+if __name__ == '__main__':
+    for f in sys.argv[1:]:
+        add_phony_targets(f)
--- a/config/expandlibs_exec.py
+++ b/config/expandlibs_exec.py
@@ -320,11 +320,14 @@ def main():
         if proc.returncode:
             exit(proc.returncode)
     if not options.depend:
         return
     ensureParentDir(options.depend)
     with open(options.depend, 'w') as depfile:
         depfile.write("%s : %s\n" % (options.target, ' '.join(dep for dep in deps if os.path.isfile(dep) and dep != options.target)))
 
+        for dep in deps:
+            if os.path.isfile(dep) and dep != options.target:
+                depfile.write("%s :\n" % dep)
 
 if __name__ == '__main__':
     main()
--- a/config/expandlibs_gen.py
+++ b/config/expandlibs_gen.py
@@ -39,9 +39,12 @@ if __name__ == '__main__':
         raise Exception("Missing option: -o")
 
     ensureParentDir(options.output)
     with open(options.output, 'w') as outfile:
         print >>outfile, generate(args)
     if options.depend:
         ensureParentDir(options.depend)
         with open(options.depend, 'w') as depfile:
-            depfile.write("%s : %s\n" % (options.output, ' '.join(ExpandLibsDeps(args))))
+            deps = ExpandLibsDeps(args)
+            depfile.write("%s : %s\n" % (options.output, ' '.join(deps)))
+            for dep in deps:
+                depfile.write("%s :\n" % dep)
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -981,27 +981,29 @@ endif
 
 ifeq ($(SOLARIS_SUNPRO_CC),1)
 _MDDEPFILE = $(MDDEPDIR)/$(@F).pp
 
 define MAKE_DEPS_AUTO_CC
 if test -d $(@D); then \
 	echo "Building deps for $< using Sun Studio cc"; \
 	$(CC) $(COMPILE_CFLAGS) -xM  $< >$(_MDDEPFILE) ; \
+	$(PYTHON) $(topsrcdir)/build/unix/add_phony_targets.py $(_MDDEPFILE) ; \
 fi
 endef
 define MAKE_DEPS_AUTO_CXX
 if test -d $(@D); then \
 	echo "Building deps for $< using Sun Studio CC"; \
 	$(CXX) $(COMPILE_CXXFLAGS) -xM $< >$(_MDDEPFILE) ; \
+	$(PYTHON) $(topsrcdir)/build/unix/add_phony_targets.py $(_MDDEPFILE) ; \
 fi
 endef
 endif # Sun Studio on Solaris
 
-$(OBJS) $(HOST_OBJS): $(GLOBAL_DEPS)
+$(OBJS) $(HOST_OBJS) $(PROGOBJS) $(HOST_PROGOBJS): $(GLOBAL_DEPS)
 
 # Rules for building native targets must come first because of the host_ prefix
 $(HOST_COBJS): host_%.$(OBJ_SUFFIX): %.c
 	$(REPORT_BUILD)
 	$(ELOG) $(HOST_CC) $(HOST_OUTOPTION)$@ -c $(HOST_CFLAGS) $(INCLUDES) $(NSPR_CFLAGS) $(_VPATH_SRCS)
 
 $(HOST_CPPOBJS): host_%.$(OBJ_SUFFIX): %.cpp
 	$(REPORT_BUILD)
@@ -1608,24 +1610,17 @@ endif
 #   a previous build in the source tree) and thus neglect to create a
 #   dependency directory in the object directory, where we really need
 #   it.
 
 ifneq (,$(filter-out all chrome default export realchrome tools clean clobber clobber_all distclean realclean,$(MAKECMDGOALS)))
 MDDEPEND_FILES		:= $(strip $(wildcard $(foreach file,$(OBJS) $(PROGOBJS) $(HOST_OBJS) $(HOST_PROGOBJS) $(TARGETS) $(XPIDLSRCS:.idl=.h) $(XPIDLSRCS:.idl=.xpt),$(MDDEPDIR)/$(notdir $(file)).pp) $(addprefix $(MDDEPDIR)/,$(EXTRA_MDDEPEND_FILES))))
 
 ifneq (,$(MDDEPEND_FILES))
-# The script mddepend.pl checks the dependencies and writes to stdout
-# one rule to force out-of-date objects. For example,
-#   foo.o boo.o: FORCE
-# The script has an advantage over including the *.pp files directly
-# because it handles the case when header files are removed from the build.
-# 'make' would complain that there is no way to build missing headers.
-ALL_PP_RESULTS = $(shell $(PERL) $(BUILD_TOOLS)/mddepend.pl - $(MDDEPEND_FILES))
-$(eval $(ALL_PP_RESULTS))
+include $(MDDEPEND_FILES)
 endif
 
 endif
 #############################################################################
 
 -include $(topsrcdir)/$(MOZ_BUILD_APP)/app-rules.mk
 -include $(MY_RULES)
 
--- a/configure.in
+++ b/configure.in
@@ -7925,17 +7925,17 @@ fi # COMPILE_ENVIRONMENT
 dnl ========================================================
 dnl =
 dnl = Build depencency options
 dnl =
 dnl ========================================================
 MOZ_ARG_HEADER(Build dependencies)
 
 if test "$GNU_CC" -a "$GNU_CXX"; then
-  _DEPEND_CFLAGS='$(filter-out %/.pp,-MD -MF $(MDDEPDIR)/$(@F).pp)'
+  _DEPEND_CFLAGS='-MD -MP -MF $(MDDEPDIR)/$(@F).pp'
 dnl Sun Studio on Solaris use -xM instead of -MD, see config/rules.mk
 elif test "$SOLARIS_SUNPRO_CC"; then
   _DEPEND_CFLAGS=
 else
   dnl Don't override this for MSVC
   if test -z "$_WIN32_MSVC"; then
     _USE_CPP_INCLUDE_FLAG=
     _DEFINES_CFLAGS='$(ACDEFINES) -D_MOZILLA_CONFIG_H_ -DMOZILLA_CLIENT'
--- a/js/src/build/cl.py
+++ b/js/src/build/cl.py
@@ -52,11 +52,12 @@ def InvokeClWithDependencyGeneration(cmd
         except OSError:
             pass # This suppresses the error we get when the dir exists, at the
                  # cost of masking failure to create the directory.  We'll just
                  # die on the next line though, so it's not that much of a loss.
 
     f = open(depstarget, "w")
     for dep in sorted(deps):
         print >>f, "%s: %s" % (target, dep)
+        print >>f, "%s:" % dep
 
 if __name__ == "__main__":
     InvokeClWithDependencyGeneration(sys.argv[1:])
new file mode 100644
--- /dev/null
+++ b/js/src/build/unix/add_phony_targets.py
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import pymake.data
+import pymake.parser
+import pymake.parserdata
+import sys
+
+'''
+Modifies the output of Sun Studio's -xM to look more like the output
+of gcc's -MD -MP, adding phony targets for dependencies.
+'''
+
+
+def add_phony_targets(path):
+    print path
+    deps = set()
+    targets = set()
+    for stmt in pymake.parser.parsefile(path):
+        if isinstance(stmt, pymake.parserdata.Rule):
+            assert isinstance(stmt.depexp, pymake.data.StringExpansion)
+            assert isinstance(stmt.targetexp, pymake.data.StringExpansion)
+            for d in stmt.depexp.s.split():
+                deps.add(d)
+            for t in stmt.targetexp.s.split():
+                targets.add(t)
+    phony_targets = deps - targets
+    if not phony_targets:
+        return
+    with open(path, 'a') as f:
+        f.writelines('%s:\n' % d for d in phony_targets)
+
+
+if __name__ == '__main__':
+    for f in sys.argv[1:]:
+        add_phony_targets(f)
deleted file mode 100644
--- a/js/src/build/unix/mddepend.pl
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env perl
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# mddepend.pl - Reads in dependencies generated my -MD flag. Prints list
-#   of objects that need to be rebuilt. These can then be added to the
-#   PHONY target. Using this script copes with the problem of header
-#   files that have been removed from the build.
-#    
-# Usage:
-#   mddepend.pl <output_file> <dependency_files...>
-#
-# Send comments, improvements, bugs to Steve Lamm (slamm@netscape.com).
-
-use strict;
-
-use constant DEBUG => 0;
-
-my $outfile = shift @ARGV;
-my $silent = $ENV{MAKEFLAGS} =~ /^\w*s|\s-s/;
-
-my $line = '';
-my %alldeps;
-# Parse dependency files
-while (<>) {
-  s/\r?\n$//; # Handle both unix and DOS line endings
-  $line .= $_;
-  if ($line =~ /\\$/) {
-    chop $line;
-    next;
-  }
-
-  $line =~ s|\\|/|g;
-
-  my ($obj,$rest) = split /\s*:\s+/, $line, 2;
-  $line = '';
-  next if !$obj || !$rest;
-
-  my @deps = split /\s+/, $rest;
-  push @{$alldeps{$obj}}, @deps;
-  if (DEBUG >= 2) {
-    foreach my $dep (@deps) { print STDERR "add $obj $dep\n"; }
-  }
-}
-
-# Test dependencies
-my %modtimes; # cache
-my @objs;     # force rebuild on these
-OBJ_LOOP: foreach my $obj (keys %alldeps) {
-  my $mtime = (stat $obj)[9] or next;
-
-  my %not_in_cache;
-  my $deps = $alldeps{$obj};
-  foreach my $dep_file (@{$deps}) {
-    my $dep_mtime = $modtimes{$dep_file};
-    if (not defined $dep_mtime) {
-      print STDERR "Skipping $dep_file for $obj, will stat() later\n" if DEBUG >= 2;
-      $not_in_cache{$dep_file} = 1;
-      next;
-    }
-
-    print STDERR "Found $dep_file in cache\n" if DEBUG >= 2;
-
-    if ($dep_mtime > $mtime) {
-      print STDERR "$dep_file($dep_mtime) newer than $obj($mtime)\n" if DEBUG;
-    }
-    elsif ($dep_mtime == -1) {
-      print STDERR "Couldn't stat $dep_file for $obj\n" if DEBUG;
-    }
-    else {
-      print STDERR "$dep_file($dep_mtime) older than $obj($mtime)\n" if DEBUG >= 2;
-      next;
-    }
-
-    push @objs, $obj; # dependency is missing or newer
-    next OBJ_LOOP; # skip checking the rest of the dependencies
-  }
-
-  foreach my $dep_file (keys %not_in_cache) {
-    print STDERR "STAT $dep_file for $obj\n" if DEBUG >= 2;
-    my $dep_mtime = $modtimes{$dep_file} = (stat $dep_file)[9] || -1;
-
-    if ($dep_mtime > $mtime) {
-      print STDERR "$dep_file($dep_mtime) newer than $obj($mtime)\n" if DEBUG;
-    }
-    elsif ($dep_mtime == -1) {
-      print STDERR "Couldn't stat $dep_file for $obj\n" if DEBUG;
-    }
-    else {
-      print STDERR "$dep_file($dep_mtime) older than $obj($mtime)\n" if DEBUG >= 2;
-      next;
-    }
-
-    push @objs, $obj; # dependency is missing or newer
-    next OBJ_LOOP; # skip checking the rest of the dependencies
-  }
-
-  # If we get here it means nothing needs to be done for $obj
-}
-
-# Output objects to rebuild (if needed).
-if ($outfile eq '-') {
-    if (@objs) {
-	print "@objs: FORCE\n";
-    }
-} elsif (@objs) {
-  my $old_output;
-  my $new_output = "@objs: FORCE\n";
-
-  # Read in the current dependencies file.
-  open(OLD, "<$outfile")
-    and $old_output = <OLD>;
-  close(OLD);
-
-  # Only write out the dependencies if they are different.
-  if ($new_output ne $old_output) {
-    open(OUT, ">$outfile") and print OUT "$new_output";
-    print "Updating dependencies file, $outfile\n" unless $silent;
-    if (DEBUG) {
-      print "new: $new_output\n";
-      print "was: $old_output\n" if $old_output ne '';
-    }
-  }
-} elsif (-s $outfile) {
-  # Remove the old dependencies because all objects are up to date.
-  print "Removing old dependencies file, $outfile\n" unless $silent;
-
-  if (DEBUG) {
-    my $old_output;
-    open(OLD, "<$outfile")
-      and $old_output = <OLD>;
-    close(OLD);
-    print "was: $old_output\n";
-  }
-
-  unlink $outfile;
-}
--- a/js/src/config/expandlibs_exec.py
+++ b/js/src/config/expandlibs_exec.py
@@ -320,11 +320,14 @@ def main():
         if proc.returncode:
             exit(proc.returncode)
     if not options.depend:
         return
     ensureParentDir(options.depend)
     with open(options.depend, 'w') as depfile:
         depfile.write("%s : %s\n" % (options.target, ' '.join(dep for dep in deps if os.path.isfile(dep) and dep != options.target)))
 
+        for dep in deps:
+            if os.path.isfile(dep) and dep != options.target:
+                depfile.write("%s :\n" % dep)
 
 if __name__ == '__main__':
     main()
--- a/js/src/config/expandlibs_gen.py
+++ b/js/src/config/expandlibs_gen.py
@@ -39,9 +39,12 @@ if __name__ == '__main__':
         raise Exception("Missing option: -o")
 
     ensureParentDir(options.output)
     with open(options.output, 'w') as outfile:
         print >>outfile, generate(args)
     if options.depend:
         ensureParentDir(options.depend)
         with open(options.depend, 'w') as depfile:
-            depfile.write("%s : %s\n" % (options.output, ' '.join(ExpandLibsDeps(args))))
+            deps = ExpandLibsDeps(args)
+            depfile.write("%s : %s\n" % (options.output, ' '.join(deps)))
+            for dep in deps:
+                depfile.write("%s :\n" % dep)
--- a/js/src/config/rules.mk
+++ b/js/src/config/rules.mk
@@ -981,27 +981,29 @@ endif
 
 ifeq ($(SOLARIS_SUNPRO_CC),1)
 _MDDEPFILE = $(MDDEPDIR)/$(@F).pp
 
 define MAKE_DEPS_AUTO_CC
 if test -d $(@D); then \
 	echo "Building deps for $< using Sun Studio cc"; \
 	$(CC) $(COMPILE_CFLAGS) -xM  $< >$(_MDDEPFILE) ; \
+	$(PYTHON) $(topsrcdir)/build/unix/add_phony_targets.py $(_MDDEPFILE) ; \
 fi
 endef
 define MAKE_DEPS_AUTO_CXX
 if test -d $(@D); then \
 	echo "Building deps for $< using Sun Studio CC"; \
 	$(CXX) $(COMPILE_CXXFLAGS) -xM $< >$(_MDDEPFILE) ; \
+	$(PYTHON) $(topsrcdir)/build/unix/add_phony_targets.py $(_MDDEPFILE) ; \
 fi
 endef
 endif # Sun Studio on Solaris
 
-$(OBJS) $(HOST_OBJS): $(GLOBAL_DEPS)
+$(OBJS) $(HOST_OBJS) $(PROGOBJS) $(HOST_PROGOBJS): $(GLOBAL_DEPS)
 
 # Rules for building native targets must come first because of the host_ prefix
 $(HOST_COBJS): host_%.$(OBJ_SUFFIX): %.c
 	$(REPORT_BUILD)
 	$(ELOG) $(HOST_CC) $(HOST_OUTOPTION)$@ -c $(HOST_CFLAGS) $(INCLUDES) $(NSPR_CFLAGS) $(_VPATH_SRCS)
 
 $(HOST_CPPOBJS): host_%.$(OBJ_SUFFIX): %.cpp
 	$(REPORT_BUILD)
@@ -1608,24 +1610,17 @@ endif
 #   a previous build in the source tree) and thus neglect to create a
 #   dependency directory in the object directory, where we really need
 #   it.
 
 ifneq (,$(filter-out all chrome default export realchrome tools clean clobber clobber_all distclean realclean,$(MAKECMDGOALS)))
 MDDEPEND_FILES		:= $(strip $(wildcard $(foreach file,$(OBJS) $(PROGOBJS) $(HOST_OBJS) $(HOST_PROGOBJS) $(TARGETS) $(XPIDLSRCS:.idl=.h) $(XPIDLSRCS:.idl=.xpt),$(MDDEPDIR)/$(notdir $(file)).pp) $(addprefix $(MDDEPDIR)/,$(EXTRA_MDDEPEND_FILES))))
 
 ifneq (,$(MDDEPEND_FILES))
-# The script mddepend.pl checks the dependencies and writes to stdout
-# one rule to force out-of-date objects. For example,
-#   foo.o boo.o: FORCE
-# The script has an advantage over including the *.pp files directly
-# because it handles the case when header files are removed from the build.
-# 'make' would complain that there is no way to build missing headers.
-ALL_PP_RESULTS = $(shell $(PERL) $(BUILD_TOOLS)/mddepend.pl - $(MDDEPEND_FILES))
-$(eval $(ALL_PP_RESULTS))
+include $(MDDEPEND_FILES)
 endif
 
 endif
 #############################################################################
 
 -include $(topsrcdir)/$(MOZ_BUILD_APP)/app-rules.mk
 -include $(MY_RULES)
 
--- a/js/src/configure.in
+++ b/js/src/configure.in
@@ -3971,17 +3971,17 @@ MOZ_EXPAND_LIBS
 dnl ========================================================
 dnl =
 dnl = Build depencency options
 dnl =
 dnl ========================================================
 MOZ_ARG_HEADER(Build dependencies)
 
 if test "$GNU_CC" -a "$GNU_CXX"; then
-  _DEPEND_CFLAGS='$(filter-out %/.pp,-MD -MF $(MDDEPDIR)/$(@F).pp)'
+  _DEPEND_CFLAGS='-MD -MP -MF $(MDDEPDIR)/$(@F).pp'
 dnl Sun Studio on Solaris use -xM instead of -MD, see config/rules.mk
 elif test "$SOLARIS_SUNPRO_CC"; then
   _DEPEND_CFLAGS=
 else
   dnl Don't override this for MSVC
   if test -z "$_WIN32_MSVC"; then
     _USE_CPP_INCLUDE_FLAG=
     _DEFINES_CFLAGS='$(ACDEFINES) -D_JS_CONFDEFS_H_ -DMOZILLA_CLIENT'
--- a/python/codegen/makeutils.py
+++ b/python/codegen/makeutils.py
@@ -13,9 +13,11 @@ def writeMakeDependOutput(filename):
     with open(filename, 'w') as f:
         if len(targets) > 0:
             f.write("%s:" % makeQuote(targets[0]))
             for filename in dependencies:
                 f.write(' \\\n\t\t%s' % makeQuote(filename))
             f.write('\n\n')
             for filename in targets[1:]:
                 f.write('%s: %s\n' % (makeQuote(filename), makeQuote(targets[0])))
+            for filename in dependencies:
+                f.write('%s:\n' % filename)
 
--- a/xpcom/idl-parser/header.py
+++ b/xpcom/idl-parser/header.py
@@ -533,8 +533,10 @@ if __name__ == '__main__':
             try:
                 os.makedirs(dirname)
             except:
                 pass
         depfd = open(options.depfile, 'w')
         deps = [dep.replace('\\', '/') for dep in idl.deps]
 
         print >>depfd, "%s: %s" % (options.outfile, " ".join(deps))
+        for dep in deps:
+            print >>depfd, "%s:" % dep