Make includes rebuild include files when necessary.
authorBenjamin Smedberg <benjamin@smedbergs.us>
Tue, 10 Feb 2009 09:50:42 -0500
changeset 73 2aed4b1eb9cb281f60144743bc298a27bfde6035
parent 72 4f6121f8ad3615344d803ecf5745e1f12945de28
child 74 aae6f068303c0e2efcdca30b962326fee97a240f
push id42
push userbsmedberg@mozilla.com
push dateTue, 10 Feb 2009 16:18:16 +0000
Make includes rebuild include files when necessary.
make.py
pymake/data.py
pymake/parser.py
tests/include-dynamic.mk
tests/include-notfound.mk
--- a/make.py
+++ b/make.py
@@ -6,45 +6,53 @@ make.py
 A drop-in or mostly drop-in replacement for GNU make.
 """
 
 import os, subprocess, sys, logging
 from optparse import OptionParser
 from pymake.data import Makefile, DataError
 from pymake.parser import parsestream, parsecommandlineargs, SyntaxError
 
+log = logging.getLogger('pymake.execution')
+
 op = OptionParser()
 op.add_option('-f', '--file', '--makefile',
               action='append',
               dest='makefiles',
               default=[])
 op.add_option('-v', '--verbose',
               action="store_true",
               dest="verbose", default=True)
 
 options, arguments = op.parse_args()
 
 if options.verbose:
     logging.basicConfig(level=logging.DEBUG)
 
-m = Makefile()
 if len(options.makefiles) == 0:
     if os.path.exists('Makefile'):
         options.makefiles.append('Makefile')
     else:
         print "No makefile found"
         sys.exit(2)
 
 try:
-    targets = parsecommandlineargs(m, arguments)
+    while True:
+        m = Makefile()
+        targets = parsecommandlineargs(m, arguments)
+
+        for f in options.makefiles:
+            m.include(f)
 
-    for f in options.makefiles:
-        parsestream(open(f), f, m)
+        m.finishparsing()
+        if m.remakemakefiles():
+            log.info("restarting makefile parsing")
+            continue
 
-    m.finishparsing()
+        break
 
     if len(targets) == 0:
         if m.defaulttarget is None:
             print "No target specified and no default target found."
             sys.exit(2)
         targets = [m.defaulttarget]
 
     tlist = [m.gettarget(t) for t in targets]
--- a/pymake/data.py
+++ b/pymake/data.py
@@ -170,18 +170,21 @@ class Variables(object):
         If the variable is not present, returns (None, None, None)
 
         @param expand If true, the value will be returned as an expansion. If false,
         it will be returned as an unexpanded string.
         """
         if name in self._map:
             flavor, source, valuestr = self._map[name]
             if flavor == self.FLAVOR_APPEND:
-                assert self.parent is not None
-                pflavor, psource, pvalue = self.parent.get(name, expand)
+                if self.parent:
+                    pflavor, psource, pvalue = self.parent.get(name, expand)
+                else:
+                    pflavor, psource, pvalue = None, None, None
+
                 if pvalue is None:
                     flavor = self.FLAVOR_RECURSIVE
                     # fall through
                 else:
                     if source > psource:
                         # TODO: log a warning?
                         return pflavor, psource, pvalue
 
@@ -471,17 +474,21 @@ class Target(object):
                 continue
 
             log.info("Found implicit rule at %s for target '%s'" % (r.loc, self.target))
             self.rules.append(r)
             return
 
         log.info("Couldn't find implicit rule to remake '%s'" % (self.target,))
 
-    def resolvedeps(self, makefile, targetstack, rulestack):
+    def ruleswithcommands(self):
+        "The number of rules with commands"
+        return reduce(lambda i, rule: i + (len(rule.commands) > 0), self.rules, 0)
+
+    def resolvedeps(self, makefile, targetstack, rulestack, required=True):
         """
         Resolve the actual path of this target, using vpath if necessary.
 
         Recursively resolve dependencies of this target. This means finding implicit
         rules which match the target, if appropriate.
 
         Figure out whether this target needs to be rebuild, and set self.outofdate
         appropriately.
@@ -498,35 +505,36 @@ class Target(object):
             raise ResolutionError("Recursive dependency: %s -> %s" % (
                     " -> ".join(targetstack), self.target))
 
         targetstack = targetstack + [self.target]
 
         self.resolvevpath(makefile)
 
         # Sanity-check our rules. If we're single-colon, only one rule should have commands
-        ruleswithcommands = reduce(lambda i, rule: i + len(rule.commands) > 0, self.rules, 0)
+        ruleswithcommands = self.ruleswithcommands()
         if len(self.rules) and not self.isdoublecolon():
             if ruleswithcommands > 1:
                 # In GNU make this is a warning, not an error. I'm going to be stricter.
                 # TODO: provide locations
                 raise DataError("Target '%s' has multiple rules with commands." % self.target)
 
         if ruleswithcommands == 0:
             found = self.resolveimplicitrule(makefile, targetstack, rulestack)
 
         # If a target is mentioned, but doesn't exist, has no commands and no
         # prerequisites, it is special and exists just to say that targets which
         # depend on it are always out of date. This is like .FORCE but more
         # compatible with other makes.
         # Otherwise, we don't know how to make it.
         if not len(self.rules) and self.mtime is None and not any((len(rule.prerequisitesfor(self.target)) > 0
                                                                    for rule in self.rules)):
-            raise ResolutionError("No rule to make %s needed to make %s" % (self.target,
-                                      ' -> '.join(targetstack[:-1])))
+            if required:
+                raise ResolutionError("No rule to make %s needed by %s" % (self.target,
+                    ' -> '.join(targetstack[:-1])))
 
         for r in self.rules:
             newrulestack = rulestack + [r]
             for d in r.prerequisitesfor(self.target):
                 makefile.gettarget(d).resolvedeps(makefile, targetstack, newrulestack)
 
         for v in makefile.getpatternvariablesfor(self.target):
             self.variables.merge(v)
@@ -559,57 +567,69 @@ class Target(object):
         self.mtime = None
         self.vpathtarget = self.target
 
     def make(self, makefile):
         """
         If we are out of date, make ourself.
 
         For now, making is synchronous/serialized. -j magic will come later.
+
+        @returns True if anything was done to remake this target
         """
         assert self.vpathtarget is not None, "Target was never resolved!"
 
+        log.info("Starting potential remake of '%s'" % (self.target,))
+
+        didanything = False
+
         if len(self.rules) == 0:
             assert self.mtime is not None
         elif self.isdoublecolon():
             for r in self.rules:
                 remake = False
                 if self.mtime is None:
                     log.info("Remaking %s using rule at %s because it doesn't exist or is a forced target" % (self.target, r.loc))
                     remake = True
                 for p in r.prerequisitesfor(self.target):
                     dep = makefile.gettarget(p)
-                    dep.make(makefile)
+                    didanything = dep.make(makefile) or didanything
                     if not remake and mtimeislater(dep.mtime, self.mtime):
                         log.info("Remaking %s using rule at %s because %s is newer." % (self.target, r.loc, p))
                         remake = True
                 if remake:
                     self.remake()
                     rule.execute(self, makefile)
+                    didanything = True
         else:
             commandrule = None
             remake = False
             if self.mtime is None:
                 log.info("Remaking %s because it doesn't exist or is a forced target" % (self.target,))
                 remake = True
 
             for r in self.rules:
                 if len(r.commands):
                     assert commandrule is None, "Two command rules for a single-colon target?"
                     commandrule = r
                 for p in r.prerequisitesfor(self.target):
                     dep = makefile.gettarget(p)
-                    dep.make(makefile)
+                    didanything = dep.make(makefile) or didanything
                     if not remake and mtimeislater(dep.mtime, self.mtime):
                         log.info("Remaking %s because %s is newer" % (self.target, p))
                         remake = True
 
-            if commandrule is not None and remake:
+            if remake:
                 self.remake()
-                commandrule.execute(self, makefile)
+                if commandrule is not None:
+                    commandrule.execute(self, makefile)
+                didanything = True
+
+        log.info("Did something to rebuild '%s'? %s" % (self.target, didanything))
+        return didanything
 
 def setautomaticvariables(v, makefile, target, prerequisites):
     vprereqs = [makefile.gettarget(p).vpathtarget
                 for p in prerequisites]
 
     v.set('@', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, target.vpathtarget)
 
     if len(vprereqs):
@@ -750,16 +770,19 @@ class Makefile(object):
     def __init__(self):
         self.defaulttarget = None
         self.variables = Variables()
         self._targets = {}
         self._patternvariables = [] # of (pattern, variables)
         self.implicitrules = []
         self.parsingfinished = False
 
+        # the list of included makefiles, whether or not they existed
+        self.included = []
+
     def foundtarget(self, t):
         """
         Inform the makefile of a target which is a candidate for being the default target,
         if there isn't already a default target.
         """
         if self.defaulttarget is None:
             self.defaulttarget = t
 
@@ -814,8 +837,38 @@ class Makefile(object):
             self.vpath = filter(lambda e: e != '', re.split('[:\s]+', value.resolve(self.variables, 'VPATH')))
 
         targets = list(self._targets.itervalues())
         for t in targets:
             t.explicit = True
             for r in t.rules:
                 for p in r.prerequisitesfor(t.target):
                     self.gettarget(p).explicit = True
+
+    def include(self, path, required=True):
+        """
+        Include the makefile at `path`.
+        """
+        self.included.append(path)
+        if os.path.exists(path):
+            fd = open(path)
+            self.variables.append('MAKEFILE_LIST', Variables.SOURCE_AUTOMATIC, path, None)
+            pymake.parser.parsestream(fd, path, self)
+            self.gettarget(path).explicit = True
+        elif required:
+            raise DataError("Attempting to include file which doesn't exist.")
+
+    def remakemakefiles(self):
+        reparse = False
+
+        tlist = [self.gettarget(f) for f in self.included]
+        for t in tlist:
+            t.explicit = True
+            t.resolvedeps(self, [], [], required=False)
+        for t in tlist:
+            if len(t.rules) > 0:
+                oldmtime = t.mtime
+                t.make(self)
+                if t.mtime != oldmtime:
+                    log.info("included makefile '%s' was remade" % t.target)
+                    reparse = True
+
+        return reparse
--- a/pymake/parser.py
+++ b/pymake/parser.py
@@ -474,17 +474,17 @@ class Condition(object):
             self.active = False
             return
 
         self.active = active
         if active:
             self.everactive = True
 
 directives = [k for k in conditionkeywords.iterkeys()] + \
-    ['else', 'endif', 'define', 'endef', 'override', 'include', 'vpath']
+    ['else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'vpath']
 
 varsettokens = (':=', '+=', '=')
 
 def parsestream(fd, filename, makefile):
     """
     Parse a stream of makefile into a makefile data structure.
 
     @param fd A file-like object containing the makefile data.
@@ -556,24 +556,21 @@ def parsestream(fd, filename, makefile):
                 d.readline()
 
                 setvariable(makefile.variables, makefile.variables,
                             e.resolve(makefile.variables, None),
                             '=', d, 0, iterdefinechars)
 
                 continue
 
-            if kword == 'include':
+            if kword in ('include', '-include'):
                 incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
                 files = data.splitwords(incfile.resolve(makefile.variables, None))
                 for f in files:
-                    # TODO: include files should be dependency-checked (added to targets in
-                    # some way)
-                    fd = open(f)
-                    parsestream(fd, f, makefile)
+                    makefile.include(f, kword == 'include')
                 continue
 
             if kword in conditionkeywords:
                 m = conditionkeywords[kword](d, offset, makefile)
                 condstack.append(Condition(m, d.getloc(offset)))
                 continue
 
             assert kword is None
new file mode 100644
--- /dev/null
+++ b/tests/include-dynamic.mk
@@ -0,0 +1,20 @@
+$(shell \
+if ! test -f include-dynamic.inc; then \
+  echo "TESTVAR = oldval" > include-dynamic.inc; \
+  sleep 1; \
+  echo "TESTVAR = newval" > include-dynamic.inc.in; \
+fi \
+)
+
+
+# before running the 'all' rule, we should be rebuilding include-dynamic.inc,
+# because there is a rule to do so
+
+all:
+	test $(TESTVAR) = newval
+	@echo TEST-PASS
+
+include-dynamic.inc: include-dynamic.inc.in
+	cp $< $@
+
+include include-dynamic.inc
new file mode 100644
--- /dev/null
+++ b/tests/include-notfound.mk
@@ -0,0 +1,13 @@
+ifneq ($(strip $(MAKEFILE_LIST)),$(TESTPATH)/include-notfound.mk)
+$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)')
+endif
+
+-include notfound.inc-dummy
+
+ifneq ($(strip $(MAKEFILE_LIST)),$(TESTPATH)/include-notfound.mk)
+$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)')
+endif
+
+all:
+	@echo TEST-PASS
+