Update pymake to pick up the fix for bug 523691 and some other stuff, NPODB.
authorBenjamin Smedberg <benjamin@smedbergs.us>
Fri, 23 Oct 2009 11:16:27 -0400
changeset 34127 b288c2d37d6bf899ca173dbecf3b27ec34aa6d1c
parent 34126 152507b9c592f49ef832098a853d0f11de9040b4
child 34128 6c1cc8d8f2917ec6dd571aaa8642b687408319e0
push idunknown
push userunknown
push dateunknown
bugs523691
milestone1.9.3a1pre
Update pymake to pick up the fix for bug 523691 and some other stuff, NPODB.
build/pymake/.hg_archival.txt
build/pymake/.hgignore
build/pymake/mkparse.py
build/pymake/pymake/command.py
build/pymake/pymake/data.py
build/pymake/pymake/functions.py
build/pymake/pymake/parser.py
build/pymake/pymake/parserdata.py
build/pymake/tests/comment-parsing.mk
build/pymake/tests/continuations-in-functions.mk
build/pymake/tests/func-refs.mk
build/pymake/tests/include-missing.mk
build/pymake/tests/include-regen.mk
build/pymake/tests/include-regen2.mk
build/pymake/tests/include-regen3.mk
build/pymake/tests/includedeps-norebuild.mk
build/pymake/tests/includedeps-sideeffects.mk
build/pymake/tests/includedeps.deps
build/pymake/tests/includedeps.mk
build/pymake/tests/line-continuations.mk
build/pymake/tests/oneline-command-continuations.mk
build/pymake/tests/override-propagate.mk
build/pymake/tests/parsertests.py
build/pymake/tests/runtests.py
build/pymake/tests/windows-paths.mk
--- a/build/pymake/.hg_archival.txt
+++ b/build/pymake/.hg_archival.txt
@@ -1,2 +1,2 @@
 repo: f5ab154deef2ffa97f1b2139589ae4a1962090a4
-node: 162a3b79a8b0799a9fb3c2aed909a3d09f6f1d39
+node: 0aa14be496b38ea56b28158f375d01dc0cb91d1f
--- a/build/pymake/.hgignore
+++ b/build/pymake/.hgignore
@@ -1,3 +1,4 @@
 \.pyc$
 \.pyo$
+tests/.*.(py|g)makelog$
 ~$
--- a/build/pymake/mkparse.py
+++ b/build/pymake/mkparse.py
@@ -1,11 +1,12 @@
 #!/usr/bin/env python
 
 import sys
 import pymake.parser
 
 for f in sys.argv[1:]:
     print "Parsing %s" % f
-    fd = open(f)
-    stmts = pymake.parser.parsestream(fd, f)
+    fd = open(f, 'rU')
+    s = fd.read()
     fd.close()
+    stmts = pymake.parser.parsestring(s, f)
     print stmts
--- a/build/pymake/pymake/command.py
+++ b/build/pymake/pymake/command.py
@@ -85,17 +85,22 @@ class _MakeContext(object):
         self.ostmts = ostmts
         self.overrides = overrides
         self.cb = cb
 
         self.restarts = 0
 
         self.remakecb(True)
 
-    def remakecb(self, remade):
+    def remakecb(self, remade, error=None):
+        if error is not None:
+            print error
+            self.context.defer(self.cb, 2)
+            return
+
         if remade:
             if self.restarts > 0:
                 _log.info("make.py[%i]: Restarting makefile parsing", self.makelevel)
 
             self.makefile = data.Makefile(restarts=self.restarts,
                                           make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
                                           makeflags=self.makeflags,
                                           makeoverrides=self.overrides,
--- a/build/pymake/pymake/data.py
+++ b/build/pymake/pymake/data.py
@@ -457,17 +457,17 @@ class Pattern(object):
 
         return Pattern(replacement).resolve('', stem)
 
     def __repr__(self):
         return "<Pattern with data %r>" % (self.data,)
 
     _backre = re.compile(r'[%\\]')
     def __str__(self):
-        if not self.ispattern:
+        if not self.ispattern():
             return self._backre.sub(r'\\\1', self.data[0])
 
         return self._backre.sub(r'\\\1', self.data[0]) + '%' + self.data[1]
 
 class RemakeTargetSerially(object):
     __slots__ = ('target', 'makefile', 'indent', 'rlist')
 
     def __init__(self, target, makefile, indent, rlist):
@@ -590,52 +590,65 @@ class RemakeRuleContext(object):
         self.rule = rule
         self.deps = deps
         self.targetstack = targetstack
         self.avoidremakeloop = avoidremakeloop
 
         self.running = False
         self.error = False
         self.depsremaining = len(deps) + 1
+        self.remake = False
 
     def resolvedeps(self, serial, cb):
         self.resolvecb = cb
         self.didanything = False
         if serial:
             self._resolvedepsserial()
         else:
             self._resolvedepsparallel()
 
+    def _weakdepfinishedserial(self, error, didanything):
+        if error:
+            self.remake = True
+        self._depfinishedserial(False, didanything)
+
     def _depfinishedserial(self, error, didanything):
         assert error in (True, False)
 
         if didanything:
             self.didanything = True
 
         if error:
             self.error = True
             if not self.makefile.keepgoing:
                 self.resolvecb(error=True, didanything=self.didanything)
                 return
         
         if len(self.resolvelist):
-            self.makefile.context.defer(self.resolvelist.pop(0).make,
-                                        self.makefile, self.targetstack, self._depfinishedserial)
+            dep, weak = self.resolvelist.pop(0)
+            self.makefile.context.defer(dep.make,
+                                        self.makefile, self.targetstack, weak and self._weakdepfinishedserial or self._depfinishedserial)
         else:
             self.resolvecb(error=self.error, didanything=self.didanything)
 
     def _resolvedepsserial(self):
         self.resolvelist = list(self.deps)
         self._depfinishedserial(False, False)
 
     def _startdepparallel(self, d):
         if self.makefile.error:
             depfinished(True, False)
         else:
-            d.make(self.makefile, self.targetstack, self._depfinishedparallel)
+            dep, weak = d
+            dep.make(self.makefile, self.targetstack, weak and self._weakdepfinishedparallel or self._depfinishedparallel)
+
+    def _weakdepfinishedparallel(self, error, didanything):
+        if error:
+            self.remake = True
+        self._depfinishedparallel(False, didanything)
 
     def _depfinishedparallel(self, error, didanything):
         assert error in (True, False)
 
         if error:
             print "<%s>: Found error" % self.target.target
             self.error = True
         if didanything:
@@ -673,39 +686,46 @@ class RemakeRuleContext(object):
         self.running = True
 
         self.runcb = cb
 
         if self.rule is None or not len(self.rule.commands):
             if self.target.mtime is None:
                 self.target.beingremade()
             else:
-                for d in self.deps:
+                for d, weak in self.deps:
                     if mtimeislater(d.mtime, self.target.mtime):
                         self.target.beingremade()
                         break
             cb(error=False)
             return
 
-        remake = False
-        if self.target.mtime is None:
-            remake = True
-            _log.info("%sRemaking %s using rule at %s: target doesn't exist or is a forced target", indent, self.target.target, self.rule.loc)
+        if self.rule.doublecolon:
+            if len(self.deps) == 0:
+                if self.avoidremakeloop:
+                    _log.info("%sNot remaking %s using rule at %s because it would introduce an infinite loop.", indent, self.target.target, self.rule.loc)
+                    cb(error=False)
+                    return
+
+        remake = self.remake
+        if remake:
+            _log.info("%sRemaking %s using rule at %s: weak dependency was not found.", indent, self.target.target, self.rule.loc)
+        else:
+            if self.target.mtime is None:
+                remake = True
+                _log.info("%sRemaking %s using rule at %s: target doesn't exist or is a forced target", indent, self.target.target, self.rule.loc)
 
         if not remake:
             if self.rule.doublecolon:
                 if len(self.deps) == 0:
-                    if self.avoidremakeloop:
-                        _log.info("%sNot remaking %s using rule at %s because it would introduce an infinite loop.", indent, self.target.target, self.rule.loc)
-                    else:
-                        _log.info("%sRemaking %s using rule at %s because there are no prerequisites listed for a double-colon rule.", indent, self.target.target, self.rule.loc)
-                        remake = True
+                    _log.info("%sRemaking %s using rule at %s because there are no prerequisites listed for a double-colon rule.", indent, self.target.target, self.rule.loc)
+                    remake = True
 
         if not remake:
-            for d in self.deps:
+            for d, weak in self.deps:
                 if mtimeislater(d.mtime, self.target.mtime):
                     _log.info("%sRemaking %s using rule at %s because %s is newer.", indent, self.target.target, self.rule.loc, d.target)
                     remake = True
                     break
 
         if remake:
             self.target.beingremade()
             self.target.didanything = True
@@ -731,16 +751,18 @@ class Target(object):
 
     It holds target-specific variables and a list of rules. It may also point to a parent
     PatternTarget, if this target is being created by an implicit rule.
 
     The rules associated with this target may be Rule instances or, in the case of static pattern
     rules, PatternRule instances.
     """
 
+    wasremade = False
+
     def __init__(self, target, makefile):
         assert isinstance(target, str)
         self.target = target
         self.vpathtarget = None
         self.rules = []
         self.variables = Variables(makefile.variables)
         self.explicit = False
         self._state = MAKESTATE_NONE
@@ -971,16 +993,17 @@ class Target(object):
         """
         When we remake ourself, we need to reset our mtime and vpathtarget.
 
         We store our old mtime so that $? can calculate out-of-date prerequisites.
         """
         self.realmtime = self.mtime
         self.mtime = None
         self.vpathtarget = self.target
+        self.wasremade = True
 
     def notifydone(self, makefile):
         assert self._state == MAKESTATE_WORKING, "State was %s" % self._state
 
         self._state = MAKESTATE_FINISHED
         for cb in self._callbacks:
             makefile.context.defer(cb, error=self.error, didanything=self.didanything)
         del self._callbacks 
@@ -1034,23 +1057,23 @@ class Target(object):
             return
 
         assert self.vpathtarget is not None, "Target was never resolved!"
         if not len(self.rules):
             self.notifydone(makefile)
             return
 
         if self.isdoublecolon():
-            rulelist = [RemakeRuleContext(self, makefile, r, [makefile.gettarget(p) for p in r.prerequisites], targetstack, avoidremakeloop) for r in self.rules]
+            rulelist = [RemakeRuleContext(self, makefile, r, [(makefile.gettarget(p), False) for p in r.prerequisites], targetstack, avoidremakeloop) for r in self.rules]
         else:
             alldeps = []
 
             commandrule = None
             for r in self.rules:
-                rdeps = [makefile.gettarget(p) for p in r.prerequisites]
+                rdeps = [(makefile.gettarget(p), r.weakdeps) for p in r.prerequisites]
                 if len(r.commands):
                     assert commandrule is None
                     commandrule = r
                     # The dependencies of the command rule are resolved before other dependencies,
                     # no matter the ordering of the other no-command rules
                     alldeps[0:0] = rdeps
                 else:
                     alldeps.extend(rdeps)
@@ -1168,33 +1191,36 @@ def getcommandsforrule(rule, target, mak
                                  echo=echo)
 
 class Rule(object):
     """
     A rule contains a list of prerequisites and a list of commands. It may also
     contain rule-specific variables. This rule may be associated with multiple targets.
     """
 
-    def __init__(self, prereqs, doublecolon, loc):
+    def __init__(self, prereqs, doublecolon, loc, weakdeps):
         self.prerequisites = prereqs
         self.doublecolon = doublecolon
         self.commands = []
         self.loc = loc
+        self.weakdeps = weakdeps
 
     def addcommand(self, c):
         assert isinstance(c, (Expansion, StringExpansion))
         self.commands.append(c)
 
     def getcommands(self, target, makefile):
         assert isinstance(target, Target)
 
         return getcommandsforrule(self, target, makefile, self.prerequisites, stem=None)
         # TODO: $* in non-pattern rules?
 
 class PatternRuleInstance(object):
+    weakdeps = False
+
     """
     A pattern rule instantiated for a particular target. It has the same API as Rule, but
     different internals, forwarding most information on to the PatternRule.
     """
     def __init__(self, prule, dir, stem, ismatchany):
         assert isinstance(prule, PatternRule)
 
         self.dir = dir
@@ -1264,37 +1290,44 @@ class PatternRule(object):
                     stem = p.match(file)
                     if stem is not None:
                         yield PatternRuleInstance(self, dir, stem, False)
 
     def prerequisitesforstem(self, dir, stem):
         return [p.resolve(dir, stem) for p in self.prerequisites]
 
 class _RemakeContext(object):
-    def __init__(self, makefile, remakelist, mtimelist, cb):
+    def __init__(self, makefile, cb):
         self.makefile = makefile
-        self.remakelist = remakelist
-        self.mtimelist = mtimelist # list of (target, mtime)
+        self.included = [(makefile.gettarget(f), required)
+                         for f, required in makefile.included]
+        self.toremake = list(self.included)
         self.cb = cb
 
         self.remakecb(error=False, didanything=False)
 
     def remakecb(self, error, didanything):
         assert error in (True, False)
 
-        if error:
+        if error and self.required:
             print "Error remaking makefiles (ignored)"
 
-        if len(self.remakelist):
-            self.remakelist.pop(0).make(self.makefile, [], avoidremakeloop=True, cb=self.remakecb)
+        if len(self.toremake):
+            target, self.required = self.toremake.pop(0)
+            target.make(self.makefile, [], avoidremakeloop=True, cb=self.remakecb)
         else:
-            for t, oldmtime in self.mtimelist:
-                if t.mtime != oldmtime:
+            for t, required in self.included:
+                if t.wasremade:
+                    _log.info("Included file %s was remade, restarting make", t.target)
                     self.cb(remade=True)
                     return
+                elif required and t.mtime is None:
+                    self.cb(remade=False, error=DataError("No rule to remaking missing include file %s", t.target))
+                    return
+
             self.cb(remade=False)
 
 class Makefile(object):
     """
     The top-level data structure for makefile execution. It holds Targets, implicit rules, and other
     state data.
     """
 
@@ -1438,29 +1471,27 @@ class Makefile(object):
                     self.gettarget(p).explicit = True
 
         np = self.gettarget('.NOTPARALLEL')
         if len(np.rules):
             self.context = process.getcontext(1)
 
         self.error = False
 
-    def include(self, path, required=True, loc=None):
+    def include(self, path, required=True, weak=False, loc=None):
         """
         Include the makefile at `path`.
         """
+        self.included.append((path, required))
         fspath = os.path.join(self.workdir, path)
         if os.path.exists(fspath):
-            self.included.append(path)
             stmts = parser.parsefile(fspath)
             self.variables.append('MAKEFILE_LIST', Variables.SOURCE_AUTOMATIC, path, None, self)
-            stmts.execute(self)
+            stmts.execute(self, weak=weak)
             self.gettarget(path).explicit = True
-        elif required:
-            raise DataError("Attempting to include file '%s' which doesn't exist." % (path,), loc)
 
     def addvpath(self, pattern, dirs):
         """
         Add a directory to the vpath search for the given pattern.
         """
         self._patternvpaths.append((pattern, dirs))
 
     def clearvpath(self, pattern):
@@ -1479,25 +1510,25 @@ class Makefile(object):
         for p, dirs in self._patternvpaths:
             if p.match(target):
                 vp.extend(dirs)
 
         return withoutdups(vp)
 
     def remakemakefiles(self, cb):
         mlist = []
-        for f in self.included:
+        for f, required in self.included:
             t = self.gettarget(f)
             t.explicit = True
             t.resolvevpath(self)
             oldmtime = t.mtime
 
             mlist.append((t, oldmtime))
 
-        _RemakeContext(self, [self.gettarget(f) for f in self.included], mlist, cb)
+        _RemakeContext(self, cb)
 
     def getsubenvironment(self, variables):
         env = dict(self.env)
         for vname, v in self.exportedvars.iteritems():
             if v:
                 flavor, source, val = variables.get(vname)
                 if val is None:
                     strval = ''
--- a/build/pymake/pymake/functions.py
+++ b/build/pymake/pymake/functions.py
@@ -1,13 +1,13 @@
 """
 Makefile functions.
 """
 
-import parser, data, util
+import parser, util
 import subprocess, os, logging
 from globrelative import glob
 from cStringIO import StringIO
 
 log = logging.getLogger('pymake.data')
 
 class Function(object):
     """
@@ -541,18 +541,18 @@ class EvalFunction(Function):
     maxargs = 1
 
     def resolve(self, makefile, variables, fd, setting):
         if makefile.parsingfinished:
             # GNU make allows variables to be set by recursive expansion during
             # command execution. This seems really dumb to me, so I don't!
             raise data.DataError("$(eval) not allowed via recursive expansion after parsing is finished", self.loc)
 
-        text = StringIO(self._arguments[0].resolvestr(makefile, variables, setting))
-        stmts = parser.parsestream(text, 'evaluation from %s' % self.loc)
+        stmts = parser.parsestring(self._arguments[0].resolvestr(makefile, variables, setting),
+                                   'evaluation from %s' % self.loc)
         stmts.execute(makefile)
 
 class OriginFunction(Function):
     name = 'origin'
     minargs = 1
     maxargs = 1
 
     __slots__ = Function.__slots__
@@ -688,8 +688,10 @@ functionmap = {
     'eval': EvalFunction,
     'origin': OriginFunction,
     'flavor': FlavorFunction,
     'shell': ShellFunction,
     'error': ErrorFunction,
     'warning': WarningFunction,
     'info': InfoFunction,
 }
+
+import data
--- a/build/pymake/pymake/parser.py
+++ b/build/pymake/pymake/parser.py
@@ -16,410 +16,299 @@ Otherwise, they are parsed as makefile s
 
 This file parses into the data structures defined in the parserdata module. Those classes are what actually
 do the dirty work of "executing" the parsed data into a data.Makefile.
 
 Four iterator functions are available:
 * iterdata
 * itermakefilechars
 * itercommandchars
-* iterdefinechars
 
 The iterators handle line continuations and comments in different ways, but share a common calling
 convention:
 
-Called with (data, startoffset, tokenlist)
+Called with (data, startoffset, tokenlist, finditer)
 
 yield 4-tuples (flatstr, token, tokenoffset, afteroffset)
 flatstr is data, guaranteed to have no tokens (may be '')
 token, tokenoffset, afteroffset *may be None*. That means there is more text
 coming.
 """
 
-import logging, re, os, bisect
+import logging, re, os, sys
 import data, functions, util, parserdata
 
 _log = logging.getLogger('pymake.parser')
 
 class SyntaxError(util.MakeError):
     pass
 
+_skipws = re.compile('\S')
 class Data(object):
     """
     A single virtual "line", which can be multiple source lines joined with
     continuations.
     """
 
-    __slots__ = ('data', 'startloc')
+    __slots__ = ('s', 'lstart', 'lend', 'loc')
 
-    def __init__(self, startloc, data):
-        self.data = data
-        self.startloc = startloc
+    def __init__(self, s, lstart, lend, loc):
+        self.s = s
+        self.lstart = lstart
+        self.lend = lend
+        self.loc = loc
 
     @staticmethod
-    def fromstring(str, path):
-        return Data(parserdata.Location(path, 1, 0), str)
-
-    def append(self, data):
-        self.data += data
+    def fromstring(s, path):
+        return Data(s, 0, len(s), parserdata.Location(path, 1, 0))
 
     def getloc(self, offset):
-        return self.startloc + self.data[:offset]
-
-    def endloc(self):
-        return self.startloc + self.data
+        assert offset >= self.lstart and offset <= self.lend
+        return self.loc.offset(self.s, self.lstart, offset)
 
     def skipwhitespace(self, offset):
         """
-        Return the offset into data after skipping whitespace.
-        """
-        while offset < len(self.data):
-            c = self.data[offset]
-            if not c.isspace():
-                break
-            offset += 1
-        return offset
-
-    def findtoken(self, o, tlist, skipws):
+        Return the offset of the first non-whitespace character in data starting at offset, or None if there are
+        only whitespace characters remaining.
         """
-        Check data at position o for any of the tokens in tlist followed by whitespace
-        or end-of-data.
-
-        If a token is found, skip trailing whitespace and return (token, newoffset).
-        Otherwise return None, oldoffset
-        """
-        assert isinstance(tlist, TokenList)
-
-        if skipws:
-            m = tlist.wslist.match(self.data, pos=o)
-            if m is not None:
-                return m.group(1), m.end(0)
-        else:
-            m = tlist.simplere.match(self.data, pos=o)
-            if m is not None:
-                return m.group(0), m.end(0)
-
-        return None, o
+        m = _skipws.search(self.s, offset, self.lend)
+        if m is None:
+            return None
 
-class DynamicData(Data):
-    """
-    If we're reading from a stream, allows reading additional data dynamically.
-    """
-    __slots__ = Data.__slots__ + ('lineiter',)
-
-    def __init__(self, lineiter, path):
-        try:
-            lineno, line = lineiter.next()
-            Data.__init__(self, parserdata.Location(path, lineno + 1, 0), line)
-            self.lineiter = lineiter
-        except StopIteration:
-            self.data = None
+        return m.start(0)
 
-    def readline(self):
-        try:
-            lineno, line = self.lineiter.next()
-            self.append(line)
-            return True
-        except StopIteration:
-            return False
-
-_makefiletokensescaped = [r'\\\\#', r'\\#', '\\\\\n', '\\\\\\s+\\\\\n', r'\\.', '#', '\n']
-_continuationtokensescaped = ['\\\\\n', r'\\.', '\n']
-
-class TokenList(object):
+_linere = re.compile(r'\\*\n')
+def enumeratelines(s, filename):
     """
-    A list of tokens to search. Because these lists are static, we can perform
-    optimizations (such as escaping and compiling regexes) on construction.
+    Enumerate lines in a string as Data objects, joining line
+    continuations.
     """
 
-    __slots__ = ('tlist', 'emptylist', 'escapedlist', 'simplere', 'makefilere', 'continuationre', 'wslist')
-
-    def __init__(self, tlist):
-        self.tlist = tlist
-        self.emptylist = len(tlist) == 0
-        self.escapedlist = [re.escape(t) for t in tlist]
+    off = 0
+    lineno = 1
+    curlines = 0
+    for m in _linere.finditer(s):
+        curlines += 1
+        start, end = m.span(0)
 
-    def __getattr__(self, name):
-        if name == 'simplere':
-            self.simplere = re.compile('|'.join(self.escapedlist))
-            return self.simplere
+        if (start - end) % 2 == 0:
+            # odd number of backslashes is a continuation
+            continue
 
-        if name == 'makefilere':
-            self.makefilere = re.compile('|'.join(self.escapedlist + _makefiletokensescaped))
-            return self.makefilere
-
-        if name == 'continuationre':
-            self.continuationre = re.compile('|'.join(self.escapedlist + _continuationtokensescaped))
-            return self.continuationre
+        yield Data(s, off, end - 1, parserdata.Location(filename, lineno, 0))
 
-        if name == 'wslist':
-            self.wslist = re.compile('(' + '|'.join(self.escapedlist) + ')' + r'(\s+|$)')
-            return self.wslist
+        lineno += curlines
+        curlines = 0
+        off = end
 
-        raise AttributeError(name)
-
-    _imap = {}
+    yield Data(s, off, len(s), parserdata.Location(filename, lineno, 0))
 
-    @staticmethod
-    def get(s):
-        i = TokenList._imap.get(s, None)
-        if i is None:
-            i = TokenList(s)
-            TokenList._imap[s] = i
+_alltokens = re.compile(r'''\\*\# | # hash mark preceeded by any number of backslashes
+                            := |
+                            \+= |
+                            \?= |
+                            :: |
+                            (?:\$(?:$|[\(\{](?:%s)\s+|.)) | # dollar sign followed by EOF, a function keyword with whitespace, or any character
+                            :(?![\\/]) | # colon followed by anything except a slash (Windows path detection)
+                            [=#{}();,|'"]''' % '|'.join(functions.functionmap.iterkeys()), re.VERBOSE)
 
-        return i
-
-_emptytokenlist = TokenList.get('')
-
-def iterdata(d, offset, tokenlist):
+def iterdata(d, offset, tokenlist, it):
     """
     Iterate over flat data without line continuations, comments, or any special escaped characters.
 
     Typically used to parse recursively-expanded variables.
     """
 
-    if tokenlist.emptylist:
-        yield d.data, None, None, None
+    assert len(tokenlist), "Empty tokenlist passed to iterdata is meaningless!"
+    assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
+
+    if offset == d.lend:
         return
 
-    s = tokenlist.simplere
-    datalen = len(d.data)
+    s = d.s
+    for m in it:
+        mstart, mend = m.span(0)
+        token = s[mstart:mend]
+        if token in tokenlist or (token[0] == '$' and '$' in tokenlist):
+            yield s[offset:mstart], token, mstart, mend
+        else:
+            yield s[offset:mend], None, None, mend
+        offset = mend
 
-    while offset < datalen:
-        m = s.search(d.data, pos=offset)
-        if m is None:
-            yield d.data[offset:], None, None, None
-            return
+    yield s[offset:d.lend], None, None, None
 
-        yield d.data[offset:m.start(0)], m.group(0), m.start(0), m.end(0)
-        offset = m.end(0)
+# multiple backslashes before a newline are unescaped, halving their total number
+_makecontinuations = re.compile(r'(?:\s*|((?:\\\\)+))\\\n\s*')
+def _replacemakecontinuations(m):
+    start, end = m.span(1)
+    if start == -1:
+        return ' '
+    return ' '.rjust((end - start) / 2 + 1, '\\')
 
-def itermakefilechars(d, offset, tokenlist, ignorecomments=False):
+def itermakefilechars(d, offset, tokenlist, it, ignorecomments=False):
     """
     Iterate over data in makefile syntax. Comments are found at unescaped # characters, and escaped newlines
     are converted to single-space continuations.
     """
 
-    s = tokenlist.makefilere
+    assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
+
+    if offset == d.lend:
+        return
 
-    while offset < len(d.data):
-        m = s.search(d.data, pos=offset)
-        if m is None:
-            yield d.data[offset:], None, None, None
-            return
+    s = d.s
+    for m in it:
+        mstart, mend = m.span(0)
+        token = s[mstart:mend]
 
-        token = m.group(0)
-        start = m.start(0)
-        end = m.end(0)
+        starttext = _makecontinuations.sub(_replacemakecontinuations, s[offset:mstart])
 
-        if token == '\n':
-            assert end == len(d.data)
-            yield d.data[offset:start], None, None, None
-            return
+        if token[-1] == '#' and not ignorecomments:
+            l = mend - mstart
+            # multiple backslashes before a hash are unescaped, halving their total number
+            if l % 2:
+                # found a comment
+                yield starttext + token[:(l - 1) / 2], None, None, None
+                return
+            else:
+                yield starttext + token[-l / 2:], None, None, mend
+        elif token in tokenlist or (token[0] == '$' and '$' in tokenlist):
+            yield starttext, token, mstart, mend
+        else:
+            yield starttext + token, None, None, mend
+        offset = mend
 
-        if token == '#':
-            if ignorecomments:
-                yield d.data[offset:end], None, None, None
-                offset = end
-                continue
-
-            yield d.data[offset:start], None, None, None
-            for s in itermakefilechars(d, end, _emptytokenlist): pass
-            return
+    yield _makecontinuations.sub(_replacemakecontinuations, s[offset:d.lend]), None, None, None
 
-        if token == '\\\\#':
-            # see escape-chars.mk VARAWFUL
-            yield d.data[offset:start + 1], None, None, None
-            for s in itermakefilechars(d, end, _emptytokenlist): pass
-            return
+_findcomment = re.compile(r'\\*\#')
+def flattenmakesyntax(d, offset):
+    """
+    A shortcut method for flattening line continuations and comments in makefile syntax without
+    looking for other tokens.
+    """
 
-        if token == '\\\n':
-            yield d.data[offset:start].rstrip() + ' ', None, None, None
-            d.readline()
-            offset = d.skipwhitespace(end)
-            continue
+    assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
+    if offset == d.lend:
+        return ''
+
+    s = _makecontinuations.sub(_replacemakecontinuations, d.s[offset:d.lend])
 
-        if token.startswith('\\') and token.endswith('\n'):
-            assert end == len(d.data)
-            yield d.data[offset:start] + '\\ ', None, None, None
-            d.readline()
-            offset = d.skipwhitespace(end)
-            continue
+    elements = []
+    offset = 0
+    for m in _findcomment.finditer(s):
+        mstart, mend = m.span(0)
+        elements.append(s[offset:mstart])
+        if (mend - mstart) % 2:
+            # even number of backslashes... it's a comment
+            elements.append(''.ljust((mend - mstart - 1) / 2, '\\'))
+            return ''.join(elements)
 
-        if token == '\\#':
-            yield d.data[offset:start] + '#', None, None, None
-        elif token.startswith('\\'):
-            if token[1:] in tokenlist.tlist:
-                yield d.data[offset:start + 1], token[1:], start + 1, end
-            else:
-                yield d.data[offset:end], None, None, None
-        else:
-            yield d.data[offset:start], token, start, end
+        # odd number of backslashes
+        elements.append(''.ljust((mend - mstart - 2) / 2, '\\') + '#')
+        offset = mend
 
-        offset = end
+    elements.append(s[offset:])
+    return ''.join(elements)
 
-def itercommandchars(d, offset, tokenlist):
+def itercommandchars(d, offset, tokenlist, it):
     """
     Iterate over command syntax. # comment markers are not special, and escaped newlines are included
     in the output text.
     """
 
-    s = tokenlist.continuationre
+    assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
 
-    while offset < len(d.data):
-        m = s.search(d.data, pos=offset)
-        if m is None:
-            yield d.data[offset:], None, None, None
-            return
+    if offset == d.lend:
+        return
 
-        token = m.group(0)
-        start = m.start(0)
-        end = m.end(0)
-
-        if token == '\n':
-            assert end == len(d.data)
-            yield d.data[offset:start], None, None, None
-            return
+    s = d.s
+    for m in it:
+        mstart, mend = m.span(0)
+        token = s[mstart:mend]
+        starttext = s[offset:mstart].replace('\n\t', '\n')
 
-        if token == '\\\n':
-            yield d.data[offset:end], None, None, None
-            d.readline()
-            offset = end
-            if offset < len(d.data) and d.data[offset] == '\t':
-                offset += 1
-            continue
-        
-        if token.startswith('\\'):
-            if token[1:] in tokenlist.tlist:
-                yield d.data[offset:start + 1], token[1:], start + 1, end
-            else:
-                yield d.data[offset:end], None, None, None
+        if token in tokenlist or (token[0] == '$' and '$' in tokenlist):
+            yield starttext, token, mstart, mend
         else:
-            yield d.data[offset:start], token, start, end
+            yield starttext + token, None, None, mend
+        offset = mend
 
-        offset = end
+    yield s[offset:d.lend].replace('\n\t', '\n'), None, None, None
 
-_definestokenlist = TokenList.get(('define', 'endef'))
-
-def iterdefinechars(d, offset, tokenlist):
+_redefines = re.compile('define|endef')
+def iterdefinelines(it, startloc):
     """
-    Iterate over define blocks. Most characters are included literally. Escaped newlines are treated
+    Process the insides of a define. Most characters are included literally. Escaped newlines are treated
     as they would be in makefile syntax. Internal define/endef pairs are ignored.
     """
 
-    def checkfortoken(o):
-        """
-        Check for a define or endef token on the line starting at o.
-        Return an integer for the direction of definecount.
-        """
-        if o >= len(d.data):
-            return 0
-
-        if d.data[o] == '\t':
-            return 0
-
-        o = d.skipwhitespace(o)
-        token, o = d.findtoken(o, _definestokenlist, True)
-        if token == 'define':
-            return 1
-
-        if token == 'endef':
-            return -1
-        
-        return 0
-
-    startoffset = offset
-    definecount = 1 + checkfortoken(offset)
-    if definecount == 0:
-        return
-
-    s = tokenlist.continuationre
-
-    while offset < len(d.data):
-        m = s.search(d.data, pos=offset)
-        if m is None:
-            yield d.data[offset:], None, None, None
-            break
+    results = []
 
-        token = m.group(0)
-        start = m.start(0)
-        end = m.end(0)
-
-        if token == '\\\n':
-            yield d.data[offset:start].rstrip() + ' ', None, None, None
-            d.readline()
-            offset = d.skipwhitespace(end)
-            continue
-
-        if token == '\n':
-            assert end == len(d.data), "end: %r len(d.data): %r" % (end, len(d.data))
-            d.readline()
-            definecount += checkfortoken(end)
-            if definecount == 0:
-                yield d.data[offset:start], None, None, None
-                return
+    definecount = 1
+    for d in it:
+        m = _redefines.match(d.s, d.lstart, d.lend)
+        if m is not None:
+            directive = m.group(0)
+            if directive == 'endef':
+                definecount -= 1
+                if definecount == 0:
+                    return _makecontinuations.sub(_replacemakecontinuations, '\n'.join(results))
+            else:
+                definecount += 1
 
-            yield d.data[offset:end], None, None, None
-        elif token.startswith('\\'):
-            if token[1:] in tokenlist.tlist:
-                yield d.data[offset:start + 1], token[1:], start + 1, end
-            else:
-                yield d.data[offset:end], None, None, None
-        else:
-            yield d.data[offset:start], token, start, end
+        results.append(d.s[d.lstart:d.lend])
 
-        offset = end
+    # Falling off the end is an unterminated define!
+    raise SyntaxError("define without matching endef", startloc)
 
-    # Unlike the other iterators, if you fall off this one there is an unterminated
-    # define.
-    raise SyntaxError("Unterminated define", d.getloc(startoffset))
-
-def _iterflatten(iter, data, offset):
-    return ''.join((str for str, t, o, oo in iter(data, offset, _emptytokenlist)))
-
-def _ensureend(d, offset, msg, ifunc=itermakefilechars):
+def _ensureend(d, offset, msg):
     """
     Ensure that only whitespace remains in this data.
     """
 
-    for c, t, o, oo in ifunc(d, offset, _emptytokenlist):
-        if c != '' and not c.isspace():
-            raise SyntaxError(msg, d.getloc(o))
+    s = flattenmakesyntax(d, offset)
+    if s != '' and not s.isspace():
+        raise SyntaxError(msg, d.getloc(offset))
 
-_eqargstokenlist = TokenList.get(('(', "'", '"'))
+_eqargstokenlist = ('(', "'", '"')
 
 def ifeq(d, offset):
+    if offset > d.lend - 1:
+        raise SyntaxError("No arguments after conditional", d.getloc(offset))
+
     # the variety of formats for this directive is rather maddening
-    token, offset = d.findtoken(offset, _eqargstokenlist, False)
-    if token is None:
+    token = d.s[offset]
+    if token not in _eqargstokenlist:
         raise SyntaxError("No arguments after conditional", d.getloc(offset))
 
+    offset += 1
+
     if token == '(':
         arg1, t, offset = parsemakesyntax(d, offset, (',',), itermakefilechars)
         if t is None:
             raise SyntaxError("Expected two arguments in conditional", d.getloc(offset))
 
         arg1.rstrip()
 
         offset = d.skipwhitespace(offset)
         arg2, t, offset = parsemakesyntax(d, offset, (')',), itermakefilechars)
         if t is None:
             raise SyntaxError("Unexpected text in conditional", d.getloc(offset))
 
         _ensureend(d, offset, "Unexpected text after conditional")
     else:
         arg1, t, offset = parsemakesyntax(d, offset, (token,), itermakefilechars)
         if t is None:
-            raise SyntaxError("Unexpected text in conditional", d.getloc(offset))
+            raise SyntaxError("Unexpected text in conditional", d.getloc(d.lend))
 
         offset = d.skipwhitespace(offset)
-        if offset == len(d.data):
+        if offset == d.lend:
             raise SyntaxError("Expected two arguments in conditional", d.getloc(offset))
 
-        token = d.data[offset]
+        token = d.s[offset]
         if token not in '\'"':
             raise SyntaxError("Unexpected text in conditional", d.getloc(offset))
 
         arg2, t, offset = parsemakesyntax(d, offset + 1, (token,), itermakefilechars)
 
         _ensureend(d, offset, "Unexpected text after conditional")
 
     return parserdata.EqCondition(arg1, arg2)
@@ -443,25 +332,28 @@ def ifndef(d, offset):
 _conditionkeywords = {
     'ifeq': ifeq,
     'ifneq': ifneq,
     'ifdef': ifdef,
     'ifndef': ifndef
     }
 
 _conditiontokens = tuple(_conditionkeywords.iterkeys())
-_directivestokenlist = TokenList.get(_conditiontokens + \
-    ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'vpath', 'export', 'unexport'))
-_conditionkeywordstokenlist = TokenList.get(_conditiontokens)
+_conditionre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_conditiontokens))
+
+_directivestokenlist = _conditiontokens + \
+    ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'includedeps', '-includedeps', 'vpath', 'export', 'unexport')
+
+_directivesre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_directivestokenlist))
 
 _varsettokens = (':=', '+=', '?=', '=')
 
 def _parsefile(pathname):
     fd = open(pathname, "rU")
-    stmts = parsestream(fd, pathname)
+    stmts = parsestring(fd.read(), pathname)
     stmts.mtime = os.fstat(fd.fileno()).st_mtime
     fd.close()
     return stmts
 
 def _checktime(path, stmts):
     mtime = os.path.getmtime(path)
     if mtime != stmts.mtime:
         _log.debug("Re-parsing makefile '%s': mtimes differ", path)
@@ -475,214 +367,222 @@ def parsefile(pathname):
     """
     Parse a filename into a parserdata.StatementList. A cache is used to avoid re-parsing
     makefiles that have already been parsed and have not changed.
     """
 
     pathname = os.path.realpath(pathname)
     return _parsecache.get(pathname)
 
-def parsestream(fd, filename):
+def parsestring(s, filename):
     """
-    Parse a stream of makefile into a parserdata.StatementList. To parse a file system file, use
-    parsefile instead of this method.
-
-    @param fd A file-like object containing the makefile data.
+    Parse a string containing makefile data into a parserdata.StatementList.
     """
 
     currule = False
     condstack = [parserdata.StatementList()]
 
-    fdlines = enumerate(fd)
-
-    while True:
+    fdlines = enumeratelines(s, filename)
+    for d in fdlines:
         assert len(condstack) > 0
 
-        d = DynamicData(fdlines, filename)
-        if d.data is None:
-            break
+        offset = d.lstart
 
-        if len(d.data) > 0 and d.data[0] == '\t' and currule:
-            e, t, o = parsemakesyntax(d, 1, (), itercommandchars)
-            assert t == None
+        if currule and offset < d.lend and d.s[offset] == '\t':
+            e, token, offset = parsemakesyntax(d, offset + 1, (), itercommandchars)
+            assert token is None
+            assert offset is None
             condstack[-1].append(parserdata.Command(e))
-        else:
-            # To parse Makefile syntax, we first strip leading whitespace and
-            # look for initial keywords. If there are no keywords, it's either
-            # setting a variable or writing a rule.
+            continue
+
+        # To parse Makefile syntax, we first strip leading whitespace and
+        # look for initial keywords. If there are no keywords, it's either
+        # setting a variable or writing a rule.
 
-            offset = d.skipwhitespace(0)
+        offset = d.skipwhitespace(offset)
+        if offset is None:
+            continue
 
-            kword, offset = d.findtoken(offset, _directivestokenlist, True)
+        m = _directivesre.match(d.s, offset, d.lend)
+        if m is not None:
+            kword = m.group(1)
+            offset = m.end(0)
+
             if kword == 'endif':
                 _ensureend(d, offset, "Unexpected data after 'endif' directive")
                 if len(condstack) == 1:
                     raise SyntaxError("unmatched 'endif' directive",
                                       d.getloc(offset))
 
                 condstack.pop().endloc = d.getloc(offset)
                 continue
             
             if kword == 'else':
                 if len(condstack) == 1:
                     raise SyntaxError("unmatched 'else' directive",
                                       d.getloc(offset))
 
-                kword, offset = d.findtoken(offset, _conditionkeywordstokenlist, True)
-                if kword is None:
+                m = _conditionre.match(d.s, offset, d.lend)
+                if m is None:
                     _ensureend(d, offset, "Unexpected data after 'else' directive.")
-                    condstack[-1].addcondition(d.getloc(0), parserdata.ElseCondition())
+                    condstack[-1].addcondition(d.getloc(offset), parserdata.ElseCondition())
                 else:
+                    kword = m.group(1)
                     if kword not in _conditionkeywords:
                         raise SyntaxError("Unexpected condition after 'else' directive.",
                                           d.getloc(offset))
 
+                    startoffset = offset
+                    offset = d.skipwhitespace(m.end(1))
                     c = _conditionkeywords[kword](d, offset)
-                    condstack[-1].addcondition(d.getloc(0), c)
+                    condstack[-1].addcondition(d.getloc(startoffset), c)
                 continue
 
             if kword in _conditionkeywords:
                 c = _conditionkeywords[kword](d, offset)
-                cb = parserdata.ConditionBlock(d.getloc(0), c)
+                cb = parserdata.ConditionBlock(d.getloc(d.lstart), c)
                 condstack[-1].append(cb)
                 condstack.append(cb)
                 continue
 
             if kword == 'endef':
-                raise SyntaxError("Unmatched endef", d.getloc(offset))
+                raise SyntaxError("endef without matching define", d.getloc(offset))
 
             if kword == 'define':
                 currule = False
                 vname, t, i = parsemakesyntax(d, offset, (), itermakefilechars)
                 vname.rstrip()
 
-                startpos = len(d.data)
-                if not d.readline():
-                    raise SyntaxError("Unterminated define", d.getloc())
-
-                value = _iterflatten(iterdefinechars, d, startpos)
-                condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=d.getloc(0), token='=', targetexp=None, source=data.Variables.SOURCE_MAKEFILE, endloc=d.endloc()))
+                startloc = d.getloc(d.lstart)
+                value = iterdefinelines(fdlines, startloc)
+                condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=startloc, token='=', targetexp=None))
                 continue
 
-            if kword in ('include', '-include'):
+            if kword in ('include', '-include', 'includedeps', '-includedeps'):
+                if kword.startswith('-'):
+                    required = False
+                    kword = kword[1:]
+                else:
+                    required = True
+
+                deps = kword == 'includedeps'
+
                 currule = False
                 incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
-                condstack[-1].append(parserdata.Include(incfile, kword == 'include'))
+                condstack[-1].append(parserdata.Include(incfile, required, deps))
 
                 continue
 
             if kword == 'vpath':
                 currule = False
                 e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
                 condstack[-1].append(parserdata.VPathDirective(e))
                 continue
 
             if kword == 'override':
                 currule = False
                 vname, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars)
                 vname.lstrip()
                 vname.rstrip()
 
                 if token is None:
-                    raise SyntaxError("Malformed override directive, need =", d.getloc(offset))
+                    raise SyntaxError("Malformed override directive, need =", d.getloc(d.lstart))
 
-                value = _iterflatten(itermakefilechars, d, offset).lstrip()
+                value = flattenmakesyntax(d, offset).lstrip()
 
-                condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_OVERRIDE, endloc=d.endloc()))
+                condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_OVERRIDE))
                 continue
 
             if kword == 'export':
                 currule = False
                 e, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars)
                 e.lstrip()
                 e.rstrip()
 
                 if token is None:
                     condstack[-1].append(parserdata.ExportDirective(e, single=False))
                 else:
                     condstack[-1].append(parserdata.ExportDirective(e, single=True))
 
-                    value = _iterflatten(itermakefilechars, d, offset).lstrip()
-                    condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_MAKEFILE, endloc=d.endloc()))
+                    value = flattenmakesyntax(d, offset).lstrip()
+                    condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None))
 
                 continue
 
             if kword == 'unexport':
                 e, token, offset = parsemakesyntax(d, offset, (), itermakefilechars)
                 condstack[-1].append(parserdata.UnexportDirective(e))
                 continue
 
-            assert kword is None, "unexpected kword: %r" % (kword,)
+        e, token, offset = parsemakesyntax(d, offset, _varsettokens + ('::', ':'), itermakefilechars)
+        if token is None:
+            e.rstrip()
+            e.lstrip()
+            if not e.isempty():
+                condstack[-1].append(parserdata.EmptyDirective(e))
+            continue
+
+        # if we encountered real makefile syntax, the current rule is over
+        currule = False
+
+        if token in _varsettokens:
+            e.lstrip()
+            e.rstrip()
+
+            value = flattenmakesyntax(d, offset).lstrip()
+
+            condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None))
+        else:
+            doublecolon = token == '::'
 
-            e, token, offset = parsemakesyntax(d, offset, _varsettokens + ('::', ':'), itermakefilechars)
-            if token is None:
-                e.rstrip()
-                e.lstrip()
-                if not e.isempty():
-                    condstack[-1].append(parserdata.EmptyDirective(e))
-                continue
+            # `e` is targets or target patterns, which can end up as
+            # * a rule
+            # * an implicit rule
+            # * a static pattern rule
+            # * a target-specific variable definition
+            # * a pattern-specific variable definition
+            # any of the rules may have order-only prerequisites
+            # delimited by |, and a command delimited by ;
+            targets = e
 
-            # if we encountered real makefile syntax, the current rule is over
-            currule = False
+            e, token, offset = parsemakesyntax(d, offset,
+                                               _varsettokens + (':', '|', ';'),
+                                               itermakefilechars)
+            if token in (None, ';'):
+                condstack[-1].append(parserdata.Rule(targets, e, doublecolon))
+                currule = True
 
-            if token in _varsettokens:
+                if token == ';':
+                    offset = d.skipwhitespace(offset)
+                    e, t, offset = parsemakesyntax(d, offset, (), itercommandchars)
+                    condstack[-1].append(parserdata.Command(e))
+
+            elif token in _varsettokens:
                 e.lstrip()
                 e.rstrip()
 
-                value = _iterflatten(itermakefilechars, d, offset).lstrip()
-
-                condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_MAKEFILE, endloc=d.endloc()))
+                value = flattenmakesyntax(d, offset).lstrip()
+                condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=targets))
+            elif token == '|':
+                raise SyntaxError('order-only prerequisites not implemented', d.getloc(offset))
             else:
-                doublecolon = token == '::'
+                assert token == ':'
+                # static pattern rule
 
-                # `e` is targets or target patterns, which can end up as
-                # * a rule
-                # * an implicit rule
-                # * a static pattern rule
-                # * a target-specific variable definition
-                # * a pattern-specific variable definition
-                # any of the rules may have order-only prerequisites
-                # delimited by |, and a command delimited by ;
-                targets = e
-
-                e, token, offset = parsemakesyntax(d, offset,
-                                                   _varsettokens + (':', '|', ';'),
-                                                   itermakefilechars)
-                if token in (None, ';'):
-                    condstack[-1].append(parserdata.Rule(targets, e, doublecolon))
-                    currule = True
+                pattern = e
 
-                    if token == ';':
-                        offset = d.skipwhitespace(offset)
-                        e, t, offset = parsemakesyntax(d, offset, (), itercommandchars)
-                        condstack[-1].append(parserdata.Command(e))
+                deps, token, offset = parsemakesyntax(d, offset, (';',), itermakefilechars)
 
-                elif token in _varsettokens:
-                    e.lstrip()
-                    e.rstrip()
+                condstack[-1].append(parserdata.StaticPatternRule(targets, pattern, deps, doublecolon))
+                currule = True
 
-                    value = _iterflatten(itermakefilechars, d, offset).lstrip()
-                    condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=targets, source=data.Variables.SOURCE_MAKEFILE, endloc=d.endloc()))
-                elif token == '|':
-                    raise SyntaxError('order-only prerequisites not implemented', d.getloc(offset))
-                else:
-                    assert token == ':'
-                    # static pattern rule
-
-                    pattern = e
-
-                    deps, token, offset = parsemakesyntax(d, offset, (';',), itermakefilechars)
-
-                    condstack[-1].append(parserdata.StaticPatternRule(targets, pattern, deps, doublecolon))
-                    currule = True
-
-                    if token == ';':
-                        offset = d.skipwhitespace(offset)
-                        e, token, offset = parsemakesyntax(d, offset, (), itercommandchars)
-                        condstack[-1].append(parserdata.Command(e))
+                if token == ';':
+                    offset = d.skipwhitespace(offset)
+                    e, token, offset = parsemakesyntax(d, offset, (), itercommandchars)
+                    condstack[-1].append(parserdata.Command(e))
 
     if len(condstack) != 1:
         raise SyntaxError("Condition never terminated with endif", condstack[-1].loc)
 
     return condstack[0]
 
 _PARSESTATE_TOPLEVEL = 0    # at the top level
 _PARSESTATE_FUNCTION = 1    # expanding a function call
@@ -699,155 +599,145 @@ class ParseStackFrame(object):
         self.parent = parent
         self.expansion = expansion
         self.tokenlist = tokenlist
         self.openbrace = openbrace
         self.closebrace = closebrace
         self.function = function
         self.loc = loc
 
-_functiontokenlist = None
-
 _matchingbrace = {
     '(': ')',
     '{': '}',
     }
 
-def parsemakesyntax(d, startat, stopon, iterfunc):
+def parsemakesyntax(d, offset, stopon, iterfunc):
     """
     Given Data, parse it into a data.Expansion.
 
     @param stopon (sequence)
         Indicate characters where toplevel parsing should stop.
 
     @param iterfunc (generator function)
         A function which is used to iterate over d, yielding (char, offset, loc)
         @see iterdata
         @see itermakefilechars
         @see itercommandchars
  
     @return a tuple (expansion, token, offset). If all the data is consumed,
     token and offset will be None
     """
 
-    # print "parsemakesyntax(%r)" % d.data
-
-    global _functiontokenlist
-    if _functiontokenlist is None:
-        functiontokens = list(functions.functionmap.iterkeys())
-        functiontokens.sort(key=len, reverse=True)
-        _functiontokenlist = TokenList.get(tuple(functiontokens))
-
     assert callable(iterfunc)
 
-    stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL, None, data.Expansion(loc=d.getloc(startat)),
-                               tokenlist=TokenList.get(stopon + ('$',)),
+    stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL, None, data.Expansion(loc=d.getloc(d.lstart)),
+                               tokenlist=stopon + ('$',),
                                openbrace=None, closebrace=None)
 
-    di = iterfunc(d, startat, stacktop.tokenlist)
+    tokeniterator = _alltokens.finditer(d.s, offset, d.lend)
+
+    di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator)
     while True: # this is not a for loop because `di` changes during the function
         assert stacktop is not None
         try:
             s, token, tokenoffset, offset = di.next()
         except StopIteration:
             break
 
         stacktop.expansion.appendstr(s)
         if token is None:
             continue
 
         parsestate = stacktop.parsestate
 
-        if token == '$':
-            if len(d.data) == offset:
+        if token[0] == '$':
+            if tokenoffset + 1 == d.lend:
                 # an unterminated $ expands to nothing
                 break
 
             loc = d.getloc(tokenoffset)
-
-            c = d.data[offset]
+            c = token[1]
             if c == '$':
+                assert len(token) == 2
                 stacktop.expansion.appendstr('$')
-                offset = offset + 1
             elif c in ('(', '{'):
                 closebrace = _matchingbrace[c]
 
-                # look forward for a function name
-                fname, offset = d.findtoken(offset + 1, _functiontokenlist, True)
-                if fname is not None:
+                if len(token) > 2:
+                    fname = token[2:].rstrip()
                     fn = functions.functionmap[fname](loc)
                     e = data.Expansion()
                     if len(fn) + 1 == fn.maxargs:
-                        tokenlist = TokenList.get((c, closebrace, '$'))
+                        tokenlist = (c, closebrace, '$')
                     else:
-                        tokenlist = TokenList.get((',', c, closebrace, '$'))
+                        tokenlist = (',', c, closebrace, '$')
 
                     stacktop = ParseStackFrame(_PARSESTATE_FUNCTION, stacktop,
                                                e, tokenlist, function=fn,
                                                openbrace=c, closebrace=closebrace)
                 else:
                     e = data.Expansion()
-                    tokenlist = TokenList.get((':', c, closebrace, '$'))
+                    tokenlist = (':', c, closebrace, '$')
                     stacktop = ParseStackFrame(_PARSESTATE_VARNAME, stacktop,
                                                e, tokenlist,
                                                openbrace=c, closebrace=closebrace, loc=loc)
             else:
+                assert len(token) == 2
                 e = data.Expansion.fromstring(c, loc)
                 stacktop.expansion.appendfunc(functions.VariableRef(loc, e))
-                offset += 1
         elif token in ('(', '{'):
             assert token == stacktop.openbrace
 
             stacktop.expansion.appendstr(token)
             stacktop = ParseStackFrame(_PARSESTATE_PARENMATCH, stacktop,
                                        stacktop.expansion,
-                                       TokenList.get((token, stacktop.closebrace,)),
+                                       (token, stacktop.closebrace),
                                        openbrace=token, closebrace=stacktop.closebrace, loc=d.getloc(tokenoffset))
         elif parsestate == _PARSESTATE_PARENMATCH:
             assert token == stacktop.closebrace
             stacktop.expansion.appendstr(token)
             stacktop = stacktop.parent
         elif parsestate == _PARSESTATE_TOPLEVEL:
             assert stacktop.parent is None
             return stacktop.expansion.finish(), token, offset
         elif parsestate == _PARSESTATE_FUNCTION:
             if token == ',':
                 stacktop.function.append(stacktop.expansion.finish())
 
                 stacktop.expansion = data.Expansion()
                 if len(stacktop.function) + 1 == stacktop.function.maxargs:
-                    tokenlist = TokenList.get((stacktop.openbrace, stacktop.closebrace, '$'))
+                    tokenlist = (stacktop.openbrace, stacktop.closebrace, '$')
                     stacktop.tokenlist = tokenlist
             elif token in (')', '}'):
                 fn = stacktop.function
                 fn.append(stacktop.expansion.finish())
                 fn.setup()
                 
                 stacktop = stacktop.parent
                 stacktop.expansion.appendfunc(fn)
             else:
                 assert False, "Not reached, _PARSESTATE_FUNCTION"
         elif parsestate == _PARSESTATE_VARNAME:
             if token == ':':
                 stacktop.varname = stacktop.expansion
                 stacktop.parsestate = _PARSESTATE_SUBSTFROM
                 stacktop.expansion = data.Expansion()
-                stacktop.tokenlist = TokenList.get(('=', stacktop.openbrace, stacktop.closebrace, '$'))
+                stacktop.tokenlist = ('=', stacktop.openbrace, stacktop.closebrace, '$')
             elif token in (')', '}'):
                 fn = functions.VariableRef(stacktop.loc, stacktop.expansion.finish())
                 stacktop = stacktop.parent
                 stacktop.expansion.appendfunc(fn)
             else:
                 assert False, "Not reached, _PARSESTATE_VARNAME"
         elif parsestate == _PARSESTATE_SUBSTFROM:
             if token == '=':
                 stacktop.substfrom = stacktop.expansion
                 stacktop.parsestate = _PARSESTATE_SUBSTTO
                 stacktop.expansion = data.Expansion()
-                stacktop.tokenlist = TokenList.get((stacktop.openbrace, stacktop.closebrace, '$'))
+                stacktop.tokenlist = (stacktop.openbrace, stacktop.closebrace, '$')
             elif token in (')', '}'):
                 # A substitution of the form $(VARNAME:.ee) is probably a mistake, but make
                 # parses it. Issue a warning. Combine the varname and substfrom expansions to
                 # make the compatible varname. See tests/var-substitutions.mk SIMPLE3SUBSTNAME
                 _log.warning("%s: Variable reference looks like substitution without =", stacktop.loc)
                 stacktop.varname.appendstr(':')
                 stacktop.varname.concat(stacktop.expansion)
                 fn = functions.VariableRef(stacktop.loc, stacktop.varname.finish())
@@ -861,19 +751,19 @@ def parsemakesyntax(d, startat, stopon, 
             fn = functions.SubstitutionRef(stacktop.loc, stacktop.varname.finish(),
                                            stacktop.substfrom.finish(), stacktop.expansion.finish())
             stacktop = stacktop.parent
             stacktop.expansion.appendfunc(fn)
         else:
             assert False, "Unexpected parse state %s" % stacktop.parsestate
 
         if stacktop.parent is not None and iterfunc == itercommandchars:
-            di = itermakefilechars(d, offset, stacktop.tokenlist,
+            di = itermakefilechars(d, offset, stacktop.tokenlist, tokeniterator,
                                    ignorecomments=True)
         else:
-            di = iterfunc(d, offset, stacktop.tokenlist)
+            di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator)
 
     if stacktop.parent is not None:
         raise SyntaxError("Unterminated function call", d.getloc(offset))
 
     assert stacktop.parsestate == _PARSESTATE_TOPLEVEL
 
     return stacktop.expansion.finish(), None, None
--- a/build/pymake/pymake/parserdata.py
+++ b/build/pymake/pymake/parserdata.py
@@ -1,10 +1,10 @@
 import logging, re, os
-import data, functions, util, parser
+import data, parser, functions, util
 from cStringIO import StringIO
 from pymake.globrelative import hasglob, glob
 
 _log = logging.getLogger('pymake.data')
 _tabwidth = 4
 
 class Location(object):
     """
@@ -16,46 +16,45 @@ class Location(object):
     """
     __slots__ = ('path', 'line', 'column')
 
     def __init__(self, path, line, column):
         self.path = path
         self.line = line
         self.column = column
 
-    def __add__(self, data):
+    def offset(self, s, start, end):
         """
         Returns a new location offset by
         the specified string.
         """
 
-        if data == '':
+        if start == end:
             return self
         
-        skiplines = data.count('\n')
+        skiplines = s.count('\n', start, end)
         line = self.line + skiplines
         if skiplines:
-            lastnl = data.rfind('\n')
+            lastnl = s.rfind('\n', start, end)
             assert lastnl != -1
-            data = data[lastnl + 1:]
+            start = lastnl + 1
             column = 0
         else:
             column = self.column
 
-        i = 0
         while True:
-            j = data.find('\t', i)
+            j = s.find('\t', start, end)
             if j == -1:
-                column += len(data) - i
+                column += end - start
                 break
 
-            column += j - i
+            column += j - start
             column += _tabwidth
             column -= column % _tabwidth
-            i = j + 1
+            start = j + 1
 
         return Location(self.path, line, column)
 
     def __str__(self):
         return "%s:%s:%s" % (self.path, self.line, self.column)
 
 def _expandwildcards(makefile, tlist):
     for t in tlist:
@@ -86,17 +85,17 @@ def parsecommandlineargs(args):
         if t != '':
             overrides.append(_flagescape.sub(r'\\\1', a))
 
             vname = vname.strip()
             vnameexp = data.Expansion.fromstring(vname, "Command-line argument")
 
             stmts.append(SetVariable(vnameexp, token=t,
                                      value=val, valueloc=Location('<command-line>', i, len(vname) + len(t)),
-                                     targetexp=None, source=data.Variables.SOURCE_COMMANDLINE, endloc=None))
+                                     targetexp=None, source=data.Variables.SOURCE_COMMANDLINE))
         else:
             r.append(a)
 
     return stmts, r, ' '.join(overrides)
 
 class Statement(object):
     """
     A statement is an abstract object representing a single "chunk" of makefile syntax. Subclasses
@@ -130,24 +129,28 @@ class Rule(Statement):
             context.currule = DummyRule()
             return
 
         ispatterns = set((t.ispattern() for t in targets))
         if len(ispatterns) == 2:
             raise data.DataError("Mixed implicit and normal rule", self.targetexp.loc)
         ispattern, = ispatterns
 
+        if ispattern and context.weak:
+            raise data.DataError("Pattern rules not allowed in includedeps", self.targetexp.loc)
+
         deps = [p for p in _expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables)))]
         if ispattern:
             rule = data.PatternRule(targets, map(data.Pattern, deps), self.doublecolon, loc=self.targetexp.loc)
             makefile.appendimplicitrule(rule)
         else:
-            rule = data.Rule(deps, self.doublecolon, loc=self.targetexp.loc)
+            rule = data.Rule(deps, self.doublecolon, loc=self.targetexp.loc, weakdeps=context.weak)
             for t in targets:
                 makefile.gettarget(t.gettarget()).addrule(rule)
+
             makefile.foundtarget(targets[0].gettarget())
 
         context.currule = rule
 
     def dump(self, fd, indent):
         print >>fd, "%sRule %s: %s" % (indent, self.targetexp, self.depexp)
 
 class StaticPatternRule(Statement):
@@ -159,16 +162,19 @@ class StaticPatternRule(Statement):
         assert isinstance(depexp, (data.Expansion, data.StringExpansion))
 
         self.targetexp = targetexp
         self.patternexp = patternexp
         self.depexp = depexp
         self.doublecolon = doublecolon
 
     def execute(self, makefile, context):
+        if context.weak:
+            raise data.DataError("Static pattern rules not allowed in includedeps", self.targetexp.loc)
+
         targets = list(_expandwildcards(makefile, data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))))
 
         if not len(targets):
             context.currule = DummyRule()
             return
 
         patterns = list(data.stripdotslashes(self.patternexp.resolvesplit(makefile, makefile.variables)))
         if len(patterns) != 1:
@@ -197,37 +203,41 @@ class Command(Statement):
     __slots__ = ('exp',)
 
     def __init__(self, exp):
         assert isinstance(exp, (data.Expansion, data.StringExpansion))
         self.exp = exp
 
     def execute(self, makefile, context):
         assert context.currule is not None
+        if context.weak:
+            raise data.DataError("rules not allowed in includedeps", self.exp.loc)
+
         context.currule.addcommand(self.exp)
 
     def dump(self, fd, indent):
         print >>fd, "%sCommand %s" % (indent, self.exp,)
 
 class SetVariable(Statement):
     __slots__ = ('vnameexp', 'token', 'value', 'valueloc', 'targetexp', 'source')
 
-    def __init__(self, vnameexp, token, value, valueloc, targetexp, source,
-                 endloc):
+    def __init__(self, vnameexp, token, value, valueloc, targetexp, source=None):
         assert isinstance(vnameexp, (data.Expansion, data.StringExpansion))
         assert isinstance(value, str)
         assert targetexp is None or isinstance(targetexp, (data.Expansion, data.StringExpansion))
 
+        if source is None:
+            source = data.Variables.SOURCE_MAKEFILE
+
         self.vnameexp = vnameexp
         self.token = token
         self.value = value
         self.valueloc = valueloc
         self.targetexp = targetexp
         self.source = source
-        self.endloc = endloc
 
     def execute(self, makefile, context):
         vname = self.vnameexp.resolvestr(makefile, makefile.variables)
         if len(vname) == 0:
             raise data.DataError("Empty variable name", self.vnameexp.loc)
 
         if self.targetexp is None:
             setvariables = [makefile.variables]
@@ -261,17 +271,17 @@ class SetVariable(Statement):
                 flavor = data.Variables.FLAVOR_SIMPLE
                 d = parser.Data.fromstring(self.value, self.valueloc)
                 e, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
                 value = e.resolvestr(makefile, makefile.variables)
 
             v.set(vname, flavor, self.source, value)
 
     def dump(self, fd, indent):
-        print >>fd, "%sSetVariable<%s-%s> %s %s\n%s %r" % (indent, self.valueloc, self.endloc, self.vnameexp, self.token, indent, self.value)
+        print >>fd, "%sSetVariable<%s> %s %s\n%s %r" % (indent, self.valueloc, self.vnameexp, self.token, indent, self.value)
 
 class Condition(object):
     """
     An abstract "condition", either ifeq or ifdef, perhaps negated. Subclasses must implement:
 
     def evaluate(self, makefile)
     """
 
@@ -374,27 +384,28 @@ class ConditionBlock(Statement):
 
     def __len__(self):
         return len(self._groups)
 
     def __getitem__(self, i):
         return self._groups[i]
 
 class Include(Statement):
-    __slots__ = ('exp', 'required')
+    __slots__ = ('exp', 'required', 'deps')
 
-    def __init__(self, exp, required):
+    def __init__(self, exp, required, weak):
         assert isinstance(exp, (data.Expansion, data.StringExpansion))
         self.exp = exp
         self.required = required
+        self.weak = weak
 
     def execute(self, makefile, context):
         files = self.exp.resolvesplit(makefile, makefile.variables)
         for f in files:
-            makefile.include(f, self.required, loc=self.exp.loc)
+            makefile.include(f, self.required, loc=self.exp.loc, weak=self.weak)
 
     def dump(self, fd, indent):
         print >>fd, "%sInclude %s" % (indent, self.exp)
 
 class VPathDirective(Statement):
     __slots__ = ('exp',)
 
     def __init__(self, exp):
@@ -466,28 +477,31 @@ class EmptyDirective(Statement):
         v = self.exp.resolvestr(makefile, makefile.variables)
         if v.strip() != '':
             raise data.DataError("Line expands to non-empty value", self.exp.loc)
 
     def dump(self, fd, indent):
         print >>fd, "%sEmptyDirective: %s" % (indent, self.exp)
 
 class _EvalContext(object):
-    __slots__ = ('currule',)
+    __slots__ = ('currule', 'weak')
+
+    def __init__(self, weak):
+        self.weak = weak
 
 class StatementList(list):
     __slots__ = ('mtime',)
 
     def append(self, statement):
         assert isinstance(statement, Statement)
         list.append(self, statement)
 
-    def execute(self, makefile, context=None):
+    def execute(self, makefile, context=None, weak=False):
         if context is None:
-            context = _EvalContext()
+            context = _EvalContext(weak=weak)
 
         for s in self:
             s.execute(makefile, context)
 
     def dump(self, fd, indent):
         for s in self:
             s.dump(fd, indent)
 
@@ -496,9 +510,9 @@ class StatementList(list):
         self.dump(fd, '')
         return fd.getvalue()
 
 def iterstatements(stmts):
     for s in stmts:
         yield s
         if isinstance(s, ConditionBlock):
             for c, sl in s:
-                for s2 in iterstatements(sl): yield s2
+                for s2 in iterstatments(sl): yield s2
--- a/build/pymake/tests/comment-parsing.mk
+++ b/build/pymake/tests/comment-parsing.mk
@@ -1,17 +1,29 @@
 # where do comments take effect?
 
 VAR = val1 # comment
 VAR2 = lit2\#hash
+VAR2_1 = lit2.1\\\#hash
 VAR3 = val3
 VAR4 = lit4\\#backslash
+VAR4_1 = lit4\\\\#backslash
 VAR5 = lit5\char
+VAR6 = lit6\\char
+VAR7 = lit7\\
+VAR8 = lit8\\\\
+VAR9 = lit9\\\\extra
 # This comment extends to the next line \
 VAR3 = ignored
 
 all:
 	test "$(VAR)" = "val1 "
 	test "$(VAR2)" = "lit2#hash"
+	test '$(VAR2_1)' = 'lit2.1\#hash'
 	test "$(VAR3)" = "val3"
 	test '$(VAR4)' = 'lit4\'
+	test '$(VAR4_1)' = 'lit4\\'
 	test '$(VAR5)' = 'lit5\char'
+	test '$(VAR6)' = 'lit6\\char'
+	test '$(VAR7)' = 'lit7\\'
+	test '$(VAR8)' = 'lit8\\\\'
+	test '$(VAR9)' = 'lit9\\\\extra'
 	@echo "TEST-PASS"
--- a/build/pymake/tests/continuations-in-functions.mk
+++ b/build/pymake/tests/continuations-in-functions.mk
@@ -1,7 +1,6 @@
 all:
 	test 'Hello world.' = '$(if 1,Hello \
 	  world.)'
-	test '(Hello \
-	  world.)' = '(Hello \
+	test '(Hello world.)' != '(Hello \
 	  world.)'
 	@echo TEST-PASS
--- a/build/pymake/tests/func-refs.mk
+++ b/build/pymake/tests/func-refs.mk
@@ -1,10 +1,11 @@
 unknown var = uval
 
 all:
 	test "$(subst a,b,value)" = "vblue"
 	test "${subst a,b,va)lue}" = "vb)lue"
 	test "$(subst /,\,ab/c)" = "ab\c"
+	test '$(subst a,b,\\#)' = '\\#'
 	test "$( subst a,b,value)" = ""
 	test "$(Subst a,b,value)" = ""
 	test "$(unknown var)" = "uval"
 	@echo TEST-PASS
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/include-missing.mk
@@ -0,0 +1,9 @@
+#T returncode: 2
+
+# If an include file isn't present and doesn't have a rule to remake it, make
+# should fail.
+
+include notfound.mk
+
+all:
+	@echo TEST-FAIL
--- a/build/pymake/tests/include-regen.mk
+++ b/build/pymake/tests/include-regen.mk
@@ -1,9 +1,10 @@
-# make should only consider -included makefiles for remaking if they actually exist:
-
+# avoid infinite loops by not remaking makefiles with
+# double-colon no-dependency rules
+# http://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
 -include notfound.mk
 
 all:
 	@echo TEST-PASS
 
 notfound.mk::
 	@echo TEST-FAIL
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/include-regen2.mk
@@ -0,0 +1,10 @@
+# make should make makefiles that it has rules for if they are
+# included
+include test.mk
+
+all:
+	test "$(X)" = "1"
+	@echo "TEST-PASS"
+
+test.mk:
+	@echo "X = 1" > $@
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/include-regen3.mk
@@ -0,0 +1,10 @@
+# make should make makefiles that it has rules for if they are
+# included
+-include test.mk
+
+all:
+	test "$(X)" = "1"
+	@echo "TEST-PASS"
+
+test.mk:
+	@echo "X = 1" > $@
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/includedeps-norebuild.mk
@@ -0,0 +1,15 @@
+#T gmake skip
+
+$(shell \
+touch filemissing; \
+sleep 2; \
+touch file1; \
+)
+
+all: file1
+	@echo TEST-PASS
+
+includedeps $(TESTPATH)/includedeps.deps
+
+file1:
+	@echo TEST-FAIL
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/includedeps-sideeffects.mk
@@ -0,0 +1,10 @@
+#T gmake skip
+#T returncode: 2
+
+all: file1 filemissing
+	@echo TEST-PASS
+
+includedeps $(TESTPATH)/includedeps.deps
+
+file:
+	touch $@
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/includedeps.deps
@@ -0,0 +1,1 @@
+file1: filemissing
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/includedeps.mk
@@ -0,0 +1,9 @@
+#T gmake skip
+
+all: file1
+	@echo TEST-PASS
+
+includedeps $(TESTPATH)/includedeps.deps
+
+file1:
+	touch $@
--- a/build/pymake/tests/line-continuations.mk
+++ b/build/pymake/tests/line-continuations.mk
@@ -1,17 +1,21 @@
 VAR = val1 	 \
   	  val2  
 
 VAR2 = val1space\
 val2
 
+VAR3 = val3 \\\
+	cont3
+
 all: otarget test.target
 	test "$(VAR)" = "val1 val2  "
 	test "$(VAR2)" = "val1space val2"
+	test '$(VAR3)' = 'val3 \ cont3'
 	test "hello \
 	  world" = "hello   world"
 	test "hello" = \
 "hello"
 	@echo TEST-PASS
 
 otarget: ; test "hello\
 	world" = "helloworld"
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/oneline-command-continuations.mk
@@ -0,0 +1,5 @@
+all: test
+	@echo TEST-PASS
+
+test: ; test "Hello \
+	  world" = "Hello   world"
--- a/build/pymake/tests/override-propagate.mk
+++ b/build/pymake/tests/override-propagate.mk
@@ -1,17 +1,22 @@
 #T commandline: ['-w', 'OVAR=oval']
 
 OVAR=mval
 
 all: vartest run-override
 	$(MAKE) -f $(TESTPATH)/override-propagate.mk vartest
 	@echo TEST-PASS
 
-SORTED_CLINE := $(sort OVAR=oval TESTPATH=$(TESTPATH) NATIVE_TESTPATH=$(NATIVE_TESTPATH))
+CLINE := OVAR=oval TESTPATH=$(TESTPATH) NATIVE_TESTPATH=$(NATIVE_TESTPATH)
+ifdef __WIN32__
+CLINE += __WIN32__=1
+endif
+
+SORTED_CLINE := $(subst \,\\,$(sort $(CLINE)))
 
 vartest:
 	@echo MAKELEVEL: '$(MAKELEVEL)'
 	test '$(value MAKEFLAGS)' = 'w -- $$(MAKEOVERRIDES)'
 	test '$(origin MAKEFLAGS)' = 'file'
 	test '$(value MAKEOVERRIDES)' = '$${-*-command-variables-*-}'
 	test "$(sort $(MAKEOVERRIDES))" = "$(SORTED_CLINE)"
 	test '$(origin MAKEOVERRIDES)' = 'environment'
--- a/build/pymake/tests/parsertests.py
+++ b/build/pymake/tests/parsertests.py
@@ -23,61 +23,80 @@ class DataTest(TestBase):
             ("He\tllo", "f", 1, 0,
              ((0, "f", 1, 0), (2, "f", 1, 2), (3, "f", 1, 4))),
         'twoline':
             ("line1 \n\tl\tine2", "f", 1, 4,
              ((0, "f", 1, 4), (5, "f", 1, 9), (6, "f", 1, 10), (7, "f", 2, 0), (8, "f", 2, 4), (10, "f", 2, 8), (13, "f", 2, 11))),
     }
 
     def runSingle(self, data, filename, line, col, results):
-        d = pymake.parser.Data(pymake.parserdata.Location(filename, line, col),
-                               data)
+        d = pymake.parser.Data(data, 0, len(data), pymake.parserdata.Location(filename, line, col))
         for pos, file, lineno, col in results:
             loc = d.getloc(pos)
             self.assertEqual(loc.path, file, "data file offset %i" % pos)
             self.assertEqual(loc.line, lineno, "data line offset %i" % pos)
             self.assertEqual(loc.column, col, "data col offset %i" % pos)
 multitest(DataTest)
 
-class TokenTest(TestBase):
+class LineEnumeratorTest(TestBase):
     testdata = {
-        'wsmatch': ('  ifdef FOO', 2, ('ifdef', 'else'), True, 'ifdef', 8),
-        'wsnomatch': ('  unexpected FOO', 2, ('ifdef', 'else'), True, None, 2),
-        'wsnows': ('  ifdefFOO', 2, ('ifdef', 'else'), True, None, 2),
-        'paren': (' "hello"', 1, ('(', "'", '"'), False, '"', 2),
+        'simple': (
+            'Hello, world', [
+                ('Hello, world', 1),
+                ]
+            ),
+        'multi': (
+            'Hello\nhappy  \n\nworld\n', [
+                ('Hello', 1),
+                ('happy  ', 2),
+                ('', 3),
+                ('world', 4),
+                ('', 5),
+                ]
+            ),
+        'continuation': (
+            'Hello, \\\n world\nJellybeans!', [
+                ('Hello, \\\n world', 1),
+                ('Jellybeans!', 3),
+                ]
+            ),
+        'multislash': (
+            'Hello, \\\\\n world', [
+                ('Hello, \\\\', 1),
+                (' world', 2),
+                ]
+            )
         }
 
-    def runSingle(self, s, start, tlist, needws, etoken, eoffset):
-        d = pymake.parser.Data.fromstring(s, None)
-        tl = pymake.parser.TokenList.get(tlist)
-        atoken, aoffset = d.findtoken(start, tl, needws)
-        self.assertEqual(atoken, etoken)
-        self.assertEqual(aoffset, eoffset)
-multitest(TokenTest)
+    def runSingle(self, s, lines):
+        gotlines = [(d.s[d.lstart:d.lend], d.loc.line) for d in pymake.parser.enumeratelines(s, 'path')]
+        self.assertEqual(gotlines, lines)
+
+multitest(LineEnumeratorTest)
 
 class IterTest(TestBase):
     testdata = {
         'plaindata': (
             pymake.parser.iterdata,
             "plaindata # test\n",
             "plaindata # test\n"
             ),
         'makecomment': (
             pymake.parser.itermakefilechars,
             "VAR = val # comment",
             "VAR = val "
             ),
         'makeescapedcomment': (
             pymake.parser.itermakefilechars,
-            "VAR = val \# escaped hash\n",
+            "VAR = val \# escaped hash",
             "VAR = val # escaped hash"
             ),
         'makeescapedslash': (
             pymake.parser.itermakefilechars,
-            "VAR = val\\\\\n",
+            "VAR = val\\\\",
             "VAR = val\\\\",
             ),
         'makecontinuation': (
             pymake.parser.itermakefilechars,
             "VAR = VAL  \\\n  continuation # comment \\\n  continuation",
             "VAR = VAL continuation "
             ),
         'makecontinuation2': (
@@ -87,65 +106,68 @@ class IterTest(TestBase):
             ),
         'makeawful': (
             pymake.parser.itermakefilechars,
             "VAR = VAL  \\\\# comment\n",
             "VAR = VAL  \\"
             ),
         'command': (
             pymake.parser.itercommandchars,
-            "echo boo # comment\n",
+            "echo boo # comment",
             "echo boo # comment",
             ),
         'commandcomment': (
             pymake.parser.itercommandchars,
-            "echo boo \# comment\n",
+            "echo boo \# comment",
             "echo boo \# comment",
             ),
         'commandcontinue': (
             pymake.parser.itercommandchars,
-            "echo boo # \\\n\t  command 2\n",
+            "echo boo # \\\n\t  command 2",
             "echo boo # \\\n  command 2"
             ),
-        'define': (
-            pymake.parser.iterdefinechars,
-            "endef",
-            ""
-            ),
-        'definenesting': (
-            pymake.parser.iterdefinechars,
-            """define BAR # comment
-random text
-endef not what you think!
-endef # comment is ok\n""",
-            """define BAR # comment
-random text
-endef not what you think!"""
-            ),
-        'defineescaped': (
-            pymake.parser.iterdefinechars,
-            """value   \\
-endef
-endef\n""",
-            "value endef"
-        ),
     }
 
     def runSingle(self, ifunc, idata, expected):
-        fd = StringIO(idata)
-        lineiter = enumerate(fd)
+        d = pymake.parser.Data.fromstring(idata, 'IterTest data')
 
-        d = pymake.parser.DynamicData(lineiter, 'PlainIterTest-data')
-
-        actual = ''.join( (c for c, t, o, oo in ifunc(d, 0, pymake.parser._emptytokenlist)) )
+        it = pymake.parser._alltokens.finditer(d.s, 0, d.lend)
+        actual = ''.join( [c for c, t, o, oo in ifunc(d, 0, ('dummy-token',), it)] )
         self.assertEqual(actual, expected)
 
-        self.assertRaises(StopIteration, lambda: fd.next())
+        if ifunc == pymake.parser.itermakefilechars:
+            print "testing %r" % expected
+            self.assertEqual(pymake.parser.flattenmakesyntax(d, 0), expected)
+
 multitest(IterTest)
 
+
+#         'define': (
+#             pymake.parser.iterdefinechars,
+#             "endef",
+#             ""
+#             ),
+#        'definenesting': (
+#            pymake.parser.iterdefinechars,
+#            """define BAR # comment
+#random text
+#endef not what you think!
+#endef # comment is ok\n""",
+#            """define BAR # comment
+#random text
+#endef not what you think!"""
+#            ),
+#        'defineescaped': (
+#            pymake.parser.iterdefinechars,
+#            """value   \\
+#endef
+#endef\n""",
+#            "value endef"
+#        ),
+
 class MakeSyntaxTest(TestBase):
     # (string, startat, stopat, stopoffset, expansion
     testdata = {
         'text': ('hello world', 0, (), None, ['hello world']),
         'singlechar': ('hello $W', 0, (), None,
                        ['hello ',
                         {'type': 'VariableRef',
                          '.vname': ['W']}
@@ -179,17 +201,17 @@ class MakeSyntaxTest(TestBase):
         'vadsubstref': ('  $(VAR:VAL) = $(VAL)', 15, (), None,
                         [{'type': 'VariableRef',
                           '.vname': ['VAL']},
                          ]),
         }
 
     def compareRecursive(self, actual, expected, path):
         self.assertEqual(len(actual), len(expected),
-                         "compareRecursive: %s" % (path,))
+                         "compareRecursive: %s %r" % (path, actual))
         for i in xrange(0, len(actual)):
             ipath = path + [i]
 
             a, isfunc = actual[i]
             e = expected[i]
             if isinstance(e, str):
                 self.assertEqual(a, e, "compareRecursive: %s" % (ipath,))
             else:
@@ -233,18 +255,17 @@ class VariableTest(TestBase):
                 'VARNAME': 'newname',
                 'TESTVAR': 'testvalue',
                 'TESTVAL': 'moretesting',
                 'IMM': 'TESTVAR ',
                 'MULTIVAR': 'val1 val2',
                 'UNDEF': None}
 
     def runTest(self):
-        stream = StringIO(self.testdata)
-        stmts = pymake.parser.parsestream(stream, 'testdata')
+        stmts = pymake.parser.parsestring(self.testdata, 'VariableTest')
 
         m = pymake.data.Makefile()
         stmts.execute(m)
         for k, v in self.expected.iteritems():
             flavor, source, val = m.variables.get(k)
             if val is None:
                 self.assertEqual(val, v, 'variable named %s' % k)
             else:
@@ -258,18 +279,17 @@ all: TSPEC = myrule
 all:: test test2 $(VAR)
 	echo "Hello, $(TSPEC)"
 
 %.o: %.c
 	$(CC) -o $@ $<
 """
 
     def runTest(self):
-        stream = StringIO(self.testdata)
-        stmts = pymake.parser.parsestream(stream, 'testdata')
+        stmts = pymake.parser.parsestring(self.testdata, 'SimpleRuleTest')
 
         m = pymake.data.Makefile()
         stmts.execute(m)
         self.assertEqual(m.defaulttarget, 'all', "Default target")
 
         self.assertTrue(m.hastarget('all'), "Has 'all' target")
         target = m.gettarget('all')
         rules = target.rules
--- a/build/pymake/tests/runtests.py
+++ b/build/pymake/tests/runtests.py
@@ -12,109 +12,199 @@ The test file may contain lines at the b
 #T returncode: 2
 #T returncode-on: {'win32': 2}
 #T environment: {'VAR': 'VALUE}
 #T grep-for: "text"
 """
 
 from subprocess import Popen, PIPE, STDOUT
 from optparse import OptionParser
-import os, re, sys, shutil
+import os, re, sys, shutil, glob
+
+class ParentDict(dict):
+    def __init__(self, parent, **kwargs):
+        self.d = dict(kwargs)
+        self.parent = parent
+
+    def __setitem__(self, k, v):
+        self.d[k] = v
+
+    def __getitem__(self, k):
+        if k in self.d:
+            return self.d[k]
+
+        return self.parent[k]
 
 thisdir = os.path.dirname(os.path.abspath(__file__))
 
+pymake = [sys.executable, os.path.join(os.path.dirname(thisdir), 'make.py')]
+manifest = os.path.join(thisdir, 'tests.manifest')
+
 o = OptionParser()
-o.add_option('-m', '--make',
-             dest="make", default="gmake")
+o.add_option('-g', '--gmake',
+             dest="gmake", default="gmake")
 o.add_option('-d', '--tempdir',
              dest="tempdir", default="_mktests")
 opts, args = o.parse_args()
 
 if len(args) == 0:
-    args = ['.']
+    args = [thisdir]
 
 makefiles = []
 for a in args:
     if os.path.isfile(a):
         makefiles.append(a)
     elif os.path.isdir(a):
-        for path, dirnames, filenames in os.walk(a):
-            for f in filenames:
-                if f.endswith('.mk'):
-                    makefiles.append('%s/%s' % (path, f))
-    else:
-        print >>sys.stderr, "Error: Unknown file on command line"
-        sys.exit(1)
+        makefiles.extend(glob.glob(os.path.join(a, '*.mk')))
 
-tre = re.compile('^#T ([a-z-]+): (.*)$')
-
-for makefile in makefiles:
-    print "Testing: %s" % makefile,
+def runTest(makefile, make, logfile, options):
+    """
+    Given a makefile path, test it with a given `make` and return
+    (pass, message).
+    """
 
     if os.path.exists(opts.tempdir): shutil.rmtree(opts.tempdir)
     os.mkdir(opts.tempdir, 0755)
 
+    logfd = open(logfile, 'w')
+    p = Popen(make + options['commandline'], stdout=logfd, stderr=STDOUT, env=options['env'])
+    logfd.close()
+    retcode = p.wait()
+
+    if retcode != options['returncode']:
+        return False, "FAIL (returncode=%i)" % retcode
+        
+    logfd = open(logfile)
+    stdout = logfd.read()
+    logfd.close()
+
+    if stdout.find('TEST-FAIL') != -1:
+        return False, "FAIL (TEST-FAIL printed)"
+
+    if options['grepfor'] and stdout.find(options['grepfor']) == -1:
+            return False, "FAIL (%s not in output)" % options['grepfor']
+
+    if options['returncode'] == 0 and stdout.find('TEST-PASS') == -1:
+        return False, 'FAIL (No TEST-PASS printed)'
+
+    if options['returncode'] != 0:
+        return True, 'PASS (retcode=%s)' % retcode
+
+    return True, 'PASS'
+
+print "%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:")
+
+gmakefails = 0
+pymakefails = 0
+
+tre = re.compile('^#T (gmake |pymake )?([a-z-]+)(?:: (.*))?$')
+
+for makefile in makefiles:
     # For some reason, MAKEFILE_LIST uses native paths in GNU make on Windows
     # (even in MSYS!) so we pass both TESTPATH and NATIVE_TESTPATH
-    cline = [opts.make, '-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir.replace('\\','/'), 'NATIVE_TESTPATH=%s' % thisdir]
+    cline = ['-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir.replace('\\','/'), 'NATIVE_TESTPATH=%s' % thisdir]
     if sys.platform == 'win32':
-        #XXX: hack to run pymake on windows
-        if opts.make.endswith('.py'):
-            cline = [sys.executable] + cline
         #XXX: hack so we can specialize the separator character on windows.
         # we really shouldn't need this, but y'know
         cline += ['__WIN32__=1']
-        
-    returncode = 0
-    grepfor = None
 
-    env = dict(os.environ)
+    options = {
+        'returncode': 0,
+        'grepfor': None,
+        'env': dict(os.environ),
+        'commandline': cline,
+        'pass': True,
+        'skip': False,
+        }
+
+    gmakeoptions = ParentDict(options)
+    pymakeoptions = ParentDict(options)
+
+    dmap = {None: options, 'gmake ': gmakeoptions, 'pymake ': pymakeoptions}
 
     mdata = open(makefile)
     for line in mdata:
         m = tre.search(line)
         if m is None:
             break
-        key, data = m.group(1, 2)
-        data = eval(data)
+
+        make, key, data = m.group(1, 2, 3)
+        d = dmap[make]
+        if data is not None:
+            data = eval(data)
         if key == 'commandline':
-            cline.extend(data)
+            assert make is None
+            d['commandline'].extend(data)
         elif key == 'returncode':
-            returncode = data
+            d['returncode'] = data
         elif key == 'returncode-on':
             if sys.platform in data:
-                returncode = data[sys.platform]
+                d['returncode'] = data[sys.platform]
         elif key == 'environment':
             for k, v in data.iteritems():
-                env[k] = v
+                d['env'][k] = v
         elif key == 'grep-for':
             grepfor = data
+        elif key == 'fail':
+            d['pass'] = False
+        elif key == 'skip':
+            d['skip'] = True
         else:
-            print >>sys.stderr, "Unexpected #T key: %s" % key
+            print >>sys.stderr, "%s: Unexpected #T key: %s" % (makefile, key)
             sys.exit(1)
 
     mdata.close()
 
-    p = Popen(cline, stdout=PIPE, stderr=STDOUT, env=env)
-    stdout, d = p.communicate()
-    if p.returncode != returncode:
-        print "FAIL (returncode=%i)" % p.returncode
-        print stdout
-    elif stdout.find('TEST-FAIL') != -1:
-        print "FAIL"
-        print stdout
-    elif returncode == 0:
-        if stdout.find(grepfor or 'TEST-PASS') != -1:
-            print "PASS"
+    if gmakeoptions['skip']:
+        gmakepass, gmakemsg = True, ''
+    else:
+        gmakepass, gmakemsg = runTest(makefile, [opts.gmake],
+                                      makefile + '.gmakelog', gmakeoptions)
+
+    if gmakeoptions['pass']:
+        if not gmakepass:
+            gmakefails += 1
+    else:
+        if gmakepass:
+            gmakefails += 1
+            gmakemsg = "UNEXPECTED PASS"
         else:
-            print "FAIL (no expected output)"
-            print stdout
-    # check that test produced the expected output while failing
-    elif grepfor:
-        if stdout.find(grepfor) != -1:
-            print "PASS"
+            gmakemsg = "KNOWN FAIL"
+
+    if pymakeoptions['skip']:
+        pymakepass, pymakemsg = True, ''
+    else:
+        pymakepass, pymakemsg = runTest(makefile, pymake,
+                                        makefile + '.pymakelog', pymakeoptions)
+
+    if pymakeoptions['pass']:
+        if not pymakepass:
+            pymakefails += 1
+    else:
+        if pymakepass:
+            pymakefails += 1
+            pymakemsg = "UNEXPECTED PASS"
         else:
-            print "FAIL (no expected output)"
-            print stdout
-    else:
-        print "EXPECTED-FAIL"
+            pymakemsg = "OK (known fail)"
+
+    print "%-30.30s%-28.28s%-28.28s" % (os.path.basename(makefile),
+                                        gmakemsg, pymakemsg)
+
+print
+print "Summary:"
+print "%-30s%-28s%-28s" % ("", "gmake:", "pymake:")
+
+if gmakefails == 0:
+    gmakemsg = 'PASS'
+else:
+    gmakemsg = 'FAIL (%i failures)' % gmakefails
+
+if pymakefails == 0:
+    pymakemsg = 'PASS'
+else:
+    pymakemsg = 'FAIL (%i failures)' % pymakefails
+
+print "%-30.30s%-28.28s%-28.28s" % ('', gmakemsg, pymakemsg)
 
 shutil.rmtree(opts.tempdir)
+
+if gmakefails or pymakefails:
+    sys.exit(1)
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/windows-paths.mk
@@ -0,0 +1,5 @@
+all:
+	touch file.in
+	printf "%s: %s\n\ttrue" '$(CURDIR)/file.out' '$(CURDIR)/file.in' >test.mk
+	$(MAKE) -f test.mk $(CURDIR)/file.out
+	@echo TEST-PASS