Update pymake with --keep-going and other fixes
authorBenjamin Smedberg <benjamin@smedbergs.us>
Tue, 31 Mar 2009 17:10:23 -0400
changeset 26792 6a6b297018f6e82dd1f20f8c6c76c664c8066cf3
parent 26791 78fb9a173674f46aa70dc03075899845d61a1726
child 26793 b9659463f9382784141cf30c2f19e1a441c50bef
push idunknown
push userunknown
push dateunknown
milestone1.9.2a1pre
Update pymake with --keep-going and other fixes
build/pymake/.hg_archival.txt
build/pymake/pymake/command.py
build/pymake/pymake/data.py
build/pymake/tests/keep-going-doublecolon.mk
build/pymake/tests/keep-going-parallel.mk
build/pymake/tests/keep-going.mk
build/pymake/tests/makeflags.mk
build/pymake/tests/runtests.py
--- a/build/pymake/.hg_archival.txt
+++ b/build/pymake/.hg_archival.txt
@@ -1,2 +1,2 @@
 repo: f5ab154deef2ffa97f1b2139589ae4a1962090a4
-node: c07f9fe9fdaa85888314efa7001cd4fd768904c4
+node: 50d8e87e8af3fb8147c33f169a8d5abe8e06ffa4
--- a/build/pymake/pymake/command.py
+++ b/build/pymake/pymake/command.py
@@ -1,37 +1,38 @@
 """
 Makefile execution.
 
 Multiple `makes` can be run within the same process. Each one has an entirely data.Makefile and .Target
 structure, environment, and working directory. Typically they will all share a parallel execution context,
 except when a submake specifies -j1 when the parent make is building in parallel.
 """
 
-import os, subprocess, sys, logging, time, traceback
+import os, subprocess, sys, logging, time, traceback, re
 from optparse import OptionParser
 import data, parserdata, process, util
 
 # TODO: If this ever goes from relocatable package to system-installed, this may need to be
 # a configured-in path.
 
 makepypath = os.path.normpath(os.path.join(os.path.dirname(__file__), '../make.py'))
 
+_simpleopts = re.compile(r'^[a-zA-Z]+\s')
 def parsemakeflags(env):
     """
     Parse MAKEFLAGS from the environment into a sequence of command-line arguments.
     """
 
     makeflags = env.get('MAKEFLAGS', '')
     makeflags = makeflags.strip()
 
     if makeflags == '':
         return []
 
-    if makeflags[0] not in ('-', ' '):
+    if _simpleopts.match(makeflags):
         makeflags = '-' + makeflags
 
     opts = []
     curopt = ''
 
     i = 0
     while i < len(makeflags):
         c = makeflags[i]
@@ -84,16 +85,19 @@ def main(args, env, cwd, cb):
         op = OptionParser()
         op.add_option('-f', '--file', '--makefile',
                       action='append',
                       dest='makefiles',
                       default=[])
         op.add_option('-d',
                       action="store_true",
                       dest="verbose", default=False)
+        op.add_option('-k', '--keep-going',
+                      action="store_true",
+                      dest="keepgoing", default=False)
         op.add_option('--debug-log',
                       dest="debuglog", default=None)
         op.add_option('-C', '--directory',
                       dest="directory", default=None)
         op.add_option('-v', '--version', action="store_true",
                       dest="printversion", default=False)
         op.add_option('-j', '--jobs', type="int",
                       dest="jobcount", default=1)
@@ -108,32 +112,35 @@ def main(args, env, cwd, cb):
         if options.printversion:
             _version()
             cb(0)
             return
 
         shortflags = []
         longflags = []
 
+        if options.keepgoing:
+            shortflags.append('k');
+
         loglevel = logging.WARNING
         if options.verbose:
             loglevel = logging.DEBUG
             shortflags.append('d')
 
         logkwargs = {}
         if options.debuglog:
             logkwargs['filename'] = options.debuglog
             longflags.append('--debug-log=%s' % options.debuglog)
 
         if options.directory is None:
             workdir = cwd
         else:
             workdir = os.path.join(cwd, options.directory)
 
-        shortflags.append('j%i' % (options.jobcount,))
+        longflags.append('-j%i' % (options.jobcount,))
 
         makeflags = ''.join(shortflags) + ' ' + ' '.join(longflags)
 
         logging.basicConfig(level=loglevel, **logkwargs)
 
         context = process.getcontext(options.jobcount)
 
         if options.printdir:
@@ -145,43 +152,45 @@ def main(args, env, cwd, cb):
                 options.makefiles.append('Makefile')
             else:
                 print "No makefile found"
                 cb(2)
                 return
 
         overrides, targets = parserdata.parsecommandlineargs(arguments)
 
-        def makecb(error, didanything, makefile, realtargets, tstack, i, firsterror):
-            if error is not None:
-                print error
-                if firsterror is None:
-                    firsterror = error
+        def makecb(error, didanything, makefile, realtargets, tstack, i):
+            assert error in (True, False)
+
+            if error:
+                context.defer(cb, 2)
+                return
 
             if i == len(realtargets):
                 if options.printdir:
                     print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
                 sys.stdout.flush()
 
-                context.defer(cb, firsterror and 2 or 0)
+                context.defer(cb, 0)
             else:
                 deferredmake = process.makedeferrable(makecb, makefile=makefile,
-                                                      realtargets=realtargets, tstack=tstack, i=i+1, firsterror=firsterror)
+                                                      realtargets=realtargets, tstack=tstack, i=i+1)
 
-                makefile.gettarget(realtargets[i]).make(makefile, tstack, [], cb=deferredmake)
+                makefile.gettarget(realtargets[i]).make(makefile, tstack, cb=deferredmake)
                                                                                   
 
         def remakecb(remade, restarts, makefile):
             if remade:
                 if restarts > 0:
                     _log.info("make.py[%i]: Restarting makefile parsing", makelevel)
                 makefile = data.Makefile(restarts=restarts, make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')),
                                          makeflags=makeflags, makelevel=makelevel, workdir=workdir,
                                          context=context, env=env,
-                                         targets=targets)
+                                         targets=targets,
+                                         keepgoing=options.keepgoing)
 
                 try:
                     overrides.execute(makefile)
                     for f in options.makefiles:
                         makefile.include(f)
                     makefile.finishparsing()
                     makefile.remakemakefiles(process.makedeferrable(remakecb, restarts=restarts + 1, makefile=makefile))
 
@@ -201,18 +210,18 @@ def main(args, env, cwd, cb):
                 _log.info("Making default target %s", makefile.defaulttarget)
                 realtargets = [makefile.defaulttarget]
                 tstack = ['<default-target>']
             else:
                 realtargets = targets
                 tstack = ['<command-line>']
 
             deferredmake = process.makedeferrable(makecb, makefile=makefile,
-                                                  realtargets=realtargets, tstack=tstack, i=1, firsterror=None)
-            makefile.gettarget(realtargets[0]).make(makefile, tstack, [], cb=deferredmake)
+                                                  realtargets=realtargets, tstack=tstack, i=1)
+            makefile.gettarget(realtargets[0]).make(makefile, tstack, cb=deferredmake)
 
         context.defer(remakecb, True, 0, None)
 
     except (util.MakeError), e:
         print e
         if options.printdir:
             print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
         sys.stdout.flush()
--- a/build/pymake/pymake/data.py
+++ b/build/pymake/pymake/data.py
@@ -1,13 +1,13 @@
 """
 A representation of makefile data structures.
 """
 
-import logging, re, os
+import logging, re, os, sys
 import parserdata, parser, functions, process, util, builtins
 from cStringIO import StringIO
 
 _log = logging.getLogger('pymake.data')
 
 class DataError(util.MakeError):
     pass
 
@@ -307,24 +307,16 @@ class Variables(object):
             return
 
         self._map[name] = flavor, source, value, None
 
     def append(self, name, source, value, variables, makefile):
         assert source in (self.SOURCE_OVERRIDE, self.SOURCE_MAKEFILE, self.SOURCE_AUTOMATIC)
         assert isinstance(value, str)
 
-        def expand():
-            try:
-                d = parser.Data.fromstring(value, parserdata.Location("Expansion of variable '%s'" % (name,), 1, 0))
-                valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
-                return valueexp, None
-            except parser.SyntaxError, e:
-                return None, e
-        
         if name not in self._map:
             self._map[name] = self.FLAVOR_APPEND, source, value, None
             return
 
         prevflavor, prevsource, prevvalue, valueexp = self._map[name]
         if source > prevsource:
             # TODO: log a warning?
             return
@@ -470,141 +462,273 @@ class Pattern(object):
 
     _backre = re.compile(r'[%\\]')
     def __str__(self):
         if not self.ispattern:
             return self._backre.sub(r'\\\1', self.data[0])
 
         return self._backre.sub(r'\\\1', self.data[0]) + '%' + self.data[1]
 
-MAKESTATE_NONE = 0
-MAKESTATE_FINISHED = 1
-MAKESTATE_WORKING = 2
+class RemakeTargetSerially(object):
+    __slots__ = ('target', 'makefile', 'indent', 'rlist')
+
+    def __init__(self, target, makefile, indent, rlist):
+        self.target = target
+        self.makefile = makefile
+        self.indent = indent
+        self.rlist = rlist
+        self.commandscb(False)
+
+    def resolvecb(self, error, didanything):
+        assert error in (True, False)
+
+        if didanything:
+            self.target.didanything = True
+
+        if error:
+            self.target.error = True
+            self.makefile.error = True
+            if not self.makefile.keepgoing:
+                self.target.notifydone(self.makefile)
+                return
+            else:
+                # don't run the commands!
+                del self.rlist[0]
+                self.commandscb(error=False)
+        else:
+            self.rlist.pop(0).runcommands(self.indent, self.commandscb)
+
+    def commandscb(self, error):
+        assert error in (True, False)
+
+        if error:
+            self.target.error = True
+            self.makefile.error = True
+
+        if self.target.error and not self.makefile.keepgoing:
+            self.target.notifydone(self.makefile)
+            return
+
+        if not len(self.rlist):
+            self.target.notifydone(self.makefile)
+        else:
+            self.rlist[0].resolvedeps(True, self.resolvecb)
+
+class RemakeTargetParallel(object):
+    __slots__ = ('target', 'makefile', 'indent', 'rlist', 'rulesremaining', 'currunning')
+
+    def __init__(self, target, makefile, indent, rlist):
+        self.target = target
+        self.makefile = makefile
+        self.indent = indent
+        self.rlist = rlist
+
+        self.rulesremaining = len(rlist)
+        self.currunning = False
+
+        for r in rlist:
+            makefile.context.defer(self.doresolve, r)
+
+    def doresolve(self, r):
+        if self.makefile.error and not self.makefile.keepgoing:
+            r.error = True
+            self.resolvecb(True, False)
+        else:
+            r.resolvedeps(False, self.resolvecb)
+
+    def resolvecb(self, error, didanything):
+        assert error in (True, False)
+
+        if error:
+            self.target.error = True
+
+        if didanything:
+            self.target.didanything = True
+
+        self.rulesremaining -= 1
+
+        # commandscb takes care of the details if we're currently building
+        # something
+        if self.currunning:
+            return
+
+        self.runnext()
+
+    def runnext(self):
+        assert not self.currunning
+
+        if self.makefile.error and not self.makefile.keepgoing:
+            self.rlist = []
+        else:
+            while len(self.rlist) and self.rlist[0].error:
+                del self.rlist[0]
+
+        if not len(self.rlist):
+            if not self.rulesremaining:
+                self.target.notifydone(self.makefile)
+            return
+
+        if self.rlist[0].depsremaining != 0:
+            return
+
+        self.currunning = True
+        self.rlist.pop(0).runcommands(self.indent, self.commandscb)
+
+    def commandscb(self, error):
+        assert error in (True, False)
+        if error:
+            self.target.error = True
+            self.makefile.error = True
+
+        assert self.currunning
+        self.currunning = False
+        self.runnext()
 
 class RemakeRuleContext(object):
-    __slots__ = ('rule', 'deps', 'depsremaining', 'error', 'didanything', 'running')
-
-    def __init__(self, rule, deps):
+    def __init__(self, target, makefile, rule, deps,
+                 targetstack, avoidremakeloop):
+        self.target = target
+        self.makefile = makefile
         self.rule = rule
         self.deps = deps
+        self.targetstack = targetstack
+        self.avoidremakeloop = avoidremakeloop
+
         self.running = False
+        self.error = False
         self.depsremaining = len(deps) + 1
 
-    def resolvedeps(self, target, makefile, targetstack, rulestack, serial, cb):
+    def resolvedeps(self, serial, cb):
+        self.resolvecb = cb
+        self.didanything = False
         if serial:
-            self._resolvedepsserial(target, makefile, targetstack, rulestack, cb)
+            self._resolvedepsserial()
         else:
-            self._resolvedepsparallel(target, makefile, targetstack, rulestack, cb)
+            self._resolvedepsparallel()
 
-    def _resolvedepsserial(self, target, makefile, targetstack, rulestack, cb):
-        resolvelist = list(self.deps)
-        self.didanything = False
+    def _depfinishedserial(self, error, didanything):
+        assert error in (True, False)
+
+        if didanything:
+            self.didanything = True
 
-        def depfinished(error, didanything):
-            if error is not None:
-                cb(error=error, didanything=None)
+        if error:
+            self.error = True
+            if not self.makefile.keepgoing:
+                self.resolvecb(error=True, didanything=self.didanything)
                 return
+        
+        if len(self.resolvelist):
+            self.makefile.context.defer(self.resolvelist.pop(0).make,
+                                        self.makefile, self.targetstack, self._depfinishedserial)
+        else:
+            self.resolvecb(error=self.error, didanything=self.didanything)
+
+    def _resolvedepsserial(self):
+        self.resolvelist = list(self.deps)
+        self._depfinishedserial(False, False)
 
-            if didanything:
-                self.didanything = True
-            
-            if len(resolvelist):
-                makefile.context.defer(resolvelist.pop(0).make, makefile, targetstack, rulestack, depfinished)
-            else:
-                cb(error=None, didanything=self.didanything)
+    def _startdepparallel(self, d):
+        if self.makefile.error:
+            depfinished(True, False)
+        else:
+            d.make(self.makefile, self.targetstack, self._depfinishedparallel)
 
-        depfinished(None, False)
+    def _depfinishedparallel(self, error, didanything):
+        assert error in (True, False)
 
-    def _resolvedepsparallel(self, target, makefile, targetstack, rulestack, cb):
+        if error:
+            print "<%s>: Found error" % self.target.target
+            self.error = True
+        if didanything:
+            self.didanything = True
+
         self.depsremaining -= 1
         if self.depsremaining == 0:
-            cb(error=None, didanything=False)
+            self.resolvecb(error=self.error, didanything=self.didanything)
+
+    def _resolvedepsparallel(self):
+        self.depsremaining -= 1
+        if self.depsremaining == 0:
+            self.resolvecb(error=self.error, didanything=self.didanything)
             return
 
-        self.error = None
         self.didanything = False
 
-        def startdep(d):
-            if self.error is not None:
-                depfinished(None, False)
-            else:
-                d.make(makefile, targetstack, rulestack, depfinished)
+        for d in self.deps:
+            self.makefile.context.defer(self._startdepparallel, d)
+
+    def _commandcb(self, error):
+        assert error in (True, False)
 
-        def depfinished(error, didanything):
-            if error is not None:
-                if self.error is None:
-                    self.error = error
-            elif didanything:
-                self.didanything = True
+        if error:
+            self.runcb(error=True)
+            return
 
-            self.depsremaining -= 1
-            
-            if self.depsremaining == 0:
-                cb(error=self.error, didanything=self.didanything)
+        if len(self.commands):
+            self.commands.pop(0)(self._commandcb)
+        else:
+            self.runcb(error=False)
 
-        for d in self.deps:
-            makefile.context.defer(startdep, d)
-
-    def runcommands(self, target, makefile, avoidremakeloop, indent, cb):
+    def runcommands(self, indent, cb):
         assert not self.running
         self.running = True
+
+        self.runcb = cb
+
         if self.rule is None or not len(self.rule.commands):
-            if target.mtime is None:
-                target._beingremade()
+            if self.target.mtime is None:
+                self.target.beingremade()
             else:
                 for d in self.deps:
-                    if mtimeislater(d.mtime, target.mtime):
-                        target._beingremade()
+                    if mtimeislater(d.mtime, self.target.mtime):
+                        self.target.beingremade()
                         break
-            cb(error=None)
+            cb(error=False)
             return
 
-        def commandcb(error):
-            if error is not None:
-                cb(error=error)
-                return
-
-            if len(commands):
-                commands.pop(0)(commandcb)
-            else:
-                cb(error=None)
-
         remake = False
-        if target.mtime is None:
+        if self.target.mtime is None:
             remake = True
-            _log.info("%sRemaking %s using rule at %s: target doesn't exist or is a forced target", indent, target.target, self.rule.loc)
+            _log.info("%sRemaking %s using rule at %s: target doesn't exist or is a forced target", indent, self.target.target, self.rule.loc)
 
         if not remake:
             if self.rule.doublecolon:
                 if len(self.deps) == 0:
-                    if avoidremakeloop:
-                        _log.info("%sNot remaking %s using rule at %s because it would introduce an infinite loop.", indent, target.target, self.rule.loc)
+                    if self.avoidremakeloop:
+                        _log.info("%sNot remaking %s using rule at %s because it would introduce an infinite loop.", indent, self.target.target, self.rule.loc)
                     else:
-                        _log.info("%sRemaking %s using rule at %s because there are no prerequisites listed for a double-colon rule.", indent, target.target, self.rule.loc)
+                        _log.info("%sRemaking %s using rule at %s because there are no prerequisites listed for a double-colon rule.", indent, self.target.target, self.rule.loc)
                         remake = True
 
         if not remake:
             for d in self.deps:
-                if mtimeislater(d.mtime, target.mtime):
-                    _log.info("%sRemaking %s using rule at %s because %s is newer.", indent, target.target, self.rule.loc, d.target)
+                if mtimeislater(d.mtime, self.target.mtime):
+                    _log.info("%sRemaking %s using rule at %s because %s is newer.", indent, self.target.target, self.rule.loc, d.target)
                     remake = True
                     break
 
         if remake:
-            target._beingremade()
-            target._didanything = True
+            self.target.beingremade()
+            self.target.didanything = True
             try:
-                commands = [c for c in self.rule.getcommands(target, makefile)]
+                self.commands = [c for c in self.rule.getcommands(self.target, self.makefile)]
             except util.MakeError, e:
-                cb(error=e)
+                print e
+                sys.stdout.flush()
+                cb(error=True)
                 return
 
-            commandcb(None)
+            self._commandcb(False)
         else:
-            cb(error=None)
+            cb(error=False)
+
+MAKESTATE_NONE = 0
+MAKESTATE_FINISHED = 1
+MAKESTATE_WORKING = 2
 
 class Target(object):
     """
     An actual (non-pattern) target.
 
     It holds target-specific variables and a list of rules. It may also point to a parent
     PatternTarget, if this target is being created by an implicit rule.
 
@@ -838,181 +962,112 @@ class Target(object):
             if mtime is not None:
                 self.vpathtarget = t
                 self.mtime = mtime
                 return
 
         self.vpathtarget = self.target
         self.mtime = None
         
-    def _beingremade(self):
+    def beingremade(self):
         """
         When we remake ourself, we need to reset our mtime and vpathtarget.
 
         We store our old mtime so that $? can calculate out-of-date prerequisites.
         """
         self.realmtime = self.mtime
         self.mtime = None
         self.vpathtarget = self.target
 
-    def _notifydone(self, makefile):
-        assert self._state == MAKESTATE_WORKING
+    def notifydone(self, makefile):
+        assert self._state == MAKESTATE_WORKING, "State was %s" % self._state
 
         self._state = MAKESTATE_FINISHED
         for cb in self._callbacks:
-            makefile.context.defer(cb, error=self._makeerror, didanything=self._didanything)
+            makefile.context.defer(cb, error=self.error, didanything=self.didanything)
         del self._callbacks 
 
-    def make(self, makefile, targetstack, rulestack, cb, avoidremakeloop=False):
+    def make(self, makefile, targetstack, cb, avoidremakeloop=False):
         """
         If we are out of date, asynchronously make ourself. This is a multi-stage process, mostly handled
-        by enclosed functions:
+        by the helper objects RemakeTargetSerially, RemakeTargetParallel,
+        RemakeRuleContext. These helper objects should keep us from developing
+        any cyclical dependencies.
 
         * resolve dependencies (synchronous)
         * gather a list of rules to execute and related dependencies (synchronous)
-        * for each rule (rulestart)
-        ** remake dependencies (asynchronous, toplevel, callback to start each dependency is `depstart`,
-           callback when each is finished is `depfinished``
-        ** build list of commands to execute (synchronous, in `runcommands`)
-        ** execute each command (asynchronous, runcommands.commandcb)
-        * asynchronously notify rulefinished when each rule is complete
+        * for each rule (in parallel)
+        ** remake dependencies (asynchronous)
+        ** build list of commands to execute (synchronous)
+        ** execute each command (asynchronous)
+        * asynchronously notify when all rules are complete
 
         @param cb A callback function to notify when remaking is finished. It is called
-               thusly: callback(error=exception/None, didanything=True/False/None)
+               thusly: callback(error=True/False, didanything=True/False)
                If there is no asynchronous activity to perform, the callback may be called directly.
         """
 
         serial = makefile.context.jcount == 1
         
         if self._state == MAKESTATE_FINISHED:
-            cb(error=self._makeerror, didanything=self._didanything)
+            cb(error=self.error, didanything=self.didanything)
             return
             
         if self._state == MAKESTATE_WORKING:
             assert not serial
             self._callbacks.append(cb)
             return
 
         assert self._state == MAKESTATE_NONE
 
         self._state = MAKESTATE_WORKING
         self._callbacks = [cb]
-        self._makeerror = None
-        self._didanything = False
+        self.error = False
+        self.didanything = False
 
         indent = getindent(targetstack)
 
         try:
-            self.resolvedeps(makefile, targetstack, rulestack, False)
+            self.resolvedeps(makefile, targetstack, [], False)
         except util.MakeError, e:
-            self._makeerror = e
-            self._notifydone(makefile)
+            print e
+            self.error = True
+            self.notifydone(makefile)
             return
 
         assert self.vpathtarget is not None, "Target was never resolved!"
         if not len(self.rules):
-            self._notifydone(makefile)
+            self.notifydone(makefile)
             return
 
         if self.isdoublecolon():
-            rulelist = [RemakeRuleContext(r, [makefile.gettarget(p) for p in r.prerequisites]) for r in self.rules]
+            rulelist = [RemakeRuleContext(self, makefile, r, [makefile.gettarget(p) for p in r.prerequisites], targetstack, avoidremakeloop) for r in self.rules]
         else:
             alldeps = []
 
             commandrule = None
             for r in self.rules:
                 rdeps = [makefile.gettarget(p) for p in r.prerequisites]
                 if len(r.commands):
                     assert commandrule is None
                     commandrule = r
                     # The dependencies of the command rule are resolved before other dependencies,
                     # no matter the ordering of the other no-command rules
                     alldeps[0:0] = rdeps
                 else:
                     alldeps.extend(rdeps)
 
-            rulelist = [RemakeRuleContext(commandrule, alldeps)]
+            rulelist = [RemakeRuleContext(self, makefile, commandrule, alldeps, targetstack, avoidremakeloop)]
 
         targetstack = targetstack + [self.target]
 
         if serial:
-            def resolvecb(error, didanything):
-                if error is not None:
-                    self._makeerror = error
-                    self._notifydone(makefile)
-                    return
-
-                if didanything:
-                    self._didanything = True
-                rulelist.pop(0).runcommands(self, makefile, avoidremakeloop, indent, commandscb)
-
-            def commandscb(error):
-                if error is not None:
-                    self._makeerror = error
-                    self._notifydone(makefile)
-                    return
-
-                if not len(rulelist):
-                    self._notifydone(makefile)
-                    return
-
-                rulelist[0].resolvedeps(self, makefile, targetstack, rulestack, serial, resolvecb)
-
-            commandscb(None)
+            RemakeTargetSerially(self, makefile, indent, rulelist)
         else:
-            def doresolve(r):
-                if self._makeerror is not None:
-                    resolvecb(None, False)
-                else:
-                    r.resolvedeps(self, makefile, targetstack, rulestack, serial, resolvecb)
-
-            def resolvecb(error, didanything):
-                if error is not None:
-                    if self._makeerror is None:
-                        self._makeerror = error
-                elif didanything:
-                    self._didanything = didanything
-
-                if self._makeerror is not None:
-                    r = rulelist.pop()
-                    assert not r.running
-                    if not len(rulelist):
-                        self._notifydone(makefile)
-                    return
-
-                rtop = rulelist[0]
-                if rtop.running or rtop.depsremaining != 0:
-                    return
-
-                rtop.runcommands(self, makefile, avoidremakeloop, indent, commandscb)
-
-            def commandscb(error):
-                if error is not None:
-                    if self._makeerror is None:
-                        self._makeerror = error
-
-                r = rulelist.pop(0)
-                assert r.running
-
-                if not len(rulelist):
-                    self._notifydone(makefile)
-                    return
-
-                if self._makeerror is not None:
-                    return
-
-                rtop = rulelist[0]
-                if rtop.running or rtop.depsremaining != 0:
-                    return
-
-                rtop.runcommands(self, makefile, avoidremakeloop, indent, commandscb)
-
-            for r in rulelist:
-                makefile.context.defer(doresolve, r)
-
+            RemakeTargetParallel(self, makefile, indent, rulelist)
 
 def dirpart(p):
     d, s, f = util.strrpartition(p, '/')
     if d == '':
         return '.'
 
     return d
 
@@ -1079,19 +1134,20 @@ class _CommandWrapper(object):
         self.ignoreErrors = ignoreErrors
         self.loc = loc
         self.cline = cline
         self.kwargs = kwargs
         self.context = context
 
     def _cb(self, res):
         if res != 0 and not self.ignoreErrors:
-            self.usercb(error=DataError("command '%s' failed, return code %s" % (self.cline, res), self.loc))
+            print "%s: command '%s' failed, return code %i" % (self.loc, self.cline, res)
+            self.usercb(error=True)
         else:
-            self.usercb(error=None)
+            self.usercb(error=False)
 
     def __call__(self, cb):
         self.usercb = cb
         process.call(self.cline, loc=self.loc, cb=self._cb, context=self.context, **self.kwargs)
 
 def getcommandsforrule(rule, target, makefile, prerequisites, stem):
     v = Variables(parent=target.variables)
     setautomaticvariables(v, makefile, target, prerequisites)
@@ -1213,30 +1269,31 @@ class PatternRule(object):
         return [p.resolve(dir, stem) for p in self.prerequisites]
 
 class Makefile(object):
     """
     The top-level data structure for makefile execution. It holds Targets, implicit rules, and other
     state data.
     """
 
-    def __init__(self, workdir=None, env=None, restarts=0, make=None, makeflags=None, makelevel=0, context=None, targets=()):
+    def __init__(self, workdir=None, env=None, restarts=0, make=None, makeflags=None, makelevel=0, context=None, targets=(), keepgoing=False):
         self.defaulttarget = None
 
         if env is None:
             env = os.environ
         self.env = env
 
         self.variables = Variables()
         self.variables.readfromenvironment(env)
 
         self.context = context
         self.exportedvars = {}
         self.overrides = []
         self._targets = {}
+        self.keepgoing = keepgoing
         self._patternvariables = [] # of (pattern, variables)
         self.implicitrules = []
         self.parsingfinished = False
 
         self._patternvpaths = [] # of (pattern, [dir, ...])
 
         if workdir is None:
             workdir = os.getcwd()
@@ -1344,16 +1401,18 @@ class Makefile(object):
             for r in t.rules:
                 for p in r.prerequisites:
                     self.gettarget(p).explicit = True
 
         np = self.gettarget('.NOTPARALLEL')
         if len(np.rules):
             self.context = process.getcontext(1)
 
+        self.error = False
+
     def include(self, path, required=True, loc=None):
         """
         Include the makefile at `path`.
         """
         fspath = os.path.join(self.workdir, path)
         if os.path.exists(fspath):
             self.included.append(path)
             stmts = parser.parsefile(fspath)
@@ -1407,38 +1466,40 @@ class Makefile(object):
             t.resolvevpath(self)
             oldmtime = t.mtime
 
             mlist.append((t, oldmtime))
 
         if serial:
             remakelist = [self.gettarget(f) for f in self.included]
             def remakecb(error, didanything):
-                if error is not None:
-                    print "Error remaking makefiles (ignored): ", error
+                assert error in (True, False)
+                if error:
+                    print "Error remaking makefiles (ignored)"
 
                 if len(remakelist):
                     t = remakelist.pop(0)
-                    t.make(self, [], [], avoidremakeloop=True, cb=remakecb)
+                    t.make(self, [], avoidremakeloop=True, cb=remakecb)
                 else:
                     remakedone()
 
-            remakelist.pop(0).make(self, [], [], avoidremakeloop=True, cb=remakecb)
+            remakelist.pop(0).make(self, [], avoidremakeloop=True, cb=remakecb)
         else:
             o = util.makeobject(('remakesremaining',), remakesremaining=len(self.included))
             def remakecb(error, didanything):
-                if error is not None:
-                    print "Error remaking makefiles (ignored): ", error
+                assert error in (True, False)
+                if error:
+                    print "Error remaking makefiles (ignored)"
 
                 o.remakesremaining -= 1
                 if o.remakesremaining == 0:
                     remakedone()
 
             for t, mtime in mlist:
-                t.make(self, [], [], avoidremakeloop=True, cb=remakecb)
+                t.make(self, [], avoidremakeloop=True, cb=remakecb)
 
     flagescape = re.compile(r'([\s\\])')
 
     def getsubenvironment(self, variables):
         env = dict(self.env)
         for vname, v in self.exportedvars.iteritems():
             if v:
                 flavor, source, val = variables.get(vname)
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/keep-going-doublecolon.mk
@@ -0,0 +1,16 @@
+#T commandline: ['-k']
+#T returncode: 2
+#T grep-for: "TEST-PASS"
+
+all:: t1
+	@echo TEST-FAIL "(t1)"
+
+all:: t2
+	@echo TEST-PASS
+
+t1:
+	@false
+
+t2:
+	touch $@
+
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/keep-going-parallel.mk
@@ -0,0 +1,11 @@
+#T commandline: ['-k', '-j2']
+#T returncode: 2
+#T grep-for: "TEST-PASS"
+
+all: t1 slow1 slow2 slow3 t2
+
+t2:
+	@echo TEST-PASS
+
+slow%:
+	sleep 1
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/keep-going.mk
@@ -0,0 +1,14 @@
+#T commandline: ['-k']
+#T returncode: 2
+#T grep-for: "TEST-PASS"
+
+all: t2 t3
+
+t1:
+	@false
+
+t2: t1
+	@echo TEST-FAIL
+
+t3:
+	@echo TEST-PASS
new file mode 100644
--- /dev/null
+++ b/build/pymake/tests/makeflags.mk
@@ -0,0 +1,6 @@
+#T environment: {'MAKEFLAGS': 'OVAR=oval'}
+
+all:
+	test "$(OVAR)" = "oval"
+	@echo TEST-PASS
+
--- a/build/pymake/tests/runtests.py
+++ b/build/pymake/tests/runtests.py
@@ -7,16 +7,17 @@ For each test, we run gmake -f test.mk. 
 Each test is run in an empty directory.
 
 The test file may contain lines at the beginning to alter the default behavior. These are all evaluated as python:
 
 #T commandline: ['extra', 'params', 'here']
 #T returncode: 2
 #T returncode-on: {'win32': 2}
 #T environment: {'VAR': 'VALUE}
+#T grep-for: "text"
 """
 
 from subprocess import Popen, PIPE, STDOUT
 from optparse import OptionParser
 import os, re, sys, shutil
 
 thisdir = os.path.dirname(os.path.abspath(__file__))
 
@@ -58,16 +59,17 @@ for makefile in makefiles:
         #XXX: hack to run pymake on windows
         if opts.make.endswith('.py'):
             cline = [sys.executable] + cline
         #XXX: hack so we can specialize the separator character on windows.
         # we really shouldn't need this, but y'know
         cline += ['__WIN32__=1']
         
     returncode = 0
+    grepfor = None
 
     env = dict(os.environ)
 
     mdata = open(makefile)
     for line in mdata:
         m = tre.search(line)
         if m is None:
             break
@@ -78,32 +80,41 @@ for makefile in makefiles:
         elif key == 'returncode':
             returncode = data
         elif key == 'returncode-on':
             if sys.platform in data:
                 returncode = data[sys.platform]
         elif key == 'environment':
             for k, v in data.iteritems():
                 env[k] = v
+        elif key == 'grep-for':
+            grepfor = data
         else:
             print >>sys.stderr, "Unexpected #T key: %s" % key
             sys.exit(1)
 
     mdata.close()
 
     p = Popen(cline, stdout=PIPE, stderr=STDOUT, env=env)
     stdout, d = p.communicate()
     if p.returncode != returncode:
         print "FAIL (returncode=%i)" % p.returncode
         print stdout
     elif stdout.find('TEST-FAIL') != -1:
         print "FAIL"
         print stdout
     elif returncode == 0:
-        if stdout.find('TEST-PASS') != -1:
+        if stdout.find(grepfor or 'TEST-PASS') != -1:
             print "PASS"
         else:
-            print "FAIL (no passing output)"
+            print "FAIL (no expected output)"
+            print stdout
+    # check that test produced the expected output while failing
+    elif grepfor:
+        if stdout.find(grepfor) != -1:
+            print "PASS"
+        else:
+            print "FAIL (no expected output)"
             print stdout
     else:
         print "EXPECTED-FAIL"
 
 shutil.rmtree(opts.tempdir)