merge from default, this passes all tests on Linux with gmake/pymake, but will need some fixup on win32 win32-msys
authorTed Mielczarek <ted.mielczarek@gmail.com>
Fri, 20 Feb 2009 15:13:11 -0500
branchwin32-msys
changeset 145 660f249d254dcebbd1ee456b464e0b66c255a7c6
parent 144 bf4d1889ba7117450ce0daa9c21d4d0f2292a386 (current diff)
parent 143 a2fd52a18fd2260a5bd7e2f26a94ab4e1709af31 (diff)
child 146 ddbb23e8cb015cfdf07d452237f0fbeda52e28c6
push id82
push usertmielczarek@mozilla.com
push dateFri, 20 Feb 2009 20:11:45 +0000
merge from default, this passes all tests on Linux with gmake/pymake, but will need some fixup on win32
make.py
pymake/data.py
pymake/functions.py
pymake/parser.py
tests/file-functions-symlinks.mk
tests/file-functions.mk
--- a/README
+++ b/README
@@ -16,29 +16,30 @@ The Mozilla project inspired this tool w
 
 * Enable experiments with build system. By writing a makefile parser, we can experiment
   with converting in-tree makefiles to another build system, such as SCons, waf, ant, ...insert
   your favorite build tool here. Or we could experiment along the lines of makepp, keeping
   our existing makefiles, but change the engine to build a global dependency graph.
 
 KNOWN INCOMPATIBILITIES
 
-* $(eval) is not yet supported
-
 * Parallel builds (-j > 1) are not yet supported
 
-* The vpath directive is not yet supported
+* Order-only prerequisites are not yet supported
 
-* Order-only prerequisites are not yet supported
+* Secondary expansion is not yet supported.
 
 * Target-specific variables behave differently than in GNU make: in pymake, the target-specific
   variable only applies to the specific target that is mentioned, and does not apply recursively
   to all dependencies which are remade. This is an intentional change: the behavior of GNU make
   is neither deterministic nor intuitive.
 
+* $(eval) is only supported during the parse phase. Any attempt to recursively expand
+  an $(eval) function during command execution will fail. This is an intentional incompatibility.
+
 * Windows is unlikely to work properly. There are subtle issues to figure out with Windows
   file paths and shell execution, because Python is not an MSYS project but we'd like it to use
   the MSYS shell when appropriate/necessary.
 
 * There is a subtle difference in execution order that can cause unexpected changes in the
   following circumstance:
 ** A file `foo.c` exists on the VPATH
 ** A rule for `foo.c` exists with a dependency on `tool` and no commands
--- a/make.py
+++ b/make.py
@@ -1,178 +1,14 @@
 #!/usr/bin/env python
 
 """
 make.py
 
 A drop-in or mostly drop-in replacement for GNU make.
 """
 
-import os, subprocess, sys, logging, time
-from optparse import OptionParser
-from pymake.data import Makefile, DataError
-from pymake.parser import parsestream, parsecommandlineargs, SyntaxError
-
-def parsemakeflags():
-    makeflags = os.environ.get('MAKEFLAGS', '')
-    makeflags = makeflags.strip()
-
-    if makeflags == '':
-        return []
-
-    if makeflags[0] not in ('-', ' '):
-        makeflags = '-' + makeflags
-
-    opts = []
-    curopt = ''
-
-    i = 0
-    while i < len(makeflags):
-        c = makeflags[i]
-        if c.isspace():
-            opts.append(curopt)
-            curopt = ''
-            i += 1
-            while i < len(makeflags) and makeflags[i].isspace():
-                i += 1
-            continue
-
-        if c == '\\':
-            i += 1
-            if i == len(makeflags):
-                raise DataError("MAKEFLAGS has trailing backslash")
-            c = makeflags[i]
-            
-        curopt += c
-        i += 1
-
-    if curopt != '':
-        opts.append(curopt)
-
-    return opts
-
-def version(*args):
-    print """pymake: GNU-compatible make program
-Copyright (C) 2009 The Mozilla Foundation <http://www.mozilla.org/>
-This is free software; see the source for copying conditions.
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE."""
-    sys.exit(0)
-
-log = logging.getLogger('pymake.execution')
-
-makelevel = int(os.environ.get('MAKELEVEL', '0'))
-
-op = OptionParser()
-op.add_option('-f', '--file', '--makefile',
-              action='append',
-              dest='makefiles',
-              default=[])
-op.add_option('-d',
-              action="store_true",
-              dest="verbose", default=False)
-op.add_option('--debug-log',
-              dest="debuglog", default=None)
-op.add_option('-C', '--directory',
-              dest="directory", default=None)
-op.add_option('-v', '--version',
-              action="callback", callback=version)
-op.add_option('-j', '--jobs', type="int",
-              dest="jobcount", default=1)
-op.add_option('--parse-profile',
-              dest="parseprofile", default=None)
-
-arglist = sys.argv[1:] + parsemakeflags()
-
-options, arguments = op.parse_args(arglist)
+import sys, os
+import pymake.command, pymake.process
 
-shortflags = []
-longflags = []
-
-loglevel = logging.WARNING
-if options.verbose:
-    loglevel = logging.DEBUG
-    shortflags.append('d')
-
-logkwargs = {}
-if options.debuglog:
-    logkwargs['filename'] = options.debuglog
-    longflags.append('--debug-log=%s' % options.debuglog)
-
-if options.jobcount:
-    log.info("pymake doesn't implement -j yet. ignoring")
-    shortflags.append('j%i' % options.jobcount)
-
-makeflags = ''.join(shortflags) + ' ' + ' '.join(longflags)
-
-logging.basicConfig(level=loglevel, **logkwargs)
-
-if options.directory:
-    log.info("Switching to directory: %s" % options.directory)
-    os.chdir(options.directory)
-    
-print "make.py[%i]: Entering directory '%s'" % (makelevel, os.getcwd())
-sys.stdout.flush()
-
-if len(options.makefiles) == 0:
-    if os.path.exists('Makefile'):
-        options.makefiles.append('Makefile')
-    else:
-        print "No makefile found"
-        sys.exit(2)
-
-try:
-    def parse():
-        i = 0
-
-        while True:
-            m = Makefile(restarts=i, make='%s %s' % (sys.executable.replace('\\','/'),
-                                                     sys.argv[0].replace('\\', '/')),
-                         makeflags=makeflags, makelevel=makelevel)
-
-            starttime = time.time()
-            targets = parsecommandlineargs(m, arguments)
-            for f in options.makefiles:
-                m.include(f)
-
-            log.info("Parsing[%i] took %f seconds" % (i, time.time() - starttime,))
-
-            m.finishparsing()
-            if m.remakemakefiles():
-                log.info("restarting makefile parsing")
-                i += 1
-                continue
-
-            return m, targets
-
-    if options.parseprofile is None:
-        m, targets = parse()
-    else:
-        import cProfile
-        cProfile.run("m, targets = parse()", options.parseprofile)
-
-    if len(targets) == 0:
-        if m.defaulttarget is None:
-            print "No target specified and no default target found."
-            sys.exit(2)
-        targets = [m.defaulttarget]
-        tstack = ['<default-target>']
-    else:
-        tstack = ['<command-line>']
-
-
-    starttime = time.time()
-    for t in targets:
-        m.gettarget(t).make(m, ['<command-line>'], [])
-    log.info("Execution took %f seconds" % (time.time() - starttime,))
-
-except (DataError, SyntaxError, subprocess.CalledProcessError), e:
-    print e
-    print "make.py[%i]: Leaving directory '%s'" % (makelevel, os.getcwd())
-    sys.stdout.flush()
-    sys.exit(2)
-
-print "make.py[%i]: Leaving directory '%s'" % (makelevel, os.getcwd())
+pymake.command.main(sys.argv[1:], os.environ, os.getcwd(), context=None, cb=sys.exit)
+pymake.process.ParallelContext.spin()
+assert False, "Not reached"
new file mode 100644
--- /dev/null
+++ b/pymake/command.py
@@ -0,0 +1,211 @@
+"""
+Logic to execute a command
+"""
+
+import os, subprocess, sys, logging, time, traceback
+from optparse import OptionParser
+import data, parser, process, util
+
+# TODO: If this ever goes from relocatable package to system-installed, this may need to be
+# a configured-in path.
+
+makepypath = os.path.normpath(os.path.join(os.path.dirname(__file__), '../make.py'))
+
+def parsemakeflags(env):
+    makeflags = env.get('MAKEFLAGS', '')
+    makeflags = makeflags.strip()
+
+    if makeflags == '':
+        return []
+
+    if makeflags[0] not in ('-', ' '):
+        makeflags = '-' + makeflags
+
+    opts = []
+    curopt = ''
+
+    i = 0
+    while i < len(makeflags):
+        c = makeflags[i]
+        if c.isspace():
+            opts.append(curopt)
+            curopt = ''
+            i += 1
+            while i < len(makeflags) and makeflags[i].isspace():
+                i += 1
+            continue
+
+        if c == '\\':
+            i += 1
+            if i == len(makeflags):
+                raise data.DataError("MAKEFLAGS has trailing backslash")
+            c = makeflags[i]
+            
+        curopt += c
+        i += 1
+
+    if curopt != '':
+        opts.append(curopt)
+
+    return opts
+
+def version(*args):
+    print """pymake: GNU-compatible make program
+Copyright (C) 2009 The Mozilla Foundation <http://www.mozilla.org/>
+This is free software; see the source for copying conditions.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE."""
+
+log = logging.getLogger('pymake.execution')
+
+def main(args, env, cwd, context, cb):
+    try:
+        makelevel = int(env.get('MAKELEVEL', '0'))
+
+        op = OptionParser()
+        op.add_option('-f', '--file', '--makefile',
+                      action='append',
+                      dest='makefiles',
+                      default=[])
+        op.add_option('-d',
+                      action="store_true",
+                      dest="verbose", default=False)
+        op.add_option('--debug-log',
+                      dest="debuglog", default=None)
+        op.add_option('-C', '--directory',
+                      dest="directory", default=None)
+        op.add_option('-v', '--version', action="store_true",
+                      dest="printversion", default=False)
+        op.add_option('-j', '--jobs', type="int",
+                      dest="jobcount", default=1)
+        op.add_option('--no-print-directory', action="store_false",
+                      dest="printdir", default=True)
+
+        options, arguments1 = op.parse_args(parsemakeflags(env))
+        options, arguments2 = op.parse_args(args, values=options)
+
+        arguments = arguments1 + arguments2
+
+        if options.printversion:
+            version()
+            cb(0)
+            return
+
+        shortflags = []
+        longflags = []
+
+        loglevel = logging.WARNING
+        if options.verbose:
+            loglevel = logging.DEBUG
+            shortflags.append('d')
+
+        logkwargs = {}
+        if options.debuglog:
+            logkwargs['filename'] = options.debuglog
+            longflags.append('--debug-log=%s' % options.debuglog)
+
+        if options.directory is None:
+            workdir = cwd
+        else:
+            workdir = os.path.join(cwd, options.directory)
+
+        shortflags.append('j%i' % (options.jobcount,))
+
+        makeflags = ''.join(shortflags) + ' ' + ' '.join(longflags)
+
+        logging.basicConfig(level=loglevel, **logkwargs)
+
+        if context is not None and context.jcount > 1 and options.jobcount == 1:
+            log.debug("-j1 specified, creating new serial execution context")
+            context = process.getcontext(options.jobcount)
+            subcontext = True
+        elif context is None:
+            log.debug("Creating new execution context, jobcount %s" % options.jobcount)
+            context = process.getcontext(options.jobcount)
+            subcontext = True
+        else:
+            log.debug("Using parent execution context")
+            subcontext = False
+
+        if options.printdir:
+            print "make.py[%i]: Entering directory '%s'" % (makelevel, workdir)
+            sys.stdout.flush()
+
+        if len(options.makefiles) == 0:
+            if os.path.exists(os.path.join(workdir, 'Makefile')):
+                options.makefiles.append('Makefile')
+            else:
+                print "No makefile found"
+                cb(2)
+                return
+
+        # subvert python readonly closures
+        o = util.makeobject(('restarts', 'm', 'targets', 'remade', 'error'),
+                            restarts=-1)
+
+        def remakecb(remade):
+            if remade:
+                o.restarts += 1
+                if o.restarts > 0:
+                    log.info("make.py[%i]: Restarting makefile parsing" % (makelevel,))
+                o.m = data.Makefile(restarts=o.restarts, make='%s %s' % (sys.executable, makepypath),
+                                    makeflags=makeflags, makelevel=makelevel, workdir=workdir,
+                                    context=context, env=env)
+
+                o.targets = parser.parsecommandlineargs(o.m, arguments)
+                for f in options.makefiles:
+                    o.m.include(f)
+
+                o.m.finishparsing()
+                o.m.remakemakefiles(remakecb)
+                return
+
+            if len(o.targets) == 0:
+                if o.m.defaulttarget is None:
+                    print "No target specified and no default target found."
+                    cb(2)
+                    return
+                o.targets = [o.m.defaulttarget]
+                tstack = ['<default-target>']
+            else:
+                tstack = ['<command-line>']
+
+            def makecb(error, didanything):
+                o.remade += 1
+
+                log.debug("makecb[%i]: remade %i targets" % (makelevel, o.remade))
+
+                if error is not None:
+                    print error
+                    o.error = True
+
+                if o.remade == len(o.targets):
+                    if subcontext:
+                        context.finish()
+
+                    if options.printdir:
+                        print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
+                    sys.stdout.flush()
+
+                    cb(o.error and 2 or 0)
+
+            o.remade = 0
+            o.error = False
+
+            for t in o.targets:
+                o.m.gettarget(t).make(o.m, ['<command-line>'], [], cb=makecb)
+
+        remakecb(True)
+
+    except (util.MakeError), e:
+        print e
+        if options.printdir:
+            print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
+        sys.stdout.flush()
+        cb(2)
+        return
--- a/pymake/data.py
+++ b/pymake/data.py
@@ -1,25 +1,22 @@
 """
 A representation of makefile data structures.
 """
 
-import logging, re, os, subprocess
-import pymake
+import logging, re, os
+import pymake.parser
+import pymake.functions
+import pymake.process
+import pymake.util
 
 log = logging.getLogger('pymake.data')
 
-class DataError(Exception):
-    def __init__(self, message, loc=None):
-        self.message = message
-        self.loc = loc
-
-    def __str__(self):
-        return "%s: %s" % (self.loc and self.loc or "internal error",
-                           self.message)
+class DataError(pymake.util.MakeError):
+    pass
 
 class ResolutionError(DataError):
     """
     Raised when dependency resolution fails, either due to recursion or to missing
     prerequisites.This is separately catchable so that implicit rule search can try things
     without having to commit.
     """
     pass
@@ -136,28 +133,30 @@ class Expansion(object):
             assert len(self) == 1 or not isinstance(self[-2], str), "Strings didn't fold"
             self[-1] = self[-1].rstrip()
 
     def trimlastnewline(self):
         """Strip only the last newline, if present."""
         if len(self) > 0 and isinstance(self[-1], str) and self[-1][-1] == '\n':
             self[-1] = self[-1][:-1]
 
-    def resolve(self, variables, setting=[]):
+    def resolve(self, makefile, variables, setting=[]):
         """
         Resolve this variable into a value, by interpolating the value
         of other variables.
 
         @param setting (Variable instance) the variable currently
                being set, if any. Setting variables must avoid self-referential
                loops.
         """
+        assert isinstance(makefile, Makefile)
+        assert isinstance(variables, Variables)
         assert isinstance(setting, list)
 
-        return ''.join( (_if_else(isinstance(i, str), lambda: i, lambda: i.resolve(variables, setting))
+        return ''.join( (_if_else(isinstance(i, str), lambda: i, lambda: i.resolve(makefile, variables, setting))
                          for i in self._elements) )
 
     def __len__(self):
         return len(self._elements)
 
     def __getitem__(self, key):
         return self._elements[key]
 
@@ -187,18 +186,18 @@ class Variables(object):
     SOURCE_AUTOMATIC = 4
     # I have no intention of supporting builtin rules or variables that go with them
     # SOURCE_IMPLICIT = 5
 
     def __init__(self, parent=None):
         self._map = {}
         self.parent = parent
 
-    def readfromenvironment(self):
-        for k, v in os.environ.iteritems():
+    def readfromenvironment(self, env):
+        for k, v in env.iteritems():
             self.set(k, self.FLAVOR_SIMPLE, self.SOURCE_ENVIRONMENT, v)
 
     def get(self, name, expand=True):
         """
         Get the value of a named variable. Returns a tuple (flavor, source, value)
 
         If the variable is not present, returns (None, None, None)
 
@@ -258,31 +257,31 @@ class Variables(object):
         prevflavor, prevsource, prevvalue = self.get(name)
         if prevsource is not None and source > prevsource:
             # TODO: give a location for this warning
             log.warning("not setting variable '%s', set by higher-priority source to value '%s'" % (name, prevvalue))
             return
 
         self._map[name] = (flavor, source, value)
 
-    def append(self, name, source, value, variables):
+    def append(self, name, source, value, variables, makefile):
         assert source in (self.SOURCE_OVERRIDE, self.SOURCE_MAKEFILE, self.SOURCE_AUTOMATIC)
         assert isinstance(value, str)
         
         if name in self._map:
             prevflavor, prevsource, prevvalue = self._map[name]
             if source > prevsource:
                 # TODO: log a warning?
                 return
 
             if prevflavor == self.FLAVOR_SIMPLE:
                 d = pymake.parser.Data(None, None)
                 d.append(value, pymake.parser.Location("Expansion of variable '%s'" % (name,), 1, 0))
                 e, t, o = pymake.parser.parsemakesyntax(d, 0, (), pymake.parser.iterdata)
-                val = e.resolve(variables, [name])
+                val = e.resolve(makefile, variables, [name])
             else:
                 val = value
 
             self._map[name] = prevflavor, prevsource, prevvalue + ' ' + val
         else:
             self._map[name] = self.FLAVOR_APPEND, source, value
 
     def merge(self, other):
@@ -411,16 +410,20 @@ class Pattern(object):
 
     backre = re.compile(r'[%\\]')
     def __str__(self):
         if not self.ispattern:
             return self.backre.sub(r'\\\1', self.data[0])
 
         return self.backre.sub(r'\\\1', self.data[0]) + '%' + self.data[1]
 
+MAKESTATE_NONE = 0
+MAKESTATE_FINISHED = 1
+MAKESTATE_WORKING = 2
+
 class Target(object):
     """
     An actual (non-pattern) target.
 
     It holds target-specific variables and a list of rules. It may also point to a parent
     PatternTarget, if this target is being created by an implicit rule.
 
     The rules associated with this target may be Rule instances or, in the case of static pattern
@@ -429,22 +432,17 @@ class Target(object):
 
     def __init__(self, target, makefile):
         assert isinstance(target, str)
         self.target = target
         self.vpathtarget = None
         self.rules = []
         self.variables = Variables(makefile.variables)
         self.explicit = False
-
-        # self.remade is a tri-state:
-        #   None - we haven't remade yet
-        #   True - we did something to remake ourself
-        #   False - we did nothing to remake ourself
-        self.remade = None
+        self._state = MAKESTATE_NONE
 
     def addrule(self, rule):
         assert isinstance(rule, (Rule, PatternRuleInstance))
         if len(self.rules) and rule.doublecolon != self.rules[0].doublecolon:
             raise DataError("Cannot have single- and double-colon rules for the same target. Prior rule location: %s" % self.rules[0].loc, rule.loc)
 
         if isinstance(rule, PatternRuleInstance):
             if len(rule.prule.targetpatterns) != 1:
@@ -619,127 +617,236 @@ class Target(object):
             self.vpathtarget = self.target
             self.mtime = None
             return
 
         if self.target.startswith('-l'):
             stem = self.target[2:]
             f, s, e = makefile.variables.get('.LIBPATTERNS')
             if e is not None:
-                libpatterns = map(Pattern, splitwords(e.resolve(makefile.variables, [])))
+                libpatterns = map(Pattern, splitwords(e.resolve(makefile, makefile.variables)))
                 if len(libpatterns):
-                    searchdirs = [''] + makefile.vpath
+                    searchdirs = ['']
+                    searchdirs.extend(makefile.getvpath(self.target))
 
                     for lp in libpatterns:
                         if not lp.ispattern():
                             raise DataError('.LIBPATTERNS contains a non-pattern')
 
                         libname = lp.resolve('', stem)
 
                         for dir in searchdirs:
                             libpath = os.path.join(dir, libname)
-                            mtime = getmtime(libpath)
+                            fspath = os.path.join(makefile.workdir, libpath)
+                            mtime = getmtime(fspath)
                             if mtime is not None:
                                 self.vpathtarget = libpath
                                 self.mtime = mtime
                                 return
 
                     self.vpathtarget = self.target
                     self.mtime = None
                     return
 
         search = [self.target]
         if not os.path.isabs(self.target):
-            search += [os.path.join(dir, self.target).replace('\\','/')
-                       for dir in makefile.vpath]
+            search += [os.path.join(dir, self.target)
+                       for dir in makefile.getvpath(self.target)]
 
         for t in search:
-            mtime = getmtime(t)
+            fspath = os.path.join(makefile.workdir, t)
+            mtime = getmtime(fspath)
             if mtime is not None:
                 self.vpathtarget = t
                 self.mtime = mtime
                 return
 
         self.vpathtarget = self.target
         self.mtime = None
         
-    def remake(self):
+    def _beingremade(self):
         """
         When we remake ourself, we need to reset our mtime and vpathtarget.
 
         We store our old mtime so that $? can calculate out-of-date prerequisites.
         """
         self.realmtime = self.mtime
         self.mtime = None
         self.vpathtarget = self.target
 
-    def make(self, makefile, targetstack, rulestack, required=True):
-        """
-        If we are out of date, make ourself.
+    def _notifyerror(self, e):
+        log.debug("Making target '%s' failed with error %s" % (self.target, e))
+
+        if self._state == MAKESTATE_FINISHED:
+            # multiple callbacks failed. The first one already finished us, so we ignore this one
+            return
+
+        self._state = MAKESTATE_FINISHED
+        self._makeerror = e
+        for cb in self._callbacks:
+            cb(error=e, didanything=None)
+        del self._callbacks 
 
-        For now, making is synchronous/serialized. -j magic will come later.
+    def _notifysuccess(self, didanything):
+        log.debug("Making target '%s' succeeded" % (self.target,))
+
+        self._state = MAKESTATE_FINISHED
+        self._makeerror = None
+        self._didanything = didanything
 
-        @returns True if anything was done to remake this target
+        for cb in self._callbacks:
+            cb(error=None, didanything=didanything)
+
+        del self._callbacks
+
+    def make(self, makefile, targetstack, rulestack, cb, required=True, avoidremakeloop=False):
         """
-        if self.remade is not None:
-            return self.remade
+        If we are out of date, asynchronously make ourself. This is a multi-stage process, mostly handled
+        by enclosed functions:
+
+        * resolve dependencies (synchronous)
+        * remake dependencies (asynchronous, toplevel, callback is `depcallback`
+        * build list of commands to execute (synchronous, in `makeself`)
+        * execute each command (asynchronous, makeself.commandcb)
 
-        self.resolvedeps(makefile, targetstack, rulestack, required)
-        assert self.vpathtarget is not None, "Target was never resolved!"
+        @param cb A callback function to notify when remaking is finished. It is called
+               thusly: callback(error=exception/None, didanything=True/False/None)
+               If there is no asynchronous activity to perform, the callback may be called directly.
+        """
+        if self._state == MAKESTATE_FINISHED:
+            if self._makeerror is not None:
+                log.debug("Already made target '%s', got error %s" % (self.target, self._makeerror))
+                cb(error=self._makeerror, didanything=False) #XXX?
+            else:
+                log.debug("Already made target '%s'" % (self.target,))
+                cb(error=None, didanything=self._didanything)
+            return
+            
+        if self._state == MAKESTATE_WORKING:
+            log.debug("Already making target '%s', adding callback. targetstack %r" % (self.target, targetstack))
+            self._callbacks.append(cb)
+            return
 
-        targetstack = targetstack + [self.target]
+        assert self._state == MAKESTATE_NONE
+        log.debug("Starting to make target '%s', targetstack %r" % (self.target, targetstack))
+
+        self._state = MAKESTATE_WORKING
+        self._callbacks = [cb]
 
         indent = getindent(targetstack)
 
-        didanything = False
+        # this object exists solely as a container to subvert python's read-only closures
+        o = pymake.util.makeobject(('unmadedeps', 'didanything', 'error'))
+        
+        def depcallback(error, didanything):
+            assert self._state == MAKESTATE_WORKING
+
+            if error is not None:
+                o.error = error
+            else:
+                assert didanything is not None
+                if didanything:
+                    o.didanything = True
+
+            o.unmadedeps -= 1
+
+            if o.unmadedeps != 0:
+                return
+
+            if o.error:
+                self._notifyerror(o.error)
+            else:
+                makeself()
 
-        if len(self.rules) == 0:
-            pass
-        elif self.isdoublecolon():
-            for r in self.rules:
+        def makeself():
+            """
+            Asynchronous dependency-making is finished. Now gather and asynchronously run our own commands.
+            """
+            commands = []
+            if len(self.rules) == 0:
+                pass
+            elif self.isdoublecolon():
+                for r, deps in _resolvedrules:
+                    remake = False
+                    if len(deps) == 0:
+                        if avoidremakeloop:
+                            log.info(indent + "Not remaking %s using rule at %s because it would introduce an infinite loop." % (self.target, r.loc))
+                        else:
+                            log.info(indent + "Remaking %s using rule at %s because there are no prerequisites listed for a double-colon rule." % (self.target, r.loc))
+                            remake = True
+                    else:
+                        if self.mtime is None:
+                            log.info(indent + "Remaking %s using rule at %s because it doesn't exist or is a forced target" % (self.target, r.loc))
+                            remake = True
+                        else:
+                            for d in deps:
+                                if mtimeislater(d.mtime, self.mtime):
+                                    log.info(indent + "Remaking %s using rule at %s because %s is newer." % (self.target, r.loc, d.target))
+                                    remake = True
+                                    break
+                    if remake:
+                        self._beingremade()
+                        commands.extend(r.getcommands(self, makefile))
+            else:
+                commandrule = None
                 remake = False
                 if self.mtime is None:
-                    log.info("Remaking %s using rule at %s because it doesn't exist or is a forced target" % (self.target, r.loc))
+                    log.info(indent + "Remaking %s because it doesn't exist or is a forced target" % (self.target,))
                     remake = True
-                for p in r.prerequisites:
-                    dep = makefile.gettarget(p)
-                    didanything = dep.make(makefile, targetstack, []) or didanything
-                    if not remake and mtimeislater(dep.mtime, self.mtime):
-                        log.info(indent + "Remaking %s using rule at %s because %s is newer." % (self.target, r.loc, p))
-                        remake = True
+
+                for r, deps in _resolvedrules:
+                    if len(r.commands):
+                        assert commandrule is None, "Two command rules for a single-colon target?"
+                        commandrule = r
+
+                    if not remake:
+                        for d in deps:
+                            if mtimeislater(d.mtime, self.mtime):
+                                log.info(indent + "Remaking %s because %s is newer" % (self.target, d.target))
+                                remake = True
+
                 if remake:
-                    self.remake()
-                    r.execute(self, makefile)
-                    didanything = True
-        else:
-            commandrule = None
-            remake = False
-            if self.mtime is None:
-                log.info(indent + "Remaking %s because it doesn't exist or is a forced target" % (self.target,))
-                remake = True
+                    self._beingremade()
+                    if commandrule is not None:
+                        commands.extend(commandrule.getcommands(self, makefile))
+
+            def commandcb(error):
+                if error is not None:
+                    self._notifyerror(error)
+                    return
 
-            for r in self.rules:
-                if len(r.commands):
-                    assert commandrule is None, "Two command rules for a single-colon target?"
-                    commandrule = r
-                for p in r.prerequisites:
-                    dep = makefile.gettarget(p)
-                    didanything = dep.make(makefile, targetstack, []) or didanything
-                    if not remake and mtimeislater(dep.mtime, self.mtime):
-                        log.info(indent + "Remaking %s because %s is newer" % (self.target, p))
-                        remake = True
+                if len(commands):
+                    commands.pop(0)(commandcb)
+                else:
+                    self._notifysuccess(o.didanything)
+
+            commandcb(None)
+                    
+        try:
+            self.resolvedeps(makefile, targetstack, rulestack, required)
+            assert self.vpathtarget is not None, "Target was never resolved!"
+
+            _resolvedrules = [(r, [makefile.gettarget(p) for p in r.prerequisites]) for r in self.rules]
+            log.debug("resolvedrules for %r: %r" % (self.target, _resolvedrules))
 
-            if remake:
-                self.remake()
-                if commandrule is not None:
-                    commandrule.execute(self, makefile)
-                didanything = True
+            targetstack = targetstack + [self.target]
+
+            o.didanything = False
+            o.unmadedeps = 1
+            o.error = None
 
-        self.remade = didanything
-        return didanything
+            for r, deps in _resolvedrules:
+                for d in deps:
+                    o.unmadedeps += 1
+                    d.make(makefile, targetstack, [], cb=depcallback)
+
+            depcallback(error=None, didanything=False)
+        
+        except pymake.util.MakeError, e:
+            self._notifyerror(e)
 
 def dirpart(p):
     d, s, f = p.rpartition('/')
     if d == '':
         return '.'
 
     return d
 
@@ -750,17 +857,16 @@ def filepart(p):
 def setautomatic(v, name, plist):
     v.set(name, Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join(plist))
     v.set(name + 'D', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join((dirpart(p) for p in plist)))
     v.set(name + 'F', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join((filepart(p) for p in plist)))
 
 def setautomaticvariables(v, makefile, target, prerequisites):
     prtargets = [makefile.gettarget(p) for p in prerequisites]
     prall = [pt.vpathtarget for pt in prtargets]
-
     proutofdate = [pt.vpathtarget for pt in withoutdups(prtargets)
                    if target.realmtime is None or mtimeislater(pt.mtime, target.realmtime)]
     
     setautomatic(v, '@', [target.vpathtarget])
     if len(prall):
         setautomatic(v, '<', [prall[0]])
 
     setautomatic(v, '?', proutofdate)
@@ -811,16 +917,53 @@ def findmodifiers(command):
             ignoreErrors = True
         elif c.isspace():
             command = command[1:]
         else:
             break
 
     return command, isHidden, isRecursive, ignoreErrors
 
+class CommandWrapper(object):
+    def __init__(self, cline, ignoreErrors, loc, context, **kwargs):
+        self.ignoreErrors = ignoreErrors
+        self.loc = loc
+        self.cline = cline
+        self.kwargs = kwargs
+        self.context = context
+
+    def _cb(self, res):
+        if res != 0 and not self.ignoreErrors:
+            self.usercb(error=DataError("command '%s' failed, return code %s" % (self.cline, res), self.loc))
+        else:
+            self.usercb(error=None)
+
+    def __call__(self, cb):
+        self.usercb = cb
+        pymake.process.call(self.cline, loc=self.loc, cb=self._cb, context=self.context, **self.kwargs)
+
+def getcommandsforrule(rule, target, makefile, prerequisites, stem):
+    v = Variables(parent=target.variables)
+    setautomaticvariables(v, makefile, target, prerequisites)
+    if stem is not None:
+        setautomatic(v, '*', [stem])
+
+    env = makefile.getsubenvironment(v)
+
+    for c in rule.commands:
+        cstring = c.resolve(makefile, v)
+        for cline in splitcommand(cstring):
+            cline, isHidden, isRecursive, ignoreErrors = findmodifiers(cline)
+            if isHidden:
+                echo = None
+            else:
+                echo = "%s$ %s" % (c.loc, cline)
+            yield CommandWrapper(cline, ignoreErrors=ignoreErrors, env=env, cwd=makefile.workdir, loc=c.loc, context=makefile.context,
+                                 echo=echo)
+
 class Rule(object):
     """
     A rule contains a list of prerequisites and a list of commands. It may also
     contain rule-specific variables. This rule may be associated with multiple targets.
     """
 
     def __init__(self, prereqs, doublecolon, loc):
         self.prerequisites = prereqs
@@ -828,39 +971,21 @@ class Rule(object):
         self.commands = []
         self.loc = loc
 
     def addcommand(self, c):
         """Append a command expansion."""
         assert(isinstance(c, Expansion))
         self.commands.append(c)
 
-    def execute(self, target, makefile):
+    def getcommands(self, target, makefile):
         assert isinstance(target, Target)
 
-        v = Variables(parent=target.variables)
-        setautomaticvariables(v, makefile, target, self.prerequisites)
-        # TODO: $* in non-pattern rules sucks
-
-        env = makefile.getsubenvironment(v)
-
-        shell, prependshell = checkmsyscompat()
-        for c in self.commands:
-            cstring = c.resolve(v)
-            for cline in splitcommand(cstring):
-                cline, isHidden, isRecursive, ignoreErrors = findmodifiers(cline)
-                if not len(cline) or cline.isspace():
-                    continue
-                if not isHidden:
-                    print "%s $ %s" % (c.loc, cline)
-                if prependshell:
-                    cline = [shell, "-c", cline]
-                r = subprocess.call(cline, shell=not prependshell, env=env)
-                if r != 0 and not ignoreErrors:
-                    raise DataError("command '%s' failed, return code was %s" % (cline, r), c.loc)
+        return getcommandsforrule(self, target, makefile, self.prerequisites, stem=None)
+        # TODO: $* in non-pattern rules?
 
 class PatternRuleInstance(object):
     """
     This represents a pattern rule instance for a particular target. It has the same API as Rule, but
     different internals, forwarding most information on to the PatternRule.
     """
     def __init__(self, prule, dir, stem, ismatchany):
         assert isinstance(prule, PatternRule)
@@ -869,20 +994,19 @@ class PatternRuleInstance(object):
         self.stem = stem
         self.prule = prule
         self.prerequisites = prule.prerequisitesforstem(dir, stem)
         self.doublecolon = prule.doublecolon
         self.loc = prule.loc
         self.ismatchany = ismatchany
         self.commands = prule.commands
 
-    def execute(self, target, makefile):
+    def getcommands(self, target, makefile):
         assert isinstance(target, Target)
-
-        self.prule.execute(target, makefile, self.dir, self.stem)
+        return getcommandsforrule(self, target, makefile, self.prerequisites, stem=self.dir + self.stem)
 
     def __str__(self):
         return "Pattern rule at %s with stem '%s', matchany: %s doublecolon: %s" % (self.loc,
                                                                                     self.dir + self.stem,
                                                                                     self.ismatchany,
                                                                                     self.doublecolon)
 
 class PatternRule(object):
@@ -932,55 +1056,37 @@ class PatternRule(object):
                 else:
                     stem = p.match(file)
                     if stem is not None:
                         yield PatternRuleInstance(self, dir, stem, False)
 
     def prerequisitesforstem(self, dir, stem):
         return [p.resolve(dir, stem) for p in self.prerequisites]
 
-    def execute(self, target, makefile, dir, stem):
-        assert isinstance(target, Target)
-
-        v = Variables(parent=target.variables)
-        setautomaticvariables(v, makefile, target,
-                              self.prerequisitesforstem(dir, stem))
-        setautomatic(v, '*', [dir + stem])
-
-        env = makefile.getsubenvironment(v)
-
-        shell, prependshell = checkmsyscompat()
-        for c in self.commands:
-            cstring = c.resolve(v)
-            for cline in splitcommand(cstring):
-                cline, isHidden, isRecursive, ignoreErrors = findmodifiers(cline)
-                if not len(cline) or cline.isspace():
-                    continue
-                if not isHidden:
-                    print "%s $ %s" % (c.loc, cline)
-                if prependshell:
-                    cline = [shell, "-c", cline]
-                r = subprocess.call(cline, shell=not prependshell, env=env)
-                if r != 0 and not ignoreErrors:
-                    raise DataError("command '%s' failed, return code was %s" % (cline, r), c.loc)
-
 class Makefile(object):
-    def __init__(self, workdir=None, restarts=0, make=None, makeflags=None, makelevel=0):
+    def __init__(self, workdir=None, env=None, restarts=0, make=None, makeflags=None, makelevel=0, context=None):
         self.defaulttarget = None
 
+        if env is None:
+            env = os.environ
+        self.env = env
+
         self.variables = Variables()
-        self.variables.readfromenvironment()
+        self.variables.readfromenvironment(env)
 
+        self.context = context
         self.exportedvars = set()
         self.overrides = []
         self._targets = {}
         self._patternvariables = [] # of (pattern, variables)
         self.implicitrules = []
         self.parsingfinished = False
 
+        self._patternvpaths = [] # of (pattern, [dir, ...])
+
         if workdir is None:
             workdir = os.getcwd()
         workdir = os.path.realpath(workdir)
         self.workdir = workdir
         self.variables.set('CURDIR', Variables.FLAVOR_SIMPLE,
                            Variables.SOURCE_AUTOMATIC, workdir.replace('\\','/'))
 
         # the list of included makefiles, whether or not they existed
@@ -1034,17 +1140,17 @@ class Makefile(object):
     def gettarget(self, target):
         assert isinstance(target, str)
 
         target = target.rstrip('/')
 
         assert target != '', "empty target?"
 
         if target.find('*') != -1 or target.find('?') != -1 or target.find('[') != -1:
-            raise DataError("pymake doesn't implement wildcards in targets/prerequisites.")
+            raise DataError("wildcards should have been expanded by the parser: '%s'" % (target,))
 
         t = self._targets.get(target, None)
         if t is None:
             t = Target(target, self)
             self._targets[target] = t
         return t
 
     def appendimplicitrule(self, rule):
@@ -1056,77 +1162,119 @@ class Makefile(object):
         Various activities, such as "eval", are not allowed after parsing is
         finished. In addition, various warnings and errors can only be issued
         after the parsing data model is complete. All dependency resolution
         and rule execution requires that parsing be finished.
         """
         self.parsingfinished = True
 
         flavor, source, value = self.variables.get('GPATH')
-        if value is not None and value.resolve(self.variables, ['GPATH']).strip() != '':
+        if value is not None and value.resolve(self, self.variables, ['GPATH']).strip() != '':
             raise DataError('GPATH was set: pymake does not support GPATH semantics')
 
         flavor, source, value = self.variables.get('VPATH')
         if value is None:
-            self.vpath = []
+            self._vpath = []
         else:
-            self.vpath = filter(lambda e: e != '', re.split('[:\s]+', value.resolve(self.variables, ['VPATH'])))
+            self._vpath = filter(lambda e: e != '', re.split('[:\s]+', value.resolve(self, self.variables, ['VPATH'])))
 
         targets = list(self._targets.itervalues())
         for t in targets:
             t.explicit = True
             for r in t.rules:
                 for p in r.prerequisites:
                     self.gettarget(p).explicit = True
 
     def include(self, path, required=True, loc=None):
         """
         Include the makefile at `path`.
         """
         self.included.append(path)
-        if os.path.exists(path):
-            fd = open(path)
-            self.variables.append('MAKEFILE_LIST', Variables.SOURCE_AUTOMATIC, path, None)
+
+        fspath = os.path.join(self.workdir, path)
+        if os.path.exists(fspath):
+            fd = open(fspath)
+            self.variables.append('MAKEFILE_LIST', Variables.SOURCE_AUTOMATIC, path, None, self)
             pymake.parser.parsestream(fd, path, self)
             self.gettarget(path).explicit = True
         elif required:
             raise DataError("Attempting to include file '%s' which doesn't exist." % (path,), loc)
 
-    def remakemakefiles(self):
+    def addvpath(self, pattern, dirs):
+        """
+        Add a directory to the vpath search for the given pattern.
+        """
+        self._patternvpaths.append((pattern, dirs))
+
+    def clearvpath(self, pattern):
+        """
+        Clear vpaths for the given pattern.
+        """
+        self._patternvpaths = [(p, dirs)
+                               for p, dirs in self._patternvpaths
+                               if not p.match(pattern)]
+
+    def clearallvpaths(self):
+        self._patternvpaths = []
+
+    def getvpath(self, target):
+        vp = list(self._vpath)
+        for p, dirs in self._patternvpaths:
+            if p.match(target):
+                vp.extend(dirs)
+
+        return withoutdups(vp)
+
+    def remakemakefiles(self, cb):
         reparse = False
 
+        o = pymake.util.makeobject(('remadecount',),
+                                   remadecount = 0)
+
+        def remakecb(error, didanything):
+            if error is not None:
+                print "Error remaking makefiles (ignored): ", error
+
+            o.remadecount += 1
+            if o.remadecount == len(self.included):
+                assert len(mlist) == len(self.included)
+
+                for t, oldmtime in mlist:
+                    if t.mtime != oldmtime:
+                        cb(remade=True)
+                        return
+                cb(remade=False)
+
+        mlist = []
         for f in self.included:
             t = self.gettarget(f)
             t.explicit = True
             t.resolvevpath(self)
             oldmtime = t.mtime
-            t.make(self, [], [], required=False)
-            if t.mtime != oldmtime:
-                log.info("included makefile '%s' was remade" % t.target)
-                reparse = True
 
-        return reparse
+            mlist.append((t, oldmtime))
+            t.make(self, [], [], required=False, avoidremakeloop=True, cb=remakecb)
 
     flagescape = re.compile(r'([\s\\])')
 
     def getsubenvironment(self, variables):
-        env = dict(os.environ)
+        env = dict(self.env)
         for vname in self.exportedvars:
             flavor, source, val = variables.get(vname)
             if val is None:
                 strval = ''
             else:
-                strval = val.resolve(variables, [vname])
+                strval = val.resolve(self, variables, [vname])
             env[vname] = strval
 
         makeflags = ''
 
         flavor, source, val = variables.get('MAKEFLAGS')
         if val is not None:
-            flagsval = val.resolve(variables, ['MAKEFLAGS'])
+            flagsval = val.resolve(self, variables, ['MAKEFLAGS'])
             if flagsval != '':
                 makeflags = flagsval
 
         makeflags += ' -- '
         makeflags += ' '.join((self.flagescape.sub(r'\\\1', o) for o in self.overrides))
 
         env['MAKEFLAGS'] = makeflags
 
--- a/pymake/functions.py
+++ b/pymake/functions.py
@@ -1,26 +1,29 @@
 """
 Makefile functions.
 """
 
-from pymake import data
-import subprocess, os, glob
+import parser
+import data
+import subprocess, os, logging
+from pymake.globrelative import glob
+from cStringIO import StringIO
 
-log = data.log
+log = logging.getLogger('pymake.data')
 
 class Function(object):
     """
     An object that represents a function call. This class is always subclassed
     with the following methods and attributes:
 
     minargs = minimum # of arguments
     maxargs = maximum # of arguments (0 means unlimited)
 
-    def resolve(self, variables, setting)
+    def resolve(self, makefile, variables, setting)
         Calls the function
         @returns string
     """
     def __init__(self, loc):
         self._arguments = []
         self.loc = loc
         assert self.minargs > 0
 
@@ -46,213 +49,213 @@ class VariableRef(Function):
     def __init__(self, loc, vname):
         self.loc = loc
         assert isinstance(vname, data.Expansion)
         self.vname = vname
         
     def setup(self):
         assert False, "Shouldn't get here"
 
-    def resolve(self, variables, setting):
-        vname = self.vname.resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        vname = self.vname.resolve(makefile, variables, setting)
         if vname in setting:
             raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)
 
         flavor, source, value = variables.get(vname)
         if value is None:
             log.debug("%s: variable '%s' was not set" % (self.loc, vname))
             return ''
 
-        return value.resolve(variables, setting + [vname])
+        return value.resolve(makefile, variables, setting + [vname])
 
 class SubstitutionRef(Function):
     """$(VARNAME:.c=.o) and $(VARNAME:%.c=%.o)"""
     def __init__(self, loc, varname, substfrom, substto):
         self.loc = loc
         self.vname = varname
         self.substfrom = substfrom
         self.substto = substto
 
     def setup(self):
         assert False, "Shouldn't get here"
 
-    def resolve(self, variables, setting):
-        vname = self.vname.resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        vname = self.vname.resolve(makefile, variables, setting)
         if vname in setting:
             raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)
 
-        substfrom = self.substfrom.resolve(variables, setting)
-        substto = self.substto.resolve(variables, setting)
+        substfrom = self.substfrom.resolve(makefile, variables, setting)
+        substto = self.substto.resolve(makefile, variables, setting)
 
         flavor, source, value = variables.get(vname)
         if value is None:
             log.debug("%s: variable '%s' was not set" % (self.loc, vname))
             return ''
 
-        evalue = value.resolve(variables, setting + [vname])
+        evalue = value.resolve(makefile, variables, setting + [vname])
         words = data.splitwords(evalue)
 
         f = data.Pattern(substfrom)
         if not f.ispattern():
             f = data.Pattern('%' + substfrom)
             substto = '%' + substto
 
         return " ".join((f.subst(substto, word, False)
                          for word in words))
 
 class SubstFunction(Function):
     name = 'subst'
     minargs = 3
     maxargs = 3
 
-    def resolve(self, variables, setting):
-        s = self._arguments[0].resolve(variables, setting)
-        r = self._arguments[1].resolve(variables, setting)
-        d = self._arguments[2].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        s = self._arguments[0].resolve(makefile, variables, setting)
+        r = self._arguments[1].resolve(makefile, variables, setting)
+        d = self._arguments[2].resolve(makefile, variables, setting)
         return d.replace(s, r)
 
 class PatSubstFunction(Function):
     name = 'patsubst'
     minargs = 3
     maxargs = 3
 
-    def resolve(self, variables, setting):
-        s = self._arguments[0].resolve(variables, setting)
-        r = self._arguments[1].resolve(variables, setting)
-        d = self._arguments[2].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        s = self._arguments[0].resolve(makefile, variables, setting)
+        r = self._arguments[1].resolve(makefile, variables, setting)
+        d = self._arguments[2].resolve(makefile, variables, setting)
 
         p = data.Pattern(s)
         return ' '.join((p.subst(r, word, False)
                          for word in data.splitwords(d)))
 
 class StripFunction(Function):
     name = 'strip'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        return ' '.join(data.splitwords(self._arguments[0].resolve(variables, setting)))
+    def resolve(self, makefile, variables, setting):
+        return ' '.join(data.splitwords(self._arguments[0].resolve(makefile, variables, setting)))
 
 class FindstringFunction(Function):
     name = 'findstring'
     minargs = 2
     maxargs = 2
 
-    def resolve(self, variables, setting):
-        s = self._arguments[0].resolve(variables, setting)
-        r = self._arguments[1].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        s = self._arguments[0].resolve(makefile, variables, setting)
+        r = self._arguments[1].resolve(makefile, variables, setting)
         if r.find(s) == -1:
             return ''
         return s
 
 class FilterFunction(Function):
     name = 'filter'
     minargs = 2
     maxargs = 2
 
-    def resolve(self, variables, setting):
-        ps = self._arguments[0].resolve(variables, setting)
-        d = self._arguments[1].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        ps = self._arguments[0].resolve(makefile, variables, setting)
+        d = self._arguments[1].resolve(makefile, variables, setting)
         plist = [data.Pattern(p) for p in data.splitwords(ps)]
         r = []
         for w in data.splitwords(d):
             if any((p.match(w) for p in plist)):
                     r.append(w)
                 
         return ' '.join(r)
 
 class FilteroutFunction(Function):
     name = 'filter-out'
     minargs = 2
     maxargs = 2
 
-    def resolve(self, variables, setting):
-        ps = self._arguments[0].resolve(variables, setting)
-        d = self._arguments[1].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        ps = self._arguments[0].resolve(makefile, variables, setting)
+        d = self._arguments[1].resolve(makefile, variables, setting)
         plist = [data.Pattern(p) for p in data.splitwords(ps)]
         r = []
         for w in data.splitwords(d):
             found = False
             if not any((p.match(w) for p in plist)):
                 r.append(w)
 
         return ' '.join(r)
 
 class SortFunction(Function):
     name = 'sort'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        d = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        d = self._arguments[0].resolve(makefile, variables, setting)
         w = data.splitwords(d)
         w.sort()
         return ' '.join((w for w in data.withoutdups(w)))
 
 class WordFunction(Function):
     name = 'word'
     minargs = 2
     maxargs = 2
 
-    def resolve(self, variables, setting):
-        n = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        n = self._arguments[0].resolve(makefile, variables, setting)
         # TODO: provide better error if this doesn't convert
         n = int(n)
-        words = data.splitwords(self._arguments[1].resolve(variables, setting))
+        words = data.splitwords(self._arguments[1].resolve(makefile, variables, setting))
         if n < 1 or n > len(words):
             return ''
         return words[n - 1]
 
 class WordlistFunction(Function):
     name = 'wordlist'
     minargs = 3
     maxargs = 3
 
-    def resolve(self, variables, setting):
-        nfrom = self._arguments[0].resolve(variables, setting)
-        nto = self._arguments[1].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        nfrom = self._arguments[0].resolve(makefile, variables, setting)
+        nto = self._arguments[1].resolve(makefile, variables, setting)
         # TODO: provide better errors if this doesn't convert
         nfrom = int(nfrom)
         nto = int(nto)
 
-        words = data.splitwords(self._arguments[2].resolve(variables, setting))
+        words = data.splitwords(self._arguments[2].resolve(makefile, variables, setting))
 
         if nfrom < 1:
             nfrom = 1
         if nto < 1:
             nto = 1
 
         return ' '.join(words[nfrom - 1:nto])
 
 class WordsFunction(Function):
     name = 'words'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        return str(len(data.splitwords(self._arguments[0].resolve(variables, setting))))
+    def resolve(self, makefile, variables, setting):
+        return str(len(data.splitwords(self._arguments[0].resolve(makefile, variables, setting))))
 
 class FirstWordFunction(Function):
     name = 'firstword'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        wl = data.splitwords(self._arguments[0].resolve(variables, setting))
+    def resolve(self, makefile, variables, setting):
+        wl = data.splitwords(self._arguments[0].resolve(makefile, variables, setting))
         if len(wl) == 0:
             return ''
         return wl[0]
 
 class LastWordFunction(Function):
     name = 'lastword'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        wl = data.splitwords(self._arguments[0].resolve(variables, setting))
+    def resolve(self, makefile, variables, setting):
+        wl = data.splitwords(self._arguments[0].resolve(makefile, variables, setting))
         if len(wl) == 0:
             return ''
         return wl[0]
 
 def pathsplit(path, default='./'):
     """
     Splits a path into dirpart, filepart on the last slash. If there is no slash, dirpart
     is ./
@@ -263,254 +266,267 @@ def pathsplit(path, default='./'):
 
     return dir + slash, file
 
 class DirFunction(Function):
     name = 'dir'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
+    def resolve(self, makefile, variables, setting):
         return ' '.join((pathsplit(path)[0]
-                         for path in data.splitwords(self._arguments[0].resolve(variables, setting))))
+                         for path in data.splitwords(self._arguments[0].resolve(makefile, variables, setting))))
 
 class NotDirFunction(Function):
     name = 'notdir'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
+    def resolve(self, makefile, variables, setting):
         return ' '.join((pathsplit(path)[1]
-                         for path in data.splitwords(self._arguments[0].resolve(variables, setting))))
+                         for path in data.splitwords(self._arguments[0].resolve(makefile, variables, setting))))
 
 class SuffixFunction(Function):
     name = 'suffix'
     minargs = 1
     maxargs = 1
 
     @staticmethod
     def suffixes(words):
         for w in words:
             dir, file = pathsplit(w)
             base, dot, suffix = file.rpartition('.')
             if base != '':
                 yield dot + suffix
 
-    def resolve(self, variables, setting):
-        return ' '.join(self.suffixes(data.splitwords(self._arguments[0].resolve(variables, setting))))
+    def resolve(self, makefile, variables, setting):
+        return ' '.join(self.suffixes(data.splitwords(self._arguments[0].resolve(makefile, variables, setting))))
 
 class BasenameFunction(Function):
     name = 'basename'
     minargs = 1
     maxargs = 1
 
     @staticmethod
     def basenames(words):
         for w in words:
             dir, file = pathsplit(w, '')
             base, dot, suffix = file.rpartition('.')
             if dot == '':
                 base = suffix
 
             yield dir + base
 
-    def resolve(self, variables, setting):
-        return ' '.join(self.basenames(data.splitwords(self._arguments[0].resolve(variables, setting))))
+    def resolve(self, makefile, variables, setting):
+        return ' '.join(self.basenames(data.splitwords(self._arguments[0].resolve(makefile, variables, setting))))
 
 class AddSuffixFunction(Function):
     name = 'addprefix'
     minargs = 2
     maxargs = 2
 
-    def resolve(self, variables, setting):
-        suffix = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        suffix = self._arguments[0].resolve(makefile, variables, setting)
 
-        return ' '.join((w + suffix for w in data.splitwords(self._arguments[1].resolve(variables, setting))))
+        return ' '.join((w + suffix for w in data.splitwords(self._arguments[1].resolve(makefile, variables, setting))))
 
 class AddPrefixFunction(Function):
     name = 'addsuffix'
     minargs = 2
     maxargs = 2
 
-    def resolve(self, variables, setting):
-        prefix = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        prefix = self._arguments[0].resolve(makefile, variables, setting)
 
-        return ' '.join((prefix + w for w in data.splitwords(self._arguments[1].resolve(variables, setting))))
+        return ' '.join((prefix + w for w in data.splitwords(self._arguments[1].resolve(makefile, variables, setting))))
 
 class JoinFunction(Function):
     name = 'join'
     minargs = 2
     maxargs = 2
 
     @staticmethod
     def iterjoin(l1, l2):
         for i in xrange(0, max(len(l1), len(l2))):
             i1 = i < len(l1) and l1[i] or ''
             i2 = i < len(l2) and l2[i] or ''
             yield i1 + i2
 
-    def resolve(self, variables, setting):
-        list1 = data.splitwords(self._arguments[0].resolve(variables, setting))
-        list2 = data.splitwords(self._arguments[1].resolve(variables, setting))
+    def resolve(self, makefile, variables, setting):
+        list1 = data.splitwords(self._arguments[0].resolve(makefile, variables, setting))
+        list2 = data.splitwords(self._arguments[1].resolve(makefile, variables, setting))
 
         return ' '.join(self.iterjoin(list1, list2))
 
 class WildcardFunction(Function):
     name = 'wildcard'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
+    def resolve(self, makefile, variables, setting):
         # TODO: will need work when we support -C without actually changing the OS cwd
-        pattern = self._arguments[0].resolve(variables, setting)
-        return ' '.join([x.replace('\\','/') for x in glob.glob(pattern)])
+        patterns = data.splitwords(self._arguments[0].resolve(makefile, variables, setting))
+
+        r = []
+        for p in patterns:
+            r.extend([x.replace('\\','/') for x in glob(makefile.workdir, p)])
+        return ' '.join(r)
 
 class RealpathFunction(Function):
     name = 'realpath'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        # TODO: will need work when we support -C without actually changing the OS cwd
-        return ' '.join((os.path.realpath(f).replace('\\','/')
-                         for f in data.splitwords(self._arguments[0].resolve(variables, setting))))
+    def resolve(self, makefile, variables, setting):
+        paths = data.splitwords(self._arguments[0].resolve(makefile, variables, setting))
+        fspaths = [os.path.join(makefile.workdir, path) for path in paths]
+        realpaths = [os.path.realpath(path).replace('\\','/') for path in fspaths]
+        return ' '.join(realpaths)
 
 class AbspathFunction(Function):
     name = 'abspath'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        # TODO: will need work when we support -C without actually changing the OS cwd
-        return ' '.join((os.path.abspath(f).replace('\\','/')
-                         for f in data.splitwords(self._arguments[0].resolve(variables, setting))))
+    def resolve(self, makefile, variables, setting):
+        assert os.path.isabs(makefile.workdir)
+        paths = data.splitwords(self._arguments[0].resolve(makefile, variables, setting))
+        fspaths = [os.path.join(makefile.workdir, path).replace('\\','/') for path in paths]
+        return ' '.join(fspaths)
 
 class IfFunction(Function):
     name = 'if'
     minargs = 1
     maxargs = 3
 
     def setup(self):
         Function.setup(self)
         self._arguments[0].lstrip()
         self._arguments[0].rstrip()
 
-    def resolve(self, variables, setting):
-        condition = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        condition = self._arguments[0].resolve(makefile, variables, setting)
         if len(condition):
-            return self._arguments[1].resolve(variables, setting)
+            return self._arguments[1].resolve(makefile, variables, setting)
 
         if len(self._arguments) > 2:
-            return self._arguments[2].resolve(variables, setting)
+            return self._arguments[2].resolve(makefile, variables, setting)
 
         return ''
 
 class OrFunction(Function):
     name = 'or'
     minargs = 1
     maxargs = 0
 
-    def resolve(self, variables, setting):
+    def resolve(self, makefile, variables, setting):
         for arg in self._arguments:
-            r = arg.resolve(variables, setting)
+            r = arg.resolve(makefile, variables, setting)
             if r != '':
                 return r
 
         return ''
 
 class AndFunction(Function):
     name = 'and'
     minargs = 1
     maxargs = 0
 
-    def resolve(self, variables, setting):
+    def resolve(self, makefile, variables, setting):
         r = ''
 
         for arg in self._arguments:
-            r = arg.resolve(variables, setting)
+            r = arg.resolve(makefile, variables, setting)
             if r == '':
                 return ''
 
         return r
 
 class ForEachFunction(Function):
     name = 'foreach'
     minargs = 3
     maxargs = 3
 
-    def resolve(self, variables, setting):
-        vname = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        vname = self._arguments[0].resolve(makefile, variables, setting)
 
-        words = data.splitwords(self._arguments[1].resolve(variables, setting))
+        words = data.splitwords(self._arguments[1].resolve(makefile, variables, setting))
         e = self._arguments[2]
 
         results = []
 
         v = data.Variables(parent=variables)
         for w in words:
             v.set(vname, data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, w)
-            results.append(e.resolve(v, setting))
+            results.append(e.resolve(makefile, v, setting))
 
         return ' '.join(results)
 
 class CallFunction(Function):
     name = 'call'
     minargs = 1
     maxargs = 0
 
-    def resolve(self, variables, setting):
-        vname = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        vname = self._arguments[0].resolve(makefile, variables, setting)
         if vname in setting:
             raise data.DataError("Recursively setting variable '%s'" % (vname,))
 
         v = data.Variables(parent=variables)
         v.set('0', data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, vname)
         for i in xrange(1, len(self._arguments)):
-            param = self._arguments[i].resolve(variables, setting)
+            param = self._arguments[i].resolve(makefile, variables, setting)
             v.set(str(i), data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, param)
 
         flavor, source, e = variables.get(vname)
         if e is None:
             return ''
 
         if flavor == data.Variables.FLAVOR_SIMPLE:
             log.warning("%s: calling variable '%s' which is simply-expanded" % (self.loc, vname))
 
         # but we'll do it anyway
-        return e.resolve(v, setting + [vname])
+        return e.resolve(makefile, v, setting + [vname])
 
 class ValueFunction(Function):
     name = 'value'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        varname = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        varname = self._arguments[0].resolve(makefile, variables, setting)
 
         flavor, source, value = variables.get(varname, expand=False)
         if value is None:
             return ''
 
         return value
 
 class EvalFunction(Function):
     name = 'eval'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        raise NotImplementedError('no eval yet')
+    def resolve(self, makefile, variables, setting):
+        if makefile.parsingfinished:
+            # GNU make allows variables to be set by recursive expansion during
+            # command execution. This seems really dumb to me, so I don't!
+            raise data.DataError("$(eval) not allowed via recursive expansion after parsing is finished", self.loc)
+
+        text = StringIO(self._arguments[0].resolve(makefile, variables, setting))
+        parser.parsestream(text, 'evaluation from %s' % self.loc, makefile)
+        return ''
 
 class OriginFunction(Function):
     name = 'origin'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        vname = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        vname = self._arguments[0].resolve(makefile, variables, setting)
 
         flavor, source, value = variables.get(vname)
         if source is None:
             return 'undefined'
 
         if source == data.Variables.SOURCE_OVERRIDE:
             return 'override'
 
@@ -528,18 +544,18 @@ class OriginFunction(Function):
 
         assert False, "Unexpected source value: %s" % source
 
 class FlavorFunction(Function):
     name = 'flavor'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        varname = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        varname = self._arguments[0].resolve(makefile, variables, setting)
         
         flavor, source, value = variables.get(varname)
         if flavor is None:
             return 'undefined'
 
         if flavor == data.Variables.FLAVOR_RECURSIVE:
             return 'recursive'
         elif flavor == data.Variables.FLAVOR_SIMPLE:
@@ -547,58 +563,59 @@ class FlavorFunction(Function):
 
         assert False, "Neither simple nor recursive?"
 
 class ShellFunction(Function):
     name = 'shell'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
+    def resolve(self, makefile, variables, setting):
         shell, prependshell = data.checkmsyscompat()
-        cline = self._arguments[0].resolve(variables, setting)
+        cline = self._arguments[0].resolve(makefile, variables, setting)
 
+        log.debug("%s: running shell command '%s'" % (self.loc, cline))
         if prependshell:
             cline = [shell, "-c", cline]
-        p = subprocess.Popen(cline, shell=not prependshell, stdout=subprocess.PIPE)
+        p = subprocess.Popen(cline, shell=not prependshell, stdout=subprocess.PIPE, cwd=makefile.workdir)
         stdout, stderr = p.communicate()
 
         stdout = stdout.replace('\r\n', '\n')
         if len(stdout) > 1 and stdout[-1] == '\n':
             stdout = stdout[:-1]
         stdout = stdout.replace('\n', ' ')
 
         return stdout
 
 class ErrorFunction(Function):
     name = 'error'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        v = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        v = self._arguments[0].resolve(makefile, variables, setting)
         raise data.DataError(v, self.loc)
 
 class WarningFunction(Function):
     name = 'warning'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        v = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        v = self._arguments[0].resolve(makefile, variables, setting)
         log.warning(v)
         return ''
 
 class InfoFunction(Function):
     name = 'info'
     minargs = 1
     maxargs = 1
 
-    def resolve(self, variables, setting):
-        v = self._arguments[0].resolve(variables, setting)
+    def resolve(self, makefile, variables, setting):
+        v = self._arguments[0].resolve(makefile, variables, setting)
         log.info(v)
         return ''
 
 functionmap = {
     'subst': SubstFunction,
     'patsubst': PatSubstFunction,
     'strip': StripFunction,
     'findstring': FindstringFunction,
new file mode 100644
--- /dev/null
+++ b/pymake/globrelative.py
@@ -0,0 +1,67 @@
+"""
+Filename globbing like the python glob module with minor differences:
+
+* glob relative to an arbitrary directory
+* include . and ..
+* check that link targets exist, not just links
+"""
+
+import os, re, fnmatch
+
+_globcheck = re.compile('[[*?]')
+
+def hasglob(p):
+    return _globcheck.search(p) is not None
+
+def glob(fsdir, path):
+    """
+    Yield paths matching the path glob. Sorts as a bonus. Excludes '.' and '..'
+    """
+
+    dir, leaf = os.path.split(path)
+    if dir == '':
+        return globpattern(fsdir, leaf)
+
+    if hasglob(dir):
+        dirsfound = glob(fsdir, dir)
+    else:
+        dirsfound = [dir]
+
+    r = []
+
+    for dir in dirsfound:
+        fspath = os.path.join(fsdir, dir)
+        if not os.path.isdir(fspath):
+            continue
+
+        r.extend((os.path.join(dir, found) for found in globpattern(fspath, leaf)))
+
+    return r
+
+def globpattern(dir, pattern):
+    """
+    Return leaf names in the specified directory which match the pattern.
+    """
+
+    if not hasglob(pattern):
+        if pattern == '':
+            if os.path.isdir(dir):
+                return ['']
+            return []
+
+        if os.path.exists(os.path.join(dir, pattern)):
+            return [pattern]
+        return []
+
+    leaves = os.listdir(dir) + ['.', '..']
+
+    # "hidden" filenames are a bit special
+    if not pattern.startswith('.'):
+        leaves = [leaf for leaf in leaves
+                  if not leaf.startswith('.')]
+
+    leaves = fnmatch.filter(leaves, pattern)
+    leaves = filter(lambda l: os.path.exists(os.path.join(dir, l)), leaves)
+
+    leaves.sort()
+    return leaves
--- a/pymake/parser.py
+++ b/pymake/parser.py
@@ -14,30 +14,26 @@ Lines with command syntax do not condens
 Lines with an initial tab are commands if they can be (there is a rule or a command immediately preceding).
 Otherwise, they are parsed as makefile syntax.
 
 After splitting data into parseable chunks, we use a recursive-descent parser to
 nest parenthesized syntax.
 """
 
 import logging, re
-from pymake import data, functions
+import data, functions, util
+from pymake.globrelative import hasglob, glob
 from cStringIO import StringIO
 
 tabwidth = 4
 
 log = logging.getLogger('pymake.parser')
 
-class SyntaxError(Exception):
-    def __init__(self, message, loc):
-        self.message = message
-        self.loc = loc
-
-    def __str__(self):
-        return "%s: %s" % (self.loc, self.message)
+class SyntaxError(util.MakeError):
+    pass
 
 class Location(object):
     """
     A location within a makefile.
 
     For the moment, locations are just path/line/column, but in the future
     they may reference parent locations for more accurate "included from"
     or "evaled at" error reporting.
@@ -391,35 +387,38 @@ def iterlines(fd):
     for line in fd:
         lineno += 1
 
         if line.endswith('\r\n'):
             line = line[:-2] + '\n'
 
         yield (lineno, line)
 
-def setvariable(resolvevariables, setvariables, vname, token, d, offset,
-                iterfunc=itermakefilechars, source=data.Variables.SOURCE_MAKEFILE,
+def setvariable(resolvevariables, setvariables, makefile, vname, token, d, offset,
+                iterfunc=itermakefilechars, source=None,
                 skipwhitespace=True):
     """
     Parse what's left in a data iterator di into a variable.
     """
     assert isinstance(resolvevariables, data.Variables)
     assert isinstance(setvariables, data.Variables)
 
+    if source is None:
+        source = data.Variables.SOURCE_MAKEFILE
+
     # print "setvariable: %r resvariables: %r setvariables: %r" % (vname, resolvevariables, setvariables)
 
     if len(vname) == 0:
         raise SyntaxError("Empty variable name", loc=d.getloc(offset))
 
     if token == '+=':
         val = ''.join((c for c, t, o, oo in iterfunc(d, offset, emptytokenlist)))
         if skipwhitespace:
             val = val.lstrip()
-        setvariables.append(vname, source, val, resolvevariables)
+        setvariables.append(vname, source, val, resolvevariables, makefile)
         return
 
     if token == '?=':
         flavor = data.Variables.FLAVOR_RECURSIVE
         val = ''.join((c for c, t, o, oo in iterfunc(d, offset, emptytokenlist)))
         if skipwhitespace:
             val = val.lstrip()
         oldflavor, oldsource, oldval = setvariables.get(vname, expand=False)
@@ -432,17 +431,17 @@ def setvariable(resolvevariables, setvar
             val = val.lstrip()
     else:
         assert token == ':='
 
         flavor = data.Variables.FLAVOR_SIMPLE
         e, t, o = parsemakesyntax(d, offset, (), itermakefilechars)
         if skipwhitespace:
             e.lstrip()
-        val = e.resolve(resolvevariables)
+        val = e.resolve(makefile, resolvevariables)
         
     setvariables.set(vname, flavor, source, val)
 
 def parsecommandlineargs(makefile, args):
     """
     Given a set of arguments from a command-line invocation of make,
     parse out the variable definitions and return the rest as targets.
     """
@@ -456,17 +455,17 @@ def parsecommandlineargs(makefile, args)
             vname, t, val = a.partition('=')
         if t != '':
             makefile.overrides.append(a)
 
             vname = vname.strip()
             d = Data(None, None)
             d.append(val, Location('<command-line>', i, len(vname) + len(t)))
 
-            setvariable(makefile.variables, makefile.variables,
+            setvariable(makefile.variables, makefile.variables, makefile,
                         vname, t, d, 0, source=data.Variables.SOURCE_COMMANDLINE,
                         iterfunc=iterdata)
         else:
             r.append(a)
 
     return r
 
 eqargstokenlist = TokenList.get(('(', "'", '"'))
@@ -502,29 +501,29 @@ def ifeq(d, offset, makefile):
         token = d[offset]
         if token not in '\'"':
             raise SyntaxError("Unexpected text in conditional", d.getloc(offset))
 
         arg2, t, offset = parsemakesyntax(d, offset + 1, (token,), itermakefilechars)
 
         ensureend(d, offset, "Unexpected text after conditional")
 
-    val1 = arg1.resolve(makefile.variables)
-    val2 = arg2.resolve(makefile.variables)
+    val1 = arg1.resolve(makefile, makefile.variables)
+    val2 = arg2.resolve(makefile, makefile.variables)
 
     return val1 == val2
 
 def ifneq(d, offset, makefile):
     return not ifeq(d, offset, makefile)
 
 def ifdef(d, offset, makefile):
     e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
     e.rstrip()
 
-    vname = e.resolve(makefile.variables)
+    vname = e.resolve(makefile, makefile.variables)
 
     flavor, source, value = makefile.variables.get(vname, expand=False)
 
     if value is None:
         return False
 
     # We aren't expanding the variable... we're just seeing if it was set to a non-empty
     # expansion.
@@ -554,16 +553,25 @@ class Condition(object):
         if self.everactive:
             self.active = False
             return
 
         self.active = active
         if active:
             self.everactive = True
 
+def expandwildcards(makefile, tlist):
+    for t in tlist:
+        if not hasglob(t):
+            yield t
+        else:
+            l = glob(makefile.workdir, t)
+            for r in l:
+                yield r
+
 conditiontokens = tuple(conditionkeywords.iterkeys())
 directivestokenlist = TokenList.get(conditiontokens + \
     ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'vpath', 'export', 'unexport'))
 conditionkeywordstokenlist = TokenList.get(conditiontokens)
 
 varsettokens = (':=', '+=', '?=', '=')
 
 def parsestream(fd, filename, makefile):
@@ -616,24 +624,32 @@ def parsestream(fd, filename, makefile):
                 kword, offset = d.findtoken(offset, conditionkeywordstokenlist, True)
                 if kword is None:
                     ensureend(d, offset, "Unexpected data after 'else' directive.")
                     condstack[-1].makeactive(True)
                 else:
                     if kword not in conditionkeywords:
                         raise SyntaxError("Unexpected condition after 'else' directive.",
                                           d.getloc(offset))
-                        
-                    m = conditionkeywords[kword](d, offset, makefile)
-                    condstack[-1].makeactive(m)
+
+                    if any ((not c.active for c in condstack[:-1])):
+                        pass
+                    else:
+                        m = conditionkeywords[kword](d, offset, makefile)
+                        condstack[-1].makeactive(m)
                 continue
 
             if kword in conditionkeywords:
-                m = conditionkeywords[kword](d, offset, makefile)
-                condstack.append(Condition(m, d.getloc(offset)))
+                if any((not c.active for c in condstack)):
+                    # If any conditions are currently false, we don't evaluate anything: just stick a dummy
+                    # condition on the stack
+                    condstack.append(Condition(True, d.getloc(offset)))
+                else:
+                    m = conditionkeywords[kword](d, offset, makefile)
+                    condstack.append(Condition(m, d.getloc(offset)))
                 continue
 
             if any((not c.active for c in condstack)):
                 log.debug('%s: skipping line because of active conditions' % (d.getloc(0),))
                 for c in itermakefilechars(d, offset, emptytokenlist):
                     pass
                 continue
 
@@ -641,111 +657,133 @@ def parsestream(fd, filename, makefile):
                 raise SyntaxError("Unmatched endef", d.getloc(offset))
 
             if kword == 'define':
                 e, t, i = parsemakesyntax(d, offset, (), itermakefilechars)
 
                 d = Data(fdlines, filename)
                 d.readline()
 
-                setvariable(makefile.variables, makefile.variables,
-                            e.resolve(makefile.variables),
+                setvariable(makefile.variables, makefile.variables, makefile,
+                            e.resolve(makefile, makefile.variables),
                             '=', d, 0, iterdefinechars,
                             skipwhitespace=False)
 
                 continue
 
             if kword in ('include', '-include'):
                 incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
-                files = data.splitwords(incfile.resolve(makefile.variables))
+                files = data.splitwords(incfile.resolve(makefile, makefile.variables))
                 for f in files:
                     makefile.include(f.replace('\\','/'),
                                      kword == 'include', loc=d.getloc(offset))
                 continue
 
+            if kword == 'vpath':
+                e, t, offset = parsemakesyntax(d, offset, (' ', '\t'), itermakefilechars)
+                patstr = e.resolve(makefile, makefile.variables)
+                pattern = data.Pattern(patstr)
+                if t is None:
+                    makefile.clearallvpaths()
+                else:
+                    e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
+                    dirs = e.resolve(makefile, makefile.variables)
+                    dirlist = []
+                    for direl in data.splitwords(dirs):
+                        dirlist.extend((dir
+                                        for dir in direl.split(':')
+                                        if dir != ''))
+
+                    if len(dirlist) == 0:
+                        makefile.clearvpath(pattern)
+                    else:
+                        makefile.addvpath(pattern, dirlist)
+                continue
+
             if kword == 'override':
                 e, token, offset = parsemakesyntax(d, offset, varsettokens, itermakefilechars)
                 e.lstrip()
                 e.rstrip()
 
                 if token is None:
                     raise SyntaxError("Malformed override directive, need =", d.getloc(offset))
 
-                vname = e.resolve(makefile.variables)
-                setvariable(makefile.variables, makefile.variables,
+                vname = e.resolve(makefile, makefile.variables)
+                setvariable(makefile.variables, makefile.variables, makefile,
                             vname, token, d, offset,
                             source=data.Variables.SOURCE_OVERRIDE)
                 continue
 
             if kword == 'export':
                 e, token, offset = parsemakesyntax(d, offset, varsettokens, itermakefilechars)
                 e.lstrip()
                 e.rstrip()
-                vars = e.resolve(makefile.variables)
+                vars = e.resolve(makefile, makefile.variables)
                 if token is None:
                     vlist = data.splitwords(vars)
                     if len(vlist) == 0:
                         raise SyntaxError("Exporting all variables is not supported", d.getloc(offset))
                 else:
                     vlist = [vars]
-                    setvariable(makefile.variables, makefile.variables,
+                    setvariable(makefile.variables, makefile.variables, makefile,
                                 vars, token, d, offset)
 
                 for v in vlist:
                     makefile.exportedvars.add(v)
 
                 continue
 
             if kword == 'unexport':
                 raise SyntaxError("unexporting variables is not supported", d.getloc(offset))
 
             assert kword is None, "unexpected kword: %r" % (kword,)
 
             e, token, offset = parsemakesyntax(d, offset, varsettokens + ('::', ':'), itermakefilechars)
             if token is None:
-                v = e.resolve(makefile.variables)
+                v = e.resolve(makefile, makefile.variables)
                 if v.strip() != '':
                     raise SyntaxError("Bad syntax: non-empty line is not a variable assignment or rule.", loc=d.getloc(0))
                 continue
 
             # if we encountered real makefile syntax, the current rule is over
             currule = None
 
             if token in varsettokens:
                 e.lstrip()
                 e.rstrip()
-                vname = e.resolve(makefile.variables)
-                setvariable(makefile.variables, makefile.variables,
+                vname = e.resolve(makefile, makefile.variables)
+                setvariable(makefile.variables, makefile.variables, makefile,
                             vname, token, d, offset)
             else:
                 doublecolon = token == '::'
 
                 # `e` is targets or target patterns, which can end up as
                 # * a rule
                 # * an implicit rule
                 # * a static pattern rule
                 # * a target-specific variable definition
                 # * a pattern-specific variable definition
                 # any of the rules may have order-only prerequisites
                 # delimited by |, and a command delimited by ;
-                targets = map(data.Pattern, data.splitwords(e.resolve(makefile.variables)))
+                targets = data.splitwords(e.resolve(makefile, makefile.variables))
+                targets = [data.Pattern(p) for p in expandwildcards(makefile, targets)]
 
                 if len(targets):
                     ispatterns = set((t.ispattern() for t in targets))
                     if len(ispatterns) == 2:
                         raise SyntaxError("Mixed implicit and normal rule", d.getloc(offset))
                     ispattern, = ispatterns
                 else:
                     ispattern = False
 
                 e, token, offset = parsemakesyntax(d, offset,
                                                    varsettokens + (':', '|', ';'),
                                                    itermakefilechars)
                 if token in (None, ';'):
-                    prereqs = data.splitwords(e.resolve(makefile.variables))
+                    prereqs = [p for p in expandwildcards(makefile, data.splitwords(e.resolve(makefile, makefile.variables)))]
                     if ispattern:
                         currule = data.PatternRule(targets, map(data.Pattern, prereqs), doublecolon, loc=d.getloc(0))
                         makefile.appendimplicitrule(currule)
                     else:
                         currule = data.Rule(prereqs, doublecolon, loc=d.getloc(0))
                         for t in targets:
                             makefile.gettarget(t.gettarget()).addrule(currule)
                         if len(targets):
@@ -753,45 +791,45 @@ def parsestream(fd, filename, makefile):
 
                     if token == ';':
                         offset = d.skipwhitespace(offset)
                         e, t, offset = parsemakesyntax(d, offset, (), itercommandchars)
                         currule.addcommand(e)
                 elif token in varsettokens:
                     e.lstrip()
                     e.rstrip()
-                    vname = e.resolve(makefile.variables)
+                    vname = e.resolve(makefile, makefile.variables)
                     if ispattern:
                         for target in targets:
                             setvariable(makefile.variables,
-                                        makefile.getpatternvariables(target), vname,
+                                        makefile.getpatternvariables(target), makefile, vname,
                                         token, d, offset)
                     else:
                         for target in targets:
                             setvariable(makefile.variables,
-                                        makefile.gettarget(target.gettarget()).variables,
+                                        makefile.gettarget(target.gettarget()).variables, makefile,
                                         vname, token, d, offset)
                 elif token == '|':
                     raise NotImplementedError('order-only prerequisites not implemented')
                 else:
                     assert token == ':'
 
                     # static pattern rule
                     if ispattern:
                         raise SyntaxError("static pattern rules must have static targets", d.getloc(0))
 
-                    patstr = e.resolve(makefile.variables)
+                    patstr = e.resolve(makefile, makefile.variables)
                     patterns = data.splitwords(patstr)
                     if len(patterns) != 1:
                         raise SyntaxError("A static pattern rule may have only one pattern", d.getloc(offset))
 
                     pattern = data.Pattern(patterns[0])
 
                     e, token, offset = parsemakesyntax(d, offset, (';',), itermakefilechars)
-                    prereqs = map(data.Pattern, data.splitwords(e.resolve(makefile.variables)))
+                    prereqs = [data.Pattern(p) for p in expandwildcards(makefile, data.splitwords(e.resolve(makefile, makefile.variables)))]
                     currule = data.PatternRule([pattern], prereqs, doublecolon, loc=d.getloc(0))
 
                     for t in targets:
                         tname = t.gettarget()
                         stem = pattern.match(tname)
                         if stem is None:
                             raise SyntaxError("Target '%s' of static pattern rule does not match pattern '%s'" % (tname, pattern), d.getloc(0))
                         pinstance = data.PatternRuleInstance(currule, '', stem, pattern.ismatchany())
new file mode 100644
--- /dev/null
+++ b/pymake/process.py
@@ -0,0 +1,151 @@
+"""
+Skipping shell invocations is good, when possible. This wrapper around subprocess does dirty work of
+parsing command lines into argv and making sure that no shell magic is being used.
+"""
+
+import subprocess, shlex, re, logging, sys, traceback, os
+import command
+
+_log = logging.getLogger('pymake.process')
+
+blacklist = re.compile(r'[=\\$><;*?[{~`|&]')
+def clinetoargv(cline):
+    """
+    If this command line can safely skip the shell, return an argv array.
+    """
+
+    if blacklist.search(cline) is not None:
+        return None
+
+    return shlex.split(cline, comments=True)
+
+shellwords = (':', '.', 'break', 'cd', 'continue', 'exec', 'exit', 'export',
+              'getopts', 'hash', 'pwd', 'readonly', 'return', 'shift', 
+              'test', 'times', 'trap', 'umask', 'unset', 'alias',
+              'set', 'bind', 'builtin', 'caller', 'command', 'declare',
+              'echo', 'enable', 'help', 'let', 'local', 'logout', 
+              'printf', 'read', 'shopt', 'source', 'type', 'typeset',
+              'ulimit', 'unalias', 'set')
+
+def call(cline, env, cwd, loc, cb, context, echo):
+    argv = clinetoargv(cline)
+    if argv is None or (len(argv) and argv[0] in shellwords):
+        _log.debug("%s: Running command through shell because of shell metacharacters" % (loc,))
+        context.call(cline, shell=True, env=env, cwd=cwd, cb=cb, echo=echo)
+        return
+
+    if not len(argv):
+        cb(res=0)
+        return
+
+    if argv[0] == command.makepypath:
+        command.main(argv[1:], env, cwd, context, cb)
+        return
+
+    if argv[0:2] == [sys.executable, command.makepypath]:
+        command.main(argv[2:], env, cwd, context, cb)
+        return
+
+    _log.debug("%s: skipping shell, no metacharacters found" % (loc,))
+    context.call(argv, shell=False, env=env, cwd=cwd, cb=cb, echo=echo)
+
+def statustoresult(status):
+    """
+    Convert the status returned from waitpid into a prettier numeric result.
+    """
+    sig = status & 0xFF
+    if sig:
+        return -sig
+
+    return status >>8
+
+def getcontext(jcount):
+    assert jcount > 0
+    if jcount == 1:
+        return _serialsingleton
+
+    return ParallelContext(jcount)
+
+class SerialContext(object):
+    """
+    Manages the serial execution of processes.
+    """
+
+    jcount = 1
+
+    def call(self, argv, shell, env, cwd, cb, echo):
+        if echo is not None:
+            print echo
+        p = subprocess.Popen(argv, shell=shell, env=env, cwd=cwd)
+        cb(p.wait())
+
+    def finish(self):
+        pass
+
+_serialsingleton = SerialContext()
+
+class ParallelContext(object):
+    """
+    Manages the parallel execution of processes.
+    """
+
+    _allcontexts = set()
+
+    def __init__(self, jcount):
+        self.jcount = jcount
+        self.exit = False
+
+        self.pending = [] # list of (argv, shell, env, cwd, cb, echo)
+        self.running = [] # list of (subprocess, cb)
+
+        self._allcontexts.add(self)
+
+    def finish(self):
+        assert len(self.pending) == 0 and len(self.running) == 0, "pending: %i running: %i" % (len(self.pending), len(self.running))
+        self._allcontexts.remove(self)
+
+    def run(self):
+        while (len(self.running) < self.jcount) and len(self.pending):
+            _log.debug("context<%s>: pending: %i running: %i jcount: %i running a command" % (id(self), len(self.pending), len(self.running),
+                                                                                              self.jcount))
+
+            argv, shell, env, cwd, cb, echo = self.pending.pop(0)
+
+            if echo is not None:
+                print echo
+            p = subprocess.Popen(argv, shell=shell, env=env, cwd=cwd)
+            self.running.append((p, cb))
+
+    def call(self, argv, shell, env, cwd, cb, echo):
+        """
+        Asynchronously call the process
+        """
+
+        self.pending.append((argv, shell, env, cwd, cb, echo))
+        self.run()
+
+    @staticmethod
+    def spin():
+        """
+        Spin the 'event loop', and return only when it is empty.
+        """
+
+        while True:
+            for c in ParallelContext._allcontexts:
+                c.run()
+
+            pid, status = os.waitpid(-1, 0)
+            result = statustoresult(status)
+
+            found = False
+
+            for c in ParallelContext._allcontexts:
+                for i in xrange(0, len(c.running)):
+                    p, cb = c.running[i]
+                    if p.pid == pid:
+                        del c.running[i]
+                        cb(result)
+                        found = True
+                        break
+
+                if found: break
new file mode 100644
--- /dev/null
+++ b/pymake/util.py
@@ -0,0 +1,20 @@
+def makeobject(proplist, **kwargs):
+    class P(object):
+        __slots__ = proplist
+
+    p = P()
+    for k, v in kwargs.iteritems():
+        setattr(p, k, v)
+    return p
+
+class MakeError(Exception):
+    def __init__(self, message, loc=None):
+        self.message = message
+        self.loc = loc
+
+    def __str__(self):
+        locstr = ''
+        if self.loc is not None:
+            locstr = "%s:" % (self.loc,)
+
+        return "%s%s" % (locstr, self.message)
new file mode 100644
--- /dev/null
+++ b/tests/doublecolon-exists.mk
@@ -0,0 +1,16 @@
+$(shell touch foo.testfile1 foo.testfile2)
+
+# when a rule has commands and no prerequisites, should it be executed?
+# double-colon: yes
+# single-colon: no
+
+all: foo.testfile1 foo.testfile2
+	test "$$(cat foo.testfile1)" = ""
+	test "$$(cat foo.testfile2)" = "remade:foo.testfile2"
+	@echo TEST-PASS
+
+foo.testfile1:
+	@echo TEST-FAIL
+
+foo.testfile2::
+	printf "remade:$@"> $@
new file mode 100644
--- /dev/null
+++ b/tests/eval-duringexecute.mk
@@ -0,0 +1,12 @@
+#T returncode: 2
+
+# Once parsing is finished, recursive expansion in commands are not allowed to create any new rules (it may only set variables)
+
+define MORERULE
+all:
+	@echo TEST-FAIL
+endef
+
+all:
+	$(eval $(MORERULE))
+	@echo done
new file mode 100644
--- /dev/null
+++ b/tests/eval.mk
@@ -0,0 +1,7 @@
+TESTVAR = val1
+
+$(eval TESTVAR = val2)
+
+all:
+	test "$(TESTVAR)" = "val2"
+	@echo TEST-PASS
--- a/tests/file-functions-symlinks.mk
+++ b/tests/file-functions-symlinks.mk
@@ -1,21 +1,22 @@
 #T returncode-on: {'win32': 2}
 $(shell \
 touch test.file; \
 ln -s test.file test.symlink; \
+ln -s test.missing missing.symlink; \
 touch .testhidden; \
 mkdir foo; \
 touch foo/testfile; \
+ln -s foo symdir; \
 )
 
 all:
 	test "$(abspath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.symlink"
 	test "$(realpath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.file"
-	test "$(sort $(wildcard *))" = "foo test.file test.symlink"
-# commented out because GNU make matches . and .. while python doesn't, and I don't
-# care enough
-#	test "$(sort $(wildcard .*))" = ". .. .testhidden"
+	test "$(sort $(wildcard *))" = "foo symdir test.file test.symlink"
+	test "$(sort $(wildcard .*))" = ". .. .testhidden"
 	test "$(sort $(wildcard test*))" = "test.file test.symlink"
 	test "$(sort $(wildcard foo/*))" = "foo/testfile"
-	test "$(sort $(wildcard ./*))" = "./foo ./test.file ./test.symlink"
+	test "$(sort $(wildcard ./*))" = "./foo ./symdir ./test.file ./test.symlink"
 	test "$(sort $(wildcard f?o/*))" = "foo/testfile"
+	test "$(sort $(wildcard */*))" = "foo/testfile symdir/testfile"
 	@echo TEST-PASS
new file mode 100644
--- /dev/null
+++ b/tests/ifdefs-nesting.mk
@@ -0,0 +1,13 @@
+ifdef RANDOM
+ifeq (,$(error Not evaluated!))
+endif
+endif
+
+ifdef RANDOM
+ifeq (,)
+else ifeq (,$(error Not evaluated!))
+endif
+endif
+
+all:
+	@echo TEST-PASS
new file mode 100644
--- /dev/null
+++ b/tests/include-regen.mk
@@ -0,0 +1,9 @@
+# make should only consider -included makefiles for remaking if they actually exist:
+
+-include notfound.mk
+
+all:
+	@echo TEST-PASS
+
+notfound.mk::
+	@echo TEST-FAIL
new file mode 100644
--- /dev/null
+++ b/tests/parallel-simple.mk
@@ -0,0 +1,25 @@
+#T commandline: ['-j2']
+
+define SLOWMAKE
+printf "$@:0:" >>results
+sleep 0.5
+printf "$@:1:" >>results
+sleep 0.5
+printf "$@:2:" >>results
+endef
+
+EXPECTED = target1:0:target2:0:target1:1:target2:1:target1:2:target2:2:
+
+all:: target1 target2
+	cat results
+	test "$$(cat results)" = "$(EXPECTED)"
+	@echo TEST-PASS
+
+target1:
+	$(SLOWMAKE)
+
+target2:
+	sleep 0.1
+	$(SLOWMAKE)
+
+.PHONY: all
new file mode 100644
--- /dev/null
+++ b/tests/parallel-submake.mk
@@ -0,0 +1,17 @@
+#T commandline: ['-j2']
+
+# A submake shouldn't return control to the parent until it has actually finished doing everything.
+
+all:
+	-$(MAKE) -f $(TESTPATH)/parallel-submake.mk subtarget
+	cat results
+	test "$$(cat results)" = "0123"
+	@echo TEST-PASS
+
+subtarget: succeed-slowly fail-quickly
+
+succeed-slowly:
+	printf 0 >>results; sleep 1; printf 1 >>results; sleep 1; printf 2 >>results; sleep 1; printf 3 >>results
+
+fail-quickly:
+	exit 1
new file mode 100644
--- /dev/null
+++ b/tests/parallel-toserial.mk
@@ -0,0 +1,31 @@
+#T commandline: ['-j4']
+
+# Test that -j1 in a submake has the proper effect.
+
+define SLOWCMD
+printf "$@:0:" >>$(RFILE)
+sleep 0.5
+printf "$@:1:" >>$(RFILE)
+endef
+
+all: p1 p2
+subtarget: s1 s2
+
+p1 p2: RFILE = presult
+s1 s2: RFILE = sresult
+
+p1 s1:
+	$(SLOWCMD)
+
+p2 s2:
+	sleep 0.1
+	$(SLOWCMD)
+
+all:
+	$(MAKE) -j1 -f $(TESTPATH)/parallel-toserial.mk subtarget
+	printf "presult: %s\n" "$$(cat presult)"
+	test "$$(cat presult)" = "p1:0:p2:0:p1:1:p2:1:"
+	printf "sresult: %s\n" "$$(cat sresult)"
+	test "$$(cat sresult)" = "s1:0:s1:1:s2:0:s2:1:"
+	@echo TEST-PASS
+
new file mode 100644
--- /dev/null
+++ b/tests/vpath-directive.mk
@@ -0,0 +1,28 @@
+$(shell \
+mkdir subd1 subd2 subd3; \
+printf "reallybaddata" >subd1/foo.in; \
+printf "gooddata" >subd2/foo.in; \
+printf "baddata" >subd3/foo.in; \
+touch subd1/foo.in2 subd2/foo.in2 subd3/foo.in2; \
+)
+
+vpath %.in subd
+
+vpath
+vpath %.in subd2:subd3
+
+vpath %.in2 subd0
+vpath f%.in2 subd1
+vpath %.in2 :subd2
+
+%.out: %.in
+	test "$<" = "subd2/foo.in"
+	cp $< $@
+
+%.out2: %.in2
+	test "$<" = "subd1/foo.in2"
+	cp $< $@
+
+all: foo.out foo.out2
+	test "$$(cat foo.out)" = "gooddata"
+	@echo TEST-PASS
new file mode 100644
--- /dev/null
+++ b/tests/wildcards.mk
@@ -0,0 +1,21 @@
+$(shell \
+mkdir foo; \
+touch a.c b.c c.out foo/d.c; \
+sleep 1; \
+touch c.in; \
+)
+
+VPATH = foo
+
+all: c.out prog
+	test "$$(cat $<)" = "remadec.out"
+	@echo TEST-PASS
+
+*.out: %.out: %.in
+	test "$@" = c.out
+	test "$<" = c.in
+	printf "remade$@" >$@
+
+prog: *.c
+	test "$^" = "a.c b.c"
+	touch $@