Split words while iterating them, to avoid the overhead of ''.join() on expansions in the common parsing case where we just want a list of words. Also refactors many of the Function.resolve() methods to use iterators more generously.
authorBenjamin Smedberg <benjamin@smedbergs.us>
Thu, 26 Feb 2009 13:20:19 -0500
changeset 185 cefacc0cd002
parent 184 6bc02006435c
child 186 e48ab1f6d123
push id107
push userbsmedberg@mozilla.com
push date2009-02-26 18:20 +0000
Split words while iterating them, to avoid the overhead of ''.join() on expansions in the common parsing case where we just want a list of words. Also refactors many of the Function.resolve() methods to use iterators more generously.
pymake/data.py
pymake/functions.py
pymake/parserdata.py
pymake/util.py
tests/datatests.py
--- a/pymake/data.py
+++ b/pymake/data.py
@@ -136,16 +136,19 @@ class Expansion(object):
                 yield i
             else:
                 for j in i.resolve(makefile, variables, setting):
                     yield j
                     
     def resolvestr(self, makefile, variables, setting=[]):
         return ''.join(self.resolve(makefile, variables, setting))
 
+    def resolvesplit(self, makefile, variables, setting=[]):
+        return util.itersplit(self.resolve(makefile, variables, setting))
+
     def __len__(self):
         return len(self._elements)
 
     def __getitem__(self, key):
         return self._elements[key]
 
     def __setitem__(self, key, v):
         self._elements[key] = v
@@ -640,17 +643,17 @@ class Target(object):
             self.vpathtarget = self.target
             self.mtime = None
             return
 
         if self.target.startswith('-l'):
             stem = self.target[2:]
             f, s, e = makefile.variables.get('.LIBPATTERNS')
             if e is not None:
-                libpatterns = [Pattern(stripdotslash(s)) for s in e.resolvestr(makefile, makefile.variables).split()]
+                libpatterns = [Pattern(stripdotslash(s)) for s in e.resolvesplit(makefile, makefile.variables)]
                 if len(libpatterns):
                     searchdirs = ['']
                     searchdirs.extend(makefile.getvpath(self.target))
 
                     for lp in libpatterns:
                         if not lp.ispattern():
                             raise DataError('.LIBPATTERNS contains a non-pattern')
 
--- a/pymake/functions.py
+++ b/pymake/functions.py
@@ -56,20 +56,19 @@ class VariableRef(Function):
     def resolve(self, makefile, variables, setting):
         vname = self.vname.resolvestr(makefile, variables, setting)
         if vname in setting:
             raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)
 
         flavor, source, value = variables.get(vname)
         if value is None:
             log.debug("%s: variable '%s' was not set" % (self.loc, vname))
-            return
+            return ()
 
-        for j in value.resolve(makefile, variables, setting + [vname]):
-            yield j
+        return value.resolve(makefile, variables, setting + [vname])
 
 class SubstitutionRef(Function):
     """$(VARNAME:.c=.o) and $(VARNAME:%.c=%.o)"""
     def __init__(self, loc, varname, substfrom, substto):
         self.loc = loc
         self.vname = varname
         self.substfrom = substfrom
         self.substto = substto
@@ -83,27 +82,25 @@ class SubstitutionRef(Function):
             raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc)
 
         substfrom = self.substfrom.resolvestr(makefile, variables, setting)
         substto = self.substto.resolvestr(makefile, variables, setting)
 
         flavor, source, value = variables.get(vname)
         if value is None:
             log.debug("%s: variable '%s' was not set" % (self.loc, vname))
-            return
-
-        evalue = value.resolvestr(makefile, variables, setting + [vname])
+            return ()
 
         f = data.Pattern(substfrom)
         if not f.ispattern():
             f = data.Pattern('%' + substfrom)
             substto = '%' + substto
 
-        yield " ".join((f.subst(substto, word, False)
-                        for word in evalue.split()))
+        return util.joiniter((f.subst(substto, word, False)
+                              for word in value.resolvesplit(makefile, variables, setting + [vname])))
 
 class SubstFunction(Function):
     name = 'subst'
     minargs = 3
     maxargs = 3
 
     def resolve(self, makefile, variables, setting):
         s = self._arguments[0].resolvestr(makefile, variables, setting)
@@ -114,29 +111,28 @@ class SubstFunction(Function):
 class PatSubstFunction(Function):
     name = 'patsubst'
     minargs = 3
     maxargs = 3
 
     def resolve(self, makefile, variables, setting):
         s = self._arguments[0].resolvestr(makefile, variables, setting)
         r = self._arguments[1].resolvestr(makefile, variables, setting)
-        d = self._arguments[2].resolvestr(makefile, variables, setting)
 
         p = data.Pattern(s)
-        yield ' '.join((p.subst(r, word, False)
-                        for word in d.split()))
+        return util.joiniter((p.subst(r, word, False)
+                              for word in self._arguments[2].resolvesplit(makefile, variables, setting)))
 
 class StripFunction(Function):
     name = 'strip'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
-        yield ' '.join(self._arguments[0].resolvestr(makefile, variables, setting).split())
+        return util.joiniter(self._arguments[0].resolvesplit(makefile, variables, setting))
 
 class FindstringFunction(Function):
     name = 'findstring'
     minargs = 2
     maxargs = 2
 
     def resolve(self, makefile, variables, setting):
         s = self._arguments[0].resolvestr(makefile, variables, setting)
@@ -146,118 +142,108 @@ class FindstringFunction(Function):
         yield s
 
 class FilterFunction(Function):
     name = 'filter'
     minargs = 2
     maxargs = 2
 
     def resolve(self, makefile, variables, setting):
-        ps = self._arguments[0].resolvestr(makefile, variables, setting)
-        d = self._arguments[1].resolvestr(makefile, variables, setting)
-        plist = [data.Pattern(p) for p in ps.split()]
-        r = []
-        for w in d.split():
-            if util.any((p.match(w) for p in plist)):
-                r.append(w)
-                
-        yield ' '.join(r)
+        plist = [data.Pattern(p)
+                 for p in self._arguments[0].resolvesplit(makefile, variables, setting)]
+
+        return util.joiniter((w for w in self._arguments[1].resolvesplit(makefile, variables, setting)
+                              if util.any((p.match(w) for p in plist))))
 
 class FilteroutFunction(Function):
     name = 'filter-out'
     minargs = 2
     maxargs = 2
 
     def resolve(self, makefile, variables, setting):
-        ps = self._arguments[0].resolvestr(makefile, variables, setting)
-        d = self._arguments[1].resolvestr(makefile, variables, setting)
-        plist = [data.Pattern(p) for p in ps.split()]
-        r = []
-        for w in d.split():
-            found = False
-            if not util.any((p.match(w) for p in plist)):
-                r.append(w)
+        plist = [data.Pattern(p)
+                 for p in self._arguments[0].resolvesplit(makefile, variables, setting)]
 
-        yield ' '.join(r)
+        return util.joiniter((w for w in self._arguments[1].resolvesplit(makefile, variables, setting)
+                              if not util.any((p.match(w) for p in plist))))
 
 class SortFunction(Function):
     name = 'sort'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
-        d = self._arguments[0].resolvestr(makefile, variables, setting)
-        w = d.split()
-        w.sort()
-        yield ' '.join((w for w in data.withoutdups(w)))
+        d = list(self._arguments[0].resolvesplit(makefile, variables, setting))
+        d.sort()
+        return util.joiniter(d)
 
 class WordFunction(Function):
     name = 'word'
     minargs = 2
     maxargs = 2
 
     def resolve(self, makefile, variables, setting):
         n = self._arguments[0].resolvestr(makefile, variables, setting)
         # TODO: provide better error if this doesn't convert
         n = int(n)
-        words = self._arguments[1].resolvestr(makefile, variables, setting).split()
+        words = list(self._arguments[1].resolvesplit(makefile, variables, setting))
         if n < 1 or n > len(words):
             return
         yield words[n - 1]
 
 class WordlistFunction(Function):
     name = 'wordlist'
     minargs = 3
     maxargs = 3
 
     def resolve(self, makefile, variables, setting):
         nfrom = self._arguments[0].resolvestr(makefile, variables, setting)
         nto = self._arguments[1].resolvestr(makefile, variables, setting)
         # TODO: provide better errors if this doesn't convert
         nfrom = int(nfrom)
         nto = int(nto)
 
-        words = self._arguments[2].resolvestr(makefile, variables, setting).split()
+        words = list(self._arguments[2].resolvesplit(makefile, variables, setting))
 
         if nfrom < 1:
             nfrom = 1
         if nto < 1:
             nto = 1
 
-        yield ' '.join(words[nfrom - 1:nto])
+        return util.joiniter(words[nfrom - 1:nto])
 
 class WordsFunction(Function):
     name = 'words'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
-        yield str(len(self._arguments[0].resolvestr(makefile, variables, setting).split()))
+        yield str(len(list(self._arguments[0].resolvesplit(makefile, variables, setting))))
 
 class FirstWordFunction(Function):
     name = 'firstword'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
-        wl = self._arguments[0].resolvestr(makefile, variables, setting).split()
-        if len(wl) == 0:
+        for j in self._arguments[0].resolvesplit(makefile, variables, setting):
+            yield j
             return
-        yield wl[0]
 
 class LastWordFunction(Function):
     name = 'lastword'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
-        wl = self._arguments[0].resolvestr(makefile, variables, setting).split()
-        if len(wl) == 0:
-            return
-        yield wl[0]
+        last = None
+        for j in self._arguments[0].resolvesplit(makefile, variables, setting):
+            last = j
+        if last is not None:
+            yield last
 
 def pathsplit(path, default='./'):
     """
     Splits a path into dirpart, filepart on the last slash. If there is no slash, dirpart
     is ./
     """
     dir, slash, file = util.strrpartition(path, '/')
     if dir == '':
@@ -266,43 +252,43 @@ def pathsplit(path, default='./'):
     return dir + slash, file
 
 class DirFunction(Function):
     name = 'dir'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
-        yield ' '.join((pathsplit(path)[0]
-                        for path in self._arguments[0].resolvestr(makefile, variables, setting).split()))
+        return util.joiniter((pathsplit(path)[0]
+                              for path in self._arguments[0].resolvesplit(makefile, variables, setting)))
 
 class NotDirFunction(Function):
     name = 'notdir'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
-        yield ' '.join((pathsplit(path)[1]
-                        for path in self._arguments[0].resolvestr(makefile, variables, setting).split()))
+        return util.joiniter((pathsplit(path)[1]
+                              for path in self._arguments[0].resolvesplit(makefile, variables, setting)))
 
 class SuffixFunction(Function):
     name = 'suffix'
     minargs = 1
     maxargs = 1
 
     @staticmethod
     def suffixes(words):
         for w in words:
             dir, file = pathsplit(w)
             base, dot, suffix = util.strrpartition(file, '.')
             if base != '':
                 yield dot + suffix
 
     def resolve(self, makefile, variables, setting):
-        yield ' '.join(self.suffixes(self._arguments[0].resolvestr(makefile, variables, setting).split()))
+        return util.joiniter(self.suffixes(self._arguments[0].resolvesplit(makefile, variables, setting)))
 
 class BasenameFunction(Function):
     name = 'basename'
     minargs = 1
     maxargs = 1
 
     @staticmethod
     def basenames(words):
@@ -310,152 +296,151 @@ class BasenameFunction(Function):
             dir, file = pathsplit(w, '')
             base, dot, suffix = util.strrpartition(file, '.')
             if dot == '':
                 base = suffix
 
             yield dir + base
 
     def resolve(self, makefile, variables, setting):
-        yield ' '.join(self.basenames(self._arguments[0].resolvestr(makefile, variables, setting).split()))
+        return util.joiniter(self.basenames(self._arguments[0].resolvesplit(makefile, variables, setting)))
 
 class AddSuffixFunction(Function):
     name = 'addprefix'
     minargs = 2
     maxargs = 2
 
     def resolve(self, makefile, variables, setting):
         suffix = self._arguments[0].resolvestr(makefile, variables, setting)
 
-        yield ' '.join((w + suffix for w in self._arguments[1].resolvestr(makefile, variables, setting).split()))
+        return util.joiniter((w + suffix for w in self._arguments[1].resolvesplit(makefile, variables, setting)))
 
 class AddPrefixFunction(Function):
     name = 'addsuffix'
     minargs = 2
     maxargs = 2
 
     def resolve(self, makefile, variables, setting):
         prefix = self._arguments[0].resolvestr(makefile, variables, setting)
 
-        yield ' '.join((prefix + w for w in self._arguments[1].resolvestr(makefile, variables, setting).split()))
+        return util.joiniter((prefix + w for w in self._arguments[1].resolvesplit(makefile, variables, setting)))
 
 class JoinFunction(Function):
     name = 'join'
     minargs = 2
     maxargs = 2
 
     @staticmethod
     def iterjoin(l1, l2):
         for i in xrange(0, max(len(l1), len(l2))):
             i1 = i < len(l1) and l1[i] or ''
             i2 = i < len(l2) and l2[i] or ''
             yield i1 + i2
 
     def resolve(self, makefile, variables, setting):
-        list1 = self._arguments[0].resolvestr(makefile, variables, setting).split()
-        list2 = self._arguments[1].resolvestr(makefile, variables, setting).split()
+        list1 = list(self._arguments[0].resolvesplit(makefile, variables, setting))
+        list2 = list(self._arguments[1].resolvesplit(makefile, variables, setting))
 
-        yield ' '.join(self.iterjoin(list1, list2))
+        return util.joiniter(self.iterjoin(list1, list2))
 
 class WildcardFunction(Function):
     name = 'wildcard'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
-        patterns = self._arguments[0].resolvestr(makefile, variables, setting).split()
+        patterns = self._arguments[0].resolvesplit(makefile, variables, setting)
 
-        r = []
-        for p in patterns:
-            r.extend([x.replace('\\','/') for x in glob(makefile.workdir, p)])
-        yield ' '.join(r)
+        return util.joiniter((x.replace('\\','/')
+                              for p in patterns
+                              for x in glob(makefile.workdir, p)))
 
 class RealpathFunction(Function):
     name = 'realpath'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
-        paths = self._arguments[0].resolvestr(makefile, variables, setting).split()
-        fspaths = [os.path.join(makefile.workdir, path) for path in paths]
-        realpaths = [os.path.realpath(path).replace('\\','/') for path in fspaths]
-        yield ' '.join(realpaths)
+        return util.joiniter((os.path.realpath(os.path.join(makefile.workdir, path)).replace('\\', '/')
+                              for path in self._arguments[0].resolvesplit(makefile, variables, setting)))
 
 class AbspathFunction(Function):
     name = 'abspath'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
         assert os.path.isabs(makefile.workdir)
-        paths = self._arguments[0].resolvestr(makefile, variables, setting).split()
-        fspaths = [os.path.join(makefile.workdir, path).replace('\\','/') for path in paths]
-        yield ' '.join(fspaths)
+        return util.joiniter((os.path.join(makefile.workdir, path).replace('\\', '/')
+                              for path in self._arguments[0].resolvesplit(makefile, variables, setting)))
 
 class IfFunction(Function):
     name = 'if'
     minargs = 1
     maxargs = 3
 
     def setup(self):
         Function.setup(self)
         self._arguments[0].lstrip()
         self._arguments[0].rstrip()
 
     def resolve(self, makefile, variables, setting):
         condition = self._arguments[0].resolvestr(makefile, variables, setting)
         if len(condition):
-            for j in self._arguments[1].resolvestr(makefile, variables, setting):
-                yield j
-        elif len(self._arguments) > 2:
-            for j in self._arguments[2].resolvestr(makefile, variables, setting):
-                yield j
+            return self._arguments[1].resolve(makefile, variables, setting)
+
+        if len(self._arguments) > 2:
+            return self._arguments[2].resolve(makefile, variables, setting)
+
+        return ()
 
 class OrFunction(Function):
     name = 'or'
     minargs = 1
     maxargs = 0
 
     def resolve(self, makefile, variables, setting):
         for arg in self._arguments:
             r = arg.resolvestr(makefile, variables, setting)
             if r != '':
-                yield r
-                return
+                return (r,)
+
+        return ()
 
 class AndFunction(Function):
     name = 'and'
     minargs = 1
     maxargs = 0
 
     def resolve(self, makefile, variables, setting):
         r = ''
 
         for arg in self._arguments:
             r = arg.resolvestr(makefile, variables, setting)
             if r == '':
-                return
+                return ()
 
-        yield r
+        return r,
 
 class ForEachFunction(Function):
     name = 'foreach'
     minargs = 3
     maxargs = 3
 
     def resolve(self, makefile, variables, setting):
         vname = self._arguments[0].resolvestr(makefile, variables, setting)
-        words = self._arguments[1].resolvestr(makefile, variables, setting).split()
-
         e = self._arguments[2]
 
         v = data.Variables(parent=variables)
-        for i in xrange(0, len(words)):
-            w = words[i]
-            if i > 0:
+        firstword = True
+
+        for w in self._arguments[1].resolvesplit(makefile, variables, setting):
+            if firstword:
+                firstword = False
+            else:
                 yield ' '
 
             v.set(vname, data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, w)
             for j in e.resolve(makefile, v, setting):
                 yield j
 
 class CallFunction(Function):
     name = 'call'
@@ -470,24 +455,23 @@ class CallFunction(Function):
         v = data.Variables(parent=variables)
         v.set('0', data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, vname)
         for i in xrange(1, len(self._arguments)):
             param = self._arguments[i].resolvestr(makefile, variables, setting)
             v.set(str(i), data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, param)
 
         flavor, source, e = variables.get(vname)
         if e is None:
-            return
+            return ()
 
         if flavor == data.Variables.FLAVOR_SIMPLE:
             log.warning("%s: calling variable '%s' which is simply-expanded" % (self.loc, vname))
 
         # but we'll do it anyway
-        for j in e.resolve(makefile, v, setting + [vname]):
-            yield j
+        return e.resolve(makefile, v, setting + [vname])
 
 class ValueFunction(Function):
     name = 'value'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
         varname = self._arguments[0].resolvestr(makefile, variables, setting)
@@ -507,60 +491,66 @@ class EvalFunction(Function):
         if makefile.parsingfinished:
             # GNU make allows variables to be set by recursive expansion during
             # command execution. This seems really dumb to me, so I don't!
             raise data.DataError("$(eval) not allowed via recursive expansion after parsing is finished", self.loc)
 
         text = StringIO(self._arguments[0].resolvestr(makefile, variables, setting))
         stmts = parser.parsestream(text, 'evaluation from %s' % self.loc)
         stmts.execute(makefile)
-        return
-        yield None
+        return ()
 
 class OriginFunction(Function):
     name = 'origin'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
         vname = self._arguments[0].resolvestr(makefile, variables, setting)
 
         flavor, source, value = variables.get(vname)
         if source is None:
-            yield 'undefined'
-        elif source == data.Variables.SOURCE_OVERRIDE:
-            yield 'override'
-        elif source == data.Variables.SOURCE_MAKEFILE:
-            yield 'file'
-        elif source == data.Variables.SOURCE_ENVIRONMENT:
-            yield 'environment'
-        elif source == data.Variables.SOURCE_COMMANDLINE:
-            yield 'command line'
-        elif source == data.Variables.SOURCE_AUTOMATIC:
-            yield 'automatic'
-        else:
-            assert False, "Unexpected source value: %s" % source
+            return 'undefined',
+
+        if source == data.Variables.SOURCE_OVERRIDE:
+            return 'override',
+
+        if source == data.Variables.SOURCE_MAKEFILE:
+            return 'file',
+
+        if source == data.Variables.SOURCE_ENVIRONMENT:
+            return 'environment',
+
+        if source == data.Variables.SOURCE_COMMANDLINE:
+            return 'command line',
+
+        if source == data.Variables.SOURCE_AUTOMATIC:
+            return 'automatic',
+
+        assert False, "Unexpected source value: %s" % source
 
 class FlavorFunction(Function):
     name = 'flavor'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
         varname = self._arguments[0].resolvestr(makefile, variables, setting)
         
         flavor, source, value = variables.get(varname)
         if flavor is None:
-            yield 'undefined'
-        elif flavor == data.Variables.FLAVOR_RECURSIVE:
-            yield 'recursive'
-        elif flavor == data.Variables.FLAVOR_SIMPLE:
-            yield 'simple'
-        else:
-            assert False, "Neither simple nor recursive?"
+            return 'undefined',
+
+        if flavor == data.Variables.FLAVOR_RECURSIVE:
+            return 'recursive',
+
+        if flavor == data.Variables.FLAVOR_SIMPLE:
+            return 'simple',
+
+        assert False, "Neither simple nor recursive?"
 
 class ShellFunction(Function):
     name = 'shell'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
         #TODO: call this once up-front somewhere and save the result?
@@ -592,29 +582,27 @@ class ErrorFunction(Function):
 class WarningFunction(Function):
     name = 'warning'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
         v = self._arguments[0].resolvestr(makefile, variables, setting)
         log.warning(v)
-        return
-        yield None
+        return ()
 
 class InfoFunction(Function):
     name = 'info'
     minargs = 1
     maxargs = 1
 
     def resolve(self, makefile, variables, setting):
         v = self._arguments[0].resolvestr(makefile, variables, setting)
         log.info(v)
-        return
-        yield None
+        return ()
 
 functionmap = {
     'subst': SubstFunction,
     'patsubst': PatSubstFunction,
     'strip': StripFunction,
     'findstring': FindstringFunction,
     'filter': FilterFunction,
     'filter-out': FilteroutFunction,
--- a/pymake/parserdata.py
+++ b/pymake/parserdata.py
@@ -110,29 +110,29 @@ class Rule(Statement):
         assert isinstance(targetexp, data.Expansion)
         assert isinstance(depexp, data.Expansion)
         
         self.targetexp = targetexp
         self.depexp = depexp
         self.doublecolon = doublecolon
 
     def execute(self, makefile, context):
-        atargets = data.stripdotslashes(self.targetexp.resolvestr(makefile, makefile.variables).split())
+        atargets = data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))
         targets = [data.Pattern(p) for p in _expandwildcards(makefile, atargets)]
 
         if not len(targets):
             context.currule = DummyRule()
             return
 
         ispatterns = set((t.ispattern() for t in targets))
         if len(ispatterns) == 2:
             raise data.DataError("Mixed implicit and normal rule", self.targetexp.loc)
         ispattern, = ispatterns
 
-        deps = [p for p in _expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvestr(makefile, makefile.variables).split()))]
+        deps = [p for p in _expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables)))]
         if ispattern:
             rule = data.PatternRule(targets, map(data.Pattern, deps), self.doublecolon, loc=self.targetexp.loc)
             makefile.appendimplicitrule(rule)
         else:
             rule = data.Rule(deps, self.doublecolon, loc=self.targetexp.loc)
             for t in targets:
                 makefile.gettarget(t.gettarget()).addrule(rule)
             makefile.foundtarget(targets[0].gettarget())
@@ -149,28 +149,28 @@ class StaticPatternRule(Statement):
         assert isinstance(depexp, data.Expansion)
 
         self.targetexp = targetexp
         self.patternexp = patternexp
         self.depexp = depexp
         self.doublecolon = doublecolon
 
     def execute(self, makefile, context):
-        targets = list(_expandwildcards(makefile, data.stripdotslashes(self.targetexp.resolvestr(makefile, makefile.variables).split())))
+        targets = list(_expandwildcards(makefile, data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))))
 
         if not len(targets):
             context.currule = DummyRule()
             return
 
-        patterns = list(data.stripdotslashes(self.patternexp.resolvestr(makefile, makefile.variables).split()))
+        patterns = list(data.stripdotslashes(self.patternexp.resolvesplit(makefile, makefile.variables)))
         if len(patterns) != 1:
             raise data.DataError("Static pattern rules must have a single pattern", self.patternexp.loc)
         pattern = data.Pattern(patterns[0])
 
-        deps = [data.Pattern(p) for p in _expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvestr(makefile, makefile.variables).split()))]
+        deps = [data.Pattern(p) for p in _expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables)))]
 
         rule = data.PatternRule([pattern], deps, self.doublecolon, loc=self.targetexp.loc)
 
         for t in targets:
             if data.Pattern(t).ispattern():
                 raise data.DataError("Target '%s' of a static pattern rule must not be a pattern" % (t,), self.targetexp.loc)
             stem = pattern.match(t)
             if stem is None:
@@ -216,17 +216,17 @@ class SetVariable(Statement):
         if len(vname) == 0:
             raise data.DataError("Empty variable name", self.vnameexp.loc)
 
         if self.targetexp is None:
             setvariables = [makefile.variables]
         else:
             setvariables = []
 
-            targets = [data.Pattern(t) for t in data.stripdotslashes(self.targetexp.resolvestr(makefile, makefile.variables).split())]
+            targets = [data.Pattern(t) for t in data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))]
             for t in targets:
                 if t.ispattern():
                     setvariables.append(makefile.getpatternvariables(t))
                 else:
                     setvariables.append(makefile.gettarget(t.gettarget()).variables)
 
         for v in setvariables:
             if self.token == '+=':
@@ -352,30 +352,30 @@ class ConditionBlock(Statement):
 
 class Include(Statement):
     def __init__(self, exp, required):
         assert isinstance(exp, data.Expansion)
         self.exp = exp
         self.required = required
 
     def execute(self, makefile, context):
-        files = self.exp.resolvestr(makefile, makefile.variables).split()
+        files = self.exp.resolvesplit(makefile, makefile.variables)
         for f in files:
             makefile.include(f, self.required, loc=self.exp.loc)
 
     def dump(self, fd, indent):
         print >>fd, indent, "Include %r" % (self.exp,)
 
 class VPathDirective(Statement):
     def __init__(self, exp):
         assert isinstance(exp, data.Expansion)
         self.exp = exp
 
     def execute(self, makefile, context):
-        words = list(data.stripdotslashes(self.exp.resolvestr(makefile, makefile.variables).split()))
+        words = list(data.stripdotslashes(self.exp.resolvesplit(makefile, makefile.variables)))
         if len(words) == 0:
             makefile.clearallvpaths()
         else:
             pattern = data.Pattern(words[0])
             mpaths = words[1:]
 
             if len(mpaths) == 0:
                 makefile.clearvpath(pattern)
@@ -395,17 +395,17 @@ class ExportDirective(Statement):
         assert isinstance(exp, data.Expansion)
         self.exp = exp
         self.single = single
 
     def execute(self, makefile, context):
         if self.single:
             vlist = [self.exp.resolvestr(makefile, makefile.variables)]
         else:
-            vlist = self.exp.resolvestr(makefile, makefile.variables).split()
+            vlist = list(self.exp.resolvesplit(makefile, makefile.variables))
             if not len(vlist):
                 raise data.DataError("Exporting all variables is not supported", self.exp.loc)
 
         for v in vlist:
             makefile.exportedvars.add(v)
 
     def dump(self, fd, indent):
         print >>fd, indent, "Export (single=%s) %r" % (self.single, self.exp)
--- a/pymake/util.py
+++ b/pymake/util.py
@@ -16,16 +16,61 @@ class MakeError(Exception):
 
     def __str__(self):
         locstr = ''
         if self.loc is not None:
             locstr = "%s:" % (self.loc,)
 
         return "%s%s" % (locstr, self.message)
 
+def itersplit(it):
+    """
+    Given an iterator that returns strings, yield words as if string.split() had been called on the concatenation
+    of the strings.
+    """
+
+    curword = None
+    for s in it:
+        if not len(s):
+            continue
+
+        initws = s[0].isspace()
+        trailws = s[-1].isspace()
+
+        words = s.split()
+        if curword is not None:
+            if initws:
+                yield curword
+            else:
+                words[0] = curword + words[0]
+
+        if trailws:
+            curword = None
+        else:
+            curword = words.pop()
+
+        for w in words:
+            yield w
+
+    if curword is not None:
+        yield curword
+
+def joiniter(it, j=' '):
+    """
+    Given an iterator that returns strings, yield the words with j inbetween each.
+    """
+    it = iter(it)
+    for i in it:
+        yield i
+        break
+
+    for i in it:
+        yield j
+        yield i
+
 def checkmsyscompat():
     """For msys compatibility on windows, honor the SHELL environment variable,
     and if $MSYSTEM == MINGW32, run commands through $SHELL -c instead of
     letting Python use the system shell."""
     if 'SHELL' in os.environ:
         shell = os.environ['SHELL']
     elif 'COMSPEC' in os.environ:
         shell = os.environ['COMSPEC']
--- a/tests/datatests.py
+++ b/tests/datatests.py
@@ -1,12 +1,34 @@
-import pymake.data
+import pymake.data, pymake.util
 import unittest
 import re
 
+def multitest(cls):
+    for name in cls.testdata.iterkeys():
+        def m(self, name=name):
+            return self.runSingle(*self.testdata[name])
+
+        setattr(cls, 'test_%s' % name, m)
+    return cls
+
+class IterSplitTest(unittest.TestCase):
+    testdata = {
+        'nothing': ((), []),
+        'oneword': (('word'), ['word']),
+        'twowords': (('word   word2'), ['word', 'word2']),
+        'splitwords': (('word  ', ' word2 '), (['word', 'word2'])),
+        'joined': (('word', 'word2'), (['wordword2'])),
+        }
+
+    def runSingle(self, it, expected):
+        got = list(pymake.util.itersplit(it))
+        self.assertEqual(got, expected)
+multitest(IterSplitTest)
+
 class SplitWordsTest(unittest.TestCase):
     testdata = (
         (' test test.c test.o ', ['test', 'test.c', 'test.o']),
         ('\ttest\t  test.c \ntest.o', ['test', 'test.c', 'test.o']),
     )
 
     def runTest(self):
         for s, e in self.testdata: