Lazy parsing of variable expansions. It turns out this helps performance a *lot*!
authorBenjamin Smedberg <benjamin@smedbergs.us>
Thu, 26 Feb 2009 14:05:41 -0500
changeset 187 5d7346eb92cd
parent 186 e48ab1f6d123
child 188 f9df0708b6d6
push id108
push userbsmedberg@mozilla.com
push dateThu, 26 Feb 2009 19:05:51 +0000
Lazy parsing of variable expansions. It turns out this helps performance a *lot*!
pymake/data.py
--- a/pymake/data.py
+++ b/pymake/data.py
@@ -177,34 +177,39 @@ class Variables(object):
     SOURCE_COMMANDLINE = 1
     SOURCE_MAKEFILE = 2
     SOURCE_ENVIRONMENT = 3
     SOURCE_AUTOMATIC = 4
     # I have no intention of supporting builtin rules or variables that go with them
     # SOURCE_IMPLICIT = 5
 
     def __init__(self, parent=None):
-        self._map = {} # vname -> flavor, source, valuestr, valueexp, expansionerror
+        self._map = {} # vname -> flavor, source, valuestr, valueexp
         self.parent = parent
 
     def readfromenvironment(self, env):
         for k, v in env.iteritems():
             self.set(k, self.FLAVOR_SIMPLE, self.SOURCE_ENVIRONMENT, v)
 
     def get(self, name, expand=True):
         """
         Get the value of a named variable. Returns a tuple (flavor, source, value)
 
         If the variable is not present, returns (None, None, None)
 
         @param expand If true, the value will be returned as an expansion. If false,
         it will be returned as an unexpanded string.
         """
         if name in self._map:
-            flavor, source, valuestr, valueexp, expansionerror = self._map[name]
+            flavor, source, valuestr, valueexp = self._map[name]
+            if expand and flavor != self.FLAVOR_SIMPLE and valueexp is None:
+                d = parser.Data.fromstring(valuestr, parserdata.Location("Expansion of variables '%s'" % (name,), 1, 0))
+                valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
+                self._map[name] = flavor, source, valuestr, valueexp
+
             if flavor == self.FLAVOR_APPEND:
                 if self.parent:
                     pflavor, psource, pvalue = self.parent.get(name, expand)
                 else:
                     pflavor, psource, pvalue = None, None, None
 
                 if pvalue is None:
                     flavor = self.FLAVOR_RECURSIVE
@@ -212,36 +217,26 @@ class Variables(object):
                 else:
                     if source > psource:
                         # TODO: log a warning?
                         return pflavor, psource, pvalue
 
                     if not expand:
                         return pflavor, psource, pvalue + ' ' + valuestr
 
-                    if expansionerror is not None:
-                        raise expansionerror
-
-                    
-                    d = parser.Data.fromstring(valuestr, parserdata.Location("Expansion of variable '%s'" % (name,), 1, 0))
-                    appende, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
-
                     pvalue = pvalue.clone()
                     pvalue.append(' ')
                     pvalue.concat(valueexp)
 
                     return pflavor, psource, pvalue
                     
             if not expand:
                 return flavor, source, valuestr
 
             if flavor == self.FLAVOR_RECURSIVE:
-                if expansionerror is not None:
-                    raise expansionerror
-
                 val = valueexp
             else:
                 val = Expansion.fromstring(valuestr)
 
             return flavor, source, val
 
         if self.parent is not None:
             return self.parent.get(name, expand)
@@ -254,79 +249,57 @@ class Variables(object):
         assert isinstance(value, str), "expected str, got %s" % type(value)
 
         prevflavor, prevsource, prevvalue = self.get(name)
         if prevsource is not None and source > prevsource:
             # TODO: give a location for this warning
             _log.info("not setting variable '%s', set by higher-priority source to value '%s'" % (name, prevvalue))
             return
 
-        if flavor == self.FLAVOR_SIMPLE:
-            valueexp = None
-            expansionerror = None
-        else:
-            try:
-                d = parser.Data.fromstring(value, parserdata.Location("Expansion of variable '%s'" % (name,), 1, 0))
-                valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
-                expansionerror = None
-            except parser.SyntaxError, e:
-                valueexp = None
-                expansionerror = e
-
-        self._map[name] = (flavor, source, value, valueexp, expansionerror)
+        self._map[name] = flavor, source, value, None
 
     def append(self, name, source, value, variables, makefile):
         assert source in (self.SOURCE_OVERRIDE, self.SOURCE_MAKEFILE, self.SOURCE_AUTOMATIC)
         assert isinstance(value, str)
 
         def expand():
             try:
                 d = parser.Data.fromstring(value, parserdata.Location("Expansion of variable '%s'" % (name,), 1, 0))
                 valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
                 return valueexp, None
             except parser.SyntaxError, e:
                 return None, e
         
         if name not in self._map:
-            exp, err = expand()
-            self._map[name] = self.FLAVOR_APPEND, source, value, exp, err
+            self._map[name] = self.FLAVOR_APPEND, source, value, None
             return
 
-        prevflavor, prevsource, prevvalue, valueexp, err = self._map[name]
+        prevflavor, prevsource, prevvalue, valueexp = self._map[name]
         if source > prevsource:
             # TODO: log a warning?
             return
 
         if prevflavor == self.FLAVOR_SIMPLE:
-            exp, err = expand()
-            if err is not None:
-                raise err
+            d = parser.Data.fromstring(value, parserdata.Location("Expansion of variables '%s'" % (name,), 1, 0))
+            valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
 
-            val = exp.resolvestr(makefile, variables, [name])
-            self._map[name] = prevflavor, prevsource, prevvalue + ' ' + val, None, None
+            val = valueexp.resolvestr(makefile, variables, [name])
+            self._map[name] = prevflavor, prevsource, prevvalue + ' ' + val, None
             return
 
         newvalue = prevvalue + ' ' + value
-        try:
-            d = parser.Data.fromstring(newvalue, parserdata.Location("Expansion of variable '%s'" % (name,), 1, 0))
-            valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata)
-            err = None
-        except parser.SyntaxError, e:
-            valueexp = None
-            err = e
-
-        self._map[name] = prevflavor, prevsource, newvalue, valueexp, err
+        self._map[name] = prevflavor, prevsource, newvalue, None
 
     def merge(self, other):
         assert isinstance(other, Variables)
         for k, flavor, source, value in other:
             self.set(k, flavor, source, value)
 
     def __iter__(self):
-        for k, (flavor, source, value, valueexp, expansionerr) in self._map.iteritems():
+        for k, (flavor, source, value, valueexp) in self._map.iteritems():
             yield k, flavor, source, value
 
     def __contains__(self, item):
         return item in self._map
 
 class Pattern(object):
     """
     A pattern is a string, possibly with a % substitution character. From the GNU make manual: