Performance optimizations:
authorBenjamin Smedberg <benjamin@smedbergs.us>
Thu, 26 Feb 2009 15:35:57 -0500
changeset 189 b0864b858e97
parent 188 f9df0708b6d6
child 190 e36530da06f1
push id110
push userbsmedberg@mozilla.com
push date2009-02-26 20:36 +0000
Performance optimizations: * Make Expansion derive directly from list * The length of data will not change during iterdata: cache the length * Add __slots__ to Expansion and ParseStackFrame to reduce initialization time
pymake/data.py
pymake/parser.py
--- a/pymake/data.py
+++ b/pymake/data.py
@@ -55,97 +55,104 @@ def stripdotslashes(sl):
 def getindent(stack):
     return ''.ljust(len(stack) - 1)
 
 def _if_else(c, t, f):
     if c:
         return t()
     return f()
 
-class Expansion(object):
+class Expansion(list):
     """
     A representation of expanded data, such as that for a recursively-expanded variable, a command, etc.
     """
 
+    __slots__ = ('loc',)
+
     def __init__(self, loc=None):
-        # Each element is either a string or a function
-        self._elements = [] # element, isfunc
+        # A list of (element, isfunc) tuples
+        # element is either a string or a function
         self.loc = loc
 
     @staticmethod
     def fromstring(s):
         e = Expansion()
         e.appendstr(s)
         return e
 
     def clone(self):
         e = Expansion()
-        e._elements = list(self._elements)
+        e.extend(self)
         return e
 
     def _lastisstring(self):
-        return len(self._elements) and not self._elements[-1][1]
+        return len(self) and not self[-1][1]
 
     def _firstisstring(self):
-        return len(self._elements) and not self._elements[0][1]
+        return len(self) and not self[0][1]
+
+    def append(self, e):
+        raise NotImplementedError("Don't call me!")
+
+    _append = list.append
 
     def appendstr(self, s):
         assert isinstance(s, str)
         if s == '':
             return
 
         if self._lastisstring():
-            s = self._elements[-1][0] + s
-            self._elements[-1] = s, False
+            s = self[-1][0] + s
+            self[-1] = s, False
         else:
-            self._elements.append((s, False))
+            self._append((s, False))
 
     def appendfunc(self, func):
         assert isinstance(func, functions.Function)
-        self._elements.append((func, True))
+        self._append((func, True))
 
     def concat(self, o):
         """Concatenate the other expansion on to this one."""
         if o._firstisstring() and self._lastisstring():
-            mystr = self._elements[-1][0]
-            ostr = o._elements[0][0]
-            self._elements[-1] = mystr + ostr, False
-            self._elements.extend(o._elements[1:])
+            mystr = self[-1][0]
+            ostr = o[0][0]
+            self[-1] = mystr + ostr, False
+            self.extend(o[1:])
         else:
-            self._elements.extend(o._elements)
+            self.extend(o)
 
     def isempty(self):
-        return (not len(self._elements)) or self._elements[0] == ('', False)
+        return (not len(self)) or self[0] == ('', False)
 
     def lstrip(self):
         """Strip leading literal whitespace from this expansion."""
         if self._firstisstring():
-            s = self._elements[0][0].lstrip()
-            self._elements[0] = s, False
+            s = self[0][0].lstrip()
+            self[0] = s, False
 
     def rstrip(self):
         """Strip trailing literal whitespace from this expansion."""
         if self._lastisstring():
-            s = self._elements[-1][0].rstrip()
-            self._elements[-1] = s, False
+            s = self[-1][0].rstrip()
+            self[-1] = s, False
 
     def resolve(self, makefile, variables, setting=[]):
         """
         Resolve this variable into a value, by interpolating the value
         of other variables.
 
         @param setting (Variable instance) the variable currently
                being set, if any. Setting variables must avoid self-referential
                loops.
         """
         assert isinstance(makefile, Makefile)
         assert isinstance(variables, Variables)
         assert isinstance(setting, list)
 
-        for e, isfunc in self._elements:
+        for e, isfunc in self:
             if isfunc:
                 it = e.resolve(makefile, variables, setting)
                 assert not isinstance(it, str)
                 for j in it:
                     yield j
             else:
                 assert isinstance(e, str)
                 yield e
@@ -159,21 +166,18 @@ class Expansion(object):
             except:
                 print "error appending i: %r" % (i,)
                 raise
         return s
 
     def resolvesplit(self, makefile, variables, setting=[]):
         return util.itersplit(self.resolve(makefile, variables, setting))
 
-    def __len__(self):
-        return len(self._elements)
-
     def __repr__(self):
-        return "<Expansion with elements: %r>" % (self._elements,)
+        return "<Expansion with elements: %r>" % ([repr(e) for e, isfunc in self],)
 
 class Variables(object):
     """
     A mapping from variable names to variables. Variables have flavor, source, and value. The value is an 
     expansion object.
     """
 
     FLAVOR_RECURSIVE = 0
--- a/pymake/parser.py
+++ b/pymake/parser.py
@@ -174,18 +174,19 @@ emptytokenlist = TokenList.get('')
 # coming.
 
 def iterdata(d, offset, tokenlist):
     if tokenlist.emptylist:
         yield d.data, None, None, None
         return
 
     s = tokenlist.simplere
+    datalen = len(d.data)
 
-    while offset < len(d.data):
+    while offset < datalen:
         m = s.search(d.data, pos=offset)
         if m is None:
             yield d.data[offset:], None, None, None
             return
 
         yield d.data[offset:m.start(0)], m.group(0), m.start(0), m.end(0)
         offset = m.end(0)
 
@@ -662,24 +663,26 @@ PARSESTATE_FUNCTION = 1    # expanding a
 # For the following three, data is a tuple of Expansions: (varname, substfrom, substto)
 PARSESTATE_VARNAME = 2     # expanding a variable expansion.
 PARSESTATE_SUBSTFROM = 3   # expanding a variable expansion substitution "from" value
 PARSESTATE_SUBSTTO = 4     # expanding a variable expansion substitution "to" value
 
 PARSESTATE_PARENMATCH = 5
 
 class ParseStackFrame(object):
-    def __init__(self, parsestate, expansion, tokenlist, openbrace, closebrace, **kwargs):
+    __slots__ = ('parsestate', 'expansion', 'tokenlist', 'openbrace', 'closebrace', 'function', 'loc', 'varname', 'substfrom')
+
+    def __init__(self, parsestate, expansion, tokenlist, openbrace, closebrace, function=None, loc=None):
         self.parsestate = parsestate
         self.expansion = expansion
         self.tokenlist = tokenlist
         self.openbrace = openbrace
         self.closebrace = closebrace
-        for key, value in kwargs.iteritems():
-            setattr(self, key, value)
+        self.function = function
+        self.loc = loc
 
 _functiontokenlist = None
 
 _matchingbrace = {
     '(': ')',
     '{': '}',
     }
 
@@ -708,17 +711,17 @@ def parsemakesyntax(d, startat, stopon, 
         functiontokens.sort(key=len, reverse=True)
         _functiontokenlist = TokenList.get(tuple(functiontokens))
 
     assert callable(iterfunc)
 
     stack = [
         ParseStackFrame(PARSESTATE_TOPLEVEL, data.Expansion(loc=d.getloc(startat)),
                         tokenlist=TokenList.get(stopon + ('$',)),
-                        stopon=stopon, openbrace=None, closebrace=None)
+                        openbrace=None, closebrace=None)
     ]
 
     di = iterfunc(d, startat, stack[-1].tokenlist)
     while True: # this is not a for loop because `di` changes during the function
         stacktop = stack[-1]
         try:
             s, token, tokenoffset, offset = di.next()
         except StopIteration: