Don't create locations for each iterated character... most of the time we don't use them, and they're easy to recreate from the data+offset. This is a significant performance boost.
authorBenjamin Smedberg <benjamin@smedbergs.us>
Thu, 12 Feb 2009 10:39:40 -0500
changeset 102 1685c0669a0c167ee8a37bfd14849c0a74e4b39b
parent 101 ce89bcf91bf6b611b2e396413a786c5d5a922d37
child 103 f19e091ae2d821a1cba379a88ce96a6c93c25931
push id54
push userbsmedberg@mozilla.com
push dateThu, 12 Feb 2009 16:16:35 +0000
Don't create locations for each iterated character... most of the time we don't use them, and they're easy to recreate from the data+offset. This is a significant performance boost.
pymake/parser.py
--- a/pymake/parser.py
+++ b/pymake/parser.py
@@ -154,25 +154,25 @@ class Data(object):
                     return t, end
                 elif end == len(self.data) or self.data[end].isspace():
                     end = self.skipwhitespace(end)
                     return t, end
         return None, o
 
 def iterdata(d, offset):
     """
-    A Data iterator yielding (char, offset, location) without any escaping.
+    A Data iterator yielding (char, offset) without any escaping.
     """
     while offset < len(d.data):
-        yield d.data[offset], offset, d.getloc(offset)
+        yield d.data[offset], offset
         offset += 1
 
 def itermakefilechars(d, offset):
     """
-    A Data generator yielding (char, offset, location). It will escape comments and newline
+    A Data generator yielding (char, offset). It will escape comments and newline
     continuations according to makefile syntax rules.
     """
 
     while offset < len(d.data):
         c = d.data[offset]
         if c == '\n':
             assert offset == len(d.data) - 1
             return
@@ -187,78 +187,78 @@ def itermakefilechars(d, offset):
                         assert offset == len(d.data) - 1, 'unexpected newline'
                         d.readline()
                 offset += 1
             return
         elif c == '\\' and offset < len(d.data) - 1:
             c2 = d.data[offset + 1]
             if c2 == '#':
                 offset += 1
-                yield '#', offset, d.getloc(offset)
+                yield '#', offset
                 offset += 1
             elif d[offset:offset + 3] == '\\\\#':
                 # see escape-chars.mk VARAWFUL
                 offset += 1
-                yield '\\', offset, d.getloc(offset)
+                yield '\\', offset
                 offset += 1
             elif c2 == '\n':
-                yield ' ', offset, d.getloc(offset)
+                yield ' ', offset
                 d.readline()
                 offset = d.skipwhitespace(offset + 2)
             elif c2 == '\\':
-                yield '\\', offset, d.getloc(offset)
+                yield '\\', offset
                 offset += 1
-                yield '\\', offset, d.getloc(offset)
+                yield '\\', offset
                 offset += 1
             else:
-                yield c, offset, d.getloc(offset)
+                yield c, offset
                 offset += 1
         else:
             if c.isspace():
                 o = d.skipwhitespace(offset)
                 if d.data[o:o+2] == '\\\n':
                     offset = o
                     continue
 
-            yield c, offset, d.getloc(offset)
+            yield c, offset
             offset += 1
 
 def itercommandchars(d, offset):
     """
-    A Data generator yielding (char, offset, location). It will process escapes and newlines
+    A Data generator yielding (char, offset). It will process escapes and newlines
     according to command parsing rules.
     """
 
     while offset < len(d.data):
         c = d.data[offset]
         if c == '\n':
             assert offset == len(d.data) - 1
             return
 
-        yield c, offset, d.getloc(offset)
+        yield c, offset
         offset += 1
 
         if c == '\\':
             if offset == len(d.data):
                 return
 
             c = d.data[offset]
-            yield c, offset, d.getloc(offset)
+            yield c, offset
 
             offset += 1
 
             if c == '\n':
                 assert offset == len(d.data)
                 d.readline()
                 if offset < len(d.data) and d.data[offset] == '\t':
                     offset += 1
 
 def iterdefinechars(d, offset):
     """
-    A Data generator yielding (char, offset, location). It will process define/endef
+    A Data generator yielding (char, offset). It will process define/endef
     according to define parsing rules.
     """
 
     def checkfortoken(o):
         """
         Check for a define or endef token on the line starting at o.
         Return an integer for the direction of definecount.
         """
@@ -288,43 +288,43 @@ def iterdefinechars(d, offset):
 
         if c == '\n':
             d.readline()
             definecount += checkfortoken(offset + 1)
             if definecount == 0:
                 return
 
         if c == '\\' and offset < len(d.data) - 1 and d.data[offset+1] == '\n':
-            yield ' ', offset, d.getloc(offset)
+            yield ' ', offset
             d.readline()
             offset = d.skipwhitespace(offset + 2)
             continue
 
         if c.isspace():
             o = d.skipwhitespace(offset)
             if d.data[o:o+2] == '\\\n':
                 offset = o
                 continue
 
-        yield c, offset, d.getloc(offset)
+        yield c, offset
         offset += 1
 
 
     # Unlike the other iterators, if you fall off this one there is an unterminated
     # define.
     raise SyntaxError("Unterminated define", d.getloc(startoffset))
 
 def ensureend(d, offset, msg, ifunc=itermakefilechars):
     """
     Ensure that only whitespace remains in this data.
     """
 
-    for c, o, l in ifunc(d, offset):
+    for c, o in ifunc(d, offset):
         if not c.isspace():
-            raise SyntaxError(msg, d.getloc(offset))
+            raise SyntaxError(msg, d.getloc(o))
 
 def iterlines(fd):
     """Yield (lineno, line) for each line in fd"""
 
     lineno = 0
     for line in fd:
         lineno += 1
 
@@ -343,33 +343,33 @@ def setvariable(resolvevariables, setvar
     assert isinstance(setvariables, data.Variables)
 
     # print "setvariable: %r resvariables: %r setvariables: %r" % (vname, resolvevariables, setvariables)
 
     if len(vname) == 0:
         raise SyntaxError("Empty variable name", loc=d.getloc(offset))
 
     if token == '+=':
-        val = ''.join((c for c, o, l in iterfunc(d, offset)))
+        val = ''.join((c for c, o, in iterfunc(d, offset)))
         if skipwhitespace:
             val = val.lstrip()
         setvariables.append(vname, source, val, resolvevariables)
         return
 
     if token == '?=':
         flavor = data.Variables.FLAVOR_RECURSIVE
-        val = ''.join((c for c, o, l in iterfunc(d, offset)))
+        val = ''.join((c for c, o, in iterfunc(d, offset)))
         if skipwhitespace:
             val = val.lstrip()
         oldflavor, oldsource, oldval = setvariables.get(vname, expand=False)
         if oldval is not None:
             return
     elif token == '=':
         flavor = data.Variables.FLAVOR_RECURSIVE
-        val = ''.join((c for c, o, l in iterfunc(d, offset)))
+        val = ''.join((c for c, o, in iterfunc(d, offset)))
         if skipwhitespace:
             val = val.lstrip()
     else:
         assert token == ':='
 
         flavor = data.Variables.FLAVOR_SIMPLE
         e, t, o = parsemakesyntax(d, offset, (), itermakefilechars)
         if skipwhitespace:
@@ -780,17 +780,17 @@ def parsemakesyntax(d, startat, stopon, 
     ]
 
     di = iterfunc(d, startat)
     offset = startat
 
     while True: # this is not a for loop because `di` changes during the function
         stacktop = stack[-1]
         try:
-            c, offset, loc = di.next()
+            c, offset = di.next()
         except StopIteration:
             break
 
         # print "  %i: stacklen=%i parsestate=%s looking for %r" % (offset, len(stack),
         #                                                           stacktop.parsestate, stacktop.stopon),
 
         token, offset = d.findtoken(offset, stacktop.stopon, False)
         if token is not None:
@@ -848,18 +848,19 @@ def parsemakesyntax(d, startat, stopon, 
                 stack.pop()
                 stack[-1].expansion.append(functions.SubstitutionRef(stacktop.loc, stacktop.varname,
                                                                      stacktop.substfrom, stacktop.expansion))
             else:
                 assert False, "Unexpected parse state %s" % stacktop.parsestate
 
             continue
         elif c == '$':
+            loc = d.getloc(offset)
             try:
-                c, offset, loc = di.next()
+                c, offset = di.next()
             except StopIteration:
                 # an un-terminated $ expands to nothing
                 break
 
             if c == '$':
                 stacktop.expansion.append('$')
                 continue
 
@@ -891,13 +892,13 @@ def parsemakesyntax(d, startat, stopon, 
             fe.append(c)
             stacktop.expansion.append(functions.VariableRef(loc, fe))
             continue
 
         else:
             stacktop.expansion.append(c)
 
     if len(stack) != 1:
-        raise SyntaxError("Unterminated function call", d.getloc(len(d) - 1))
+        raise SyntaxError("Unterminated function call", d.getloc(offset))
 
     assert stack[0].parsestate == PARSESTATE_TOPLEVEL
 
     return stack[0].expansion, None, None