Implement the 'define' directive. The edge cases and parsing rules for define are, once again, ridiculous. I never thought I would yearn for heredoc syntax!
authorBenjamin Smedberg <benjamin@smedbergs.us>
Fri, 06 Feb 2009 15:28:42 -0500
changeset 54 10277ab28e313cee255512993f9498185f68378a
parent 53 3e2e23603c36b4e2274a5d50a213694a9c3eb647
child 55 0c7929287af48f3d1a787a5e52cb6943320ebb24
push id31
push userbsmedberg@mozilla.com
push dateFri, 06 Feb 2009 20:29:14 +0000
Implement the 'define' directive. The edge cases and parsing rules for define are, once again, ridiculous. I never thought I would yearn for heredoc syntax!
pymake/data.py
pymake/parser.py
--- a/pymake/data.py
+++ b/pymake/data.py
@@ -98,16 +98,21 @@ class Expansion(object):
             self[0] = self[0].lstrip()
 
     def rstrip(self):
         """Strip trailing literal whitespace from this expansion."""
         if len(self) > 0 and isinstance(self[-1], str):
             assert len(self) == 1 or not isinstance(self[-2], str), "Strings didn't fold"
             self[-1] = self[-1].rstrip()
 
+    def trimlastnewline(self):
+        """Strip only the last newline, if present."""
+        if len(self) > 0 and isinstance(self[-1], str) and self[-1][-1] == '\n':
+            self[-1] = self[-1][:-1]
+
     def resolve(self, variables, setting):
         """
         Resolve this variable into a value, by interpolating the value
         of other variables.
 
         @param setting (Variable instance) the variable currently
                being set, if any. Setting variables must avoid self-referential
                loops.
@@ -458,16 +463,37 @@ def setautomaticvariables(v, makefile, t
     # TODO '?'
     v.set('^', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC,
           Expansion.fromstring(' '.join(withoutdups(prerequisites))))
     v.set('+', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC,
           Expansion.fromstring(' '.join(prerequisites)))
     # TODO '|'
     # TODO all the D and F variants
 
+def splitcommand(command):
+    """
+    Using the esoteric rules, split command lines by unescaped newlines.
+    """
+    start = 0
+    i = 0
+    while i < len(command):
+        c = command[i]
+        if c == '\\':
+            i += 1
+        elif c == '\n':
+            yield command[start:i]
+            i += 1
+            start = i
+            continue
+
+        i += 1
+
+    if i > start:
+        yield command[start:i]
+
 class Rule(object):
     """
     A rule contains a list of prerequisites and a list of commands. It may also
     contain rule-specific variables. This rule may be associated with multiple targets.
     """
 
     def __init__(self, prereqs, doublecolon, loc):
         self.prerequisites = prereqs
@@ -486,19 +512,22 @@ class Rule(object):
         assert isinstance(target, Target)
 
         v = Variables(parent=target.variables)
         setautomaticvariables(v, makefile, target, self.prerequisites)
         # TODO: $* in non-pattern rules sucks
 
         for c in self.commands:
             cstring = c.resolve(v, None)
-            if cstring[0:1] == '@':
-                cstring = cstring[1:]
-            subprocess.check_call(cstring, shell=True)
+            for cline in splitcommand(cstring):
+                if cline[0:1] == '@':
+                    cline = cline[1:]
+                if not len(cline) or cline.isspace():
+                    continue
+                subprocess.check_call(cline, shell=True)
 
 class PatternRule(object):
     """
     An implicit rule or static pattern rule containing target patterns, prerequisite patterns,
     and a list of commands.
     """
 
     def __init__(self, targetpatterns, prerequisites, doublecolon, loc):
--- a/pymake/parser.py
+++ b/pymake/parser.py
@@ -116,17 +116,17 @@ class Data(object):
         self._locs.append( (len(self.data), loc) )
         self.data += data
 
     def getloc(self, offset):
         """
         Get the location of an offset within data.
         """
         if offset >= len(self.data):
-            raise IndexError("Invalid offset", offset)
+            offset = len(self.data) - 1
 
         begin, loc = findlast(lambda (o, l): o <= offset, self._locs)
         return loc + self.data[begin:offset]
 
 def _iterlines(fd):
     """Yield (lineno, line) for each line in fd"""
 
     lineno = 0
@@ -144,17 +144,17 @@ def getkeyword(d, offset):
     and followed by a white space. Return word, newoffset or None, None
     """
     i = offset
     while True:
         c = d[i]
         if c == '-' or (c >= 'a' and c <= 'z'):
             i += 1
             continue
-        if c is not None and c.isspace():
+        if i > offset and (c is None or c.isspace()):
             return d[offset:i], skipwhitespace(d, i)
         return None, i
 
 def skipwhitespace(d, offset):
     """
     Return the offset into data after skipping whitespace.
     """
     while True:
@@ -201,58 +201,58 @@ def parsecommandlineargs(makefile, args)
             if a[eqpos-1] == ':':
                 vname = a[:eqpos-1]
             else:
                 vname = a[:eqpos]
             vname = vname.strip()
             valtext = a[eqpos+1:].lstrip()
             d = Data(None, None)
             d.append(valtext, Location('<command-line>', 1, eqpos + 1))
-            value, offset = parsemakesyntax(d, 0, '', iscommand=False)
+            value, offset = parsemakesyntax(d, 0, '', PARSESTYLE_MAKEFILE)
             assert offset == -1
             setvariable(makefile.variables, vname, a[eqpos-1] != ':', value, True)
         else:
             r.append(a)
 
     return r
 
 def ifeq(d, offset, makefile):
     # the variety of formats for this directive is rather maddening
     if d[offset] == '(':
-        arg1, offset = parsemakesyntax(d, offset + 1, ',', iscommand=False)
+        arg1, offset = parsemakesyntax(d, offset + 1, ',', PARSESTYLE_MAKEFILE)
         if offset == -1:
             raise SyntaxError("Unexpected text in conditional", d.getloc(len(d) - 1))
         offset = skipwhitespace(d, offset + 1)
-        arg2, offset = parsemakesyntax(d, offset, ')', iscommand=False)
+        arg2, offset = parsemakesyntax(d, offset, ')', PARSESTYLE_MAKEFILE)
         if offset == -1:
             raise SyntaxError("Unexpected text in conditional", d.getloc(len(d) - 1))
         offset = skipwhitespace(d, offset + 1)
         if d[offset] not in ('#', None):
             raise SyntaxError("Unexpected text after conditional", d.getloc(offset))
     elif d[offset] in '\'"':
-        arg1, offset = parsemakesyntax(d, offset + 1, d[offset], iscommand=False)
+        arg1, offset = parsemakesyntax(d, offset + 1, d[offset], PARSESTYLE_MAKEFILE)
         if offset == -1:
             raise SyntaxError("Unexpected text in conditional", d.getloc(len(d) - 1))
         offset = skipwhitespace(d, offset + 1)
         if d[offset] not in '\'"':
             raise SyntaxError("Unexpected text in conditional", d.getloc(offset))
-        arg2, offset = parsemakesyntax(d, offset + 1, d[offset], iscommand=False)
+        arg2, offset = parsemakesyntax(d, offset + 1, d[offset], PARSESTYLE_MAKEFILE)
         offset = skipwhitespace(d, offset + 1)
         if d[offset] not in ('#', None):
             raise SyntaxError("Unexpected text after conditional: %c" % (d[offset],), d.getloc(offset))
 
     val1 = arg1.resolve(makefile.variables, None)
     val2 = arg2.resolve(makefile.variables, None)
     return val1 == val2
 
 def ifneq(d, offset, makefile):
     return not ifeq(d, offset, makefile)
 
 def ifdef(d, offset, makefile):
-    e, offset = parsemakesyntax(d, offset, '', iscommand=False)
+    e, offset = parsemakesyntax(d, offset, '', PARSESTYLE_MAKEFILE)
     e.rstrip()
 
     vname = e.resolve(makefile.variables, None)
 
     flavor, source, value = makefile.variables.get(vname)
 
     if value is None:
         return False
@@ -305,17 +305,17 @@ def parsestream(fd, filename, makefile):
     for lineno, line in fdlines:
         d = Data(fdlines, filename)
         if line.startswith('\t') and currule is not None:
             if any((not c.active for c in condstack)):
                 log.info('skipping line %i, ifdefed away' % lineno)
                 continue
 
             d.append(line[1:], Location(filename, lineno, tabwidth))
-            e, stoppedat = parsemakesyntax(d, 0, '', iscommand=True)
+            e, stoppedat = parsemakesyntax(d, 0, '', PARSESTYLE_COMMAND)
             assert stoppedat == -1
             currule.addcommand(e)
         else:
             # To parse Makefile syntax, we first strip leading whitespace and
             # look for initial keywords. If there are no keywords, it's either
             # setting a variable or writing a rule.
 
             d = Data(fdlines, filename)
@@ -350,48 +350,58 @@ def parsestream(fd, filename, makefile):
                 else:
                     if kword not in conditionkeywords:
                         raise SyntaxError("Unexpected condition after 'else' directive.",
                                           d.getloc(kwoffset))
                         
                     m = conditionkeywords[kword](d, kwoffset, makefile)
                     condstack[-1].makeactive(m)
                 continue
+            elif kword == 'endef':
+                raise SyntaxError("Extraneous endef", d.getloc(0))
             elif kword == 'override':
                 raise NotImplementedError('no overrides yet')
             elif kword == 'define':
-                raise NotImplementedError('no defines yet')
+                e, i = parsemakesyntax(d, kwoffset, '', PARSESTYLE_MAKEFILE)
+                i = len(d)
+                d.readline()
+                val, i = parsemakesyntax(d, i, '', PARSESTYLE_DEFINE)
+
+                vname = e.resolve(makefile.variables, None)
+                makefile.variables.set(vname, data.Variables.FLAVOR_RECURSIVE,
+                                       data.Variables.SOURCE_MAKEFILE, val)
+                continue
             elif kword == 'include':
                 raise NotImplementedError('no includes yet')
             elif kword in conditionkeywords:
                 m = conditionkeywords[kword](d, kwoffset, makefile)
                 condstack.append(Condition(m))
                 continue
 
             if any((not c.active for c in condstack)):
                 log.info('skipping line %i, ifdefed away' % lineno)
                 continue
 
-            e, stoppedat = parsemakesyntax(d, 0, ':=', iscommand=False)
+            e, stoppedat = parsemakesyntax(d, 0, ':=', PARSESTYLE_MAKEFILE)
             if stoppedat == -1:
                 v = e.resolve(makefile.variables, None)
                 if v.strip() != '':
                     raise SyntaxError("Bad syntax: non-empty line is not a variable assignment or rule.", loc=d.getloc(0))
                 continue
 
             # if we encountered real makefile syntax, the current rule is over
             currule = None
 
             if d[stoppedat] == '=' or d[stoppedat:stoppedat+2] == ':=':
                 isrecursive = d[stoppedat] == '='
 
                 e.lstrip()
                 e.rstrip()
                 vname = e.resolve(makefile.variables, None)
-                value, stoppedat = parsemakesyntax(d, stoppedat + (isrecursive and 1 or 2), '', iscommand=False)
+                value, stoppedat = parsemakesyntax(d, stoppedat + (isrecursive and 1 or 2), '', PARSESTYLE_MAKEFILE)
                 assert stoppedat == -1
                 value.lstrip()
                 setvariable(makefile.variables, vname, isrecursive, value)
             else:
                 assert d[stoppedat] == ':'
 
                 if d[stoppedat+1] == ':':
                     doublecolon = True
@@ -412,39 +422,39 @@ def parsestream(fd, filename, makefile):
                     raise SyntaxError("No targets in rule", g.getloc(0))
 
                 ispatterns = set((t.ispattern() for t in targets))
                 if len(ispatterns) == 2:
                     raise SyntaxError("Mixed implicit and normal rule", d.getloc(0))
                 ispattern, = ispatterns
 
                 stoppedat += 1
-                e, stoppedat = parsemakesyntax(d, stoppedat, ':=|;', iscommand=False)
+                e, stoppedat = parsemakesyntax(d, stoppedat, ':=|;', PARSESTYLE_MAKEFILE)
                 if stoppedat == -1 or d[stoppedat] == ';':
                     prereqs = data.splitwords(e.resolve(makefile.variables, None))
                     if ispattern:
                         currule = data.PatternRule(targets, map(data.Pattern, prereqs), doublecolon, loc=d.getloc(0))
                         makefile.appendimplicitrule(currule)
                     else:
                         currule = data.Rule(prereqs, doublecolon, loc=d.getloc(0))
                         for t in targets:
                             makefile.gettarget(t.gettarget()).addrule(currule)
                         makefile.foundtarget(targets[0].gettarget())
 
                     if stoppedat != -1:
-                        e, stoppedat = parsemakesyntax(d, stoppedat + 1, '', iscommand=True)
+                        e, stoppedat = parsemakesyntax(d, stoppedat + 1, '', PARSESTYLE_COMMAND)
                         assert stoppedat == -1
                         e.lstrip()
                         currule.addcommand(e)
                 elif d[stoppedat] == '=' or d[stoppedat:stoppedat+2] == ':=':
                     isrecursive = d[stoppedat] == '='
                     e.lstrip()
                     e.rstrip()
                     vname = e.resolve(makefile.variables, None)
-                    value, stoppedat = parsemakesyntax(d, stoppedat + (isrecursive and 1 or 2), '', iscommand=False)
+                    value, stoppedat = parsemakesyntax(d, stoppedat + (isrecursive and 1 or 2), '', PARSESTYLE_MAKEFILE)
                     assert stoppedat == -1
                     value.lstrip()
 
                     if ispattern:
                         for target in targets:
                             setvariable(makefile.getpatternvariables(target), vname, isrecursive, value)
                     else:
                         for target in targets:
@@ -460,26 +470,26 @@ def parsestream(fd, filename, makefile):
 
                     patstr = e.resolve(makefile.variables, None)
                     patterns = data.splitwords(patstr)
                     if len(patterns) != 1:
                         raise SyntaxError("A static pattern rule may have only one pattern", d.getloc(stoppedat))
 
                     pattern = data.Pattern(patterns[0])
 
-                    e, stoppedat = parsemakesyntax(d, stoppedat + 1, ';', iscommand=False)
+                    e, stoppedat = parsemakesyntax(d, stoppedat + 1, ';', PARSESTYLE_MAKEFILE)
                     prereqs = map(data.Pattern, data.splitwords(e.resolve(makefile.variables, None)))
                     currule = data.PatternRule([pattern], prereqs, doublecolon, loc=d.getloc(0))
                     for t in targets:
                         makefile.gettarget(t.gettarget()).addrule(currule)
 
                     makefile.foundtarget(targets[0].gettarget())
 
                     if stoppedat != -1:
-                        e, stoppedat = parsemakesyntax(d, stoppedat + 1, '', iscommand=True)
+                        e, stoppedat = parsemakesyntax(d, stoppedat + 1, '', PARSESTYLE_COMMAND)
                         assert stoppedat == -1
                         e.lstrip()
                         currule.addcommand(e)
 
 PARSESTATE_TOPLEVEL = 0    # at the top level
 PARSESTATE_FUNCTION = 1    # expanding a function call. data is function
 
 # For the following three, data is a tuple of Expansions: (varname, substfrom, substto)
@@ -490,40 +500,69 @@ PARSESTATE_SUBSTTO = 4     # expanding a
 class ParseStackFrame(object):
     def __init__(self, parsestate, expansion, stopon, **kwargs):
         self.parsestate = parsestate
         self.expansion = expansion
         self.stopon = stopon
         for key, value in kwargs.iteritems():
             setattr(self, key, value)
 
-def parsemakesyntax(d, startat, stopon, iscommand):
+PARSESTYLE_COMMAND = 0
+PARSESTYLE_MAKEFILE = 1
+PARSESTYLE_DEFINE = 2
+
+def parsemakesyntax(d, startat, stopon, parsestyle):
     """
     Given Data, parse it into a data.Expansion.
 
     @param stopon (sequence)
         Indicate characters where toplevel parsing should stop.
  
     @return a tuple (expansion, stopoffset). If all the data is consumed, stopoffset will be -1
     """
 
-    # print "parsemakesyntax iscommand=%s" % iscommand
+    assert parsestyle in (PARSESTYLE_COMMAND, PARSESTYLE_MAKEFILE, PARSESTYLE_DEFINE)
 
     stack = [
         ParseStackFrame(PARSESTATE_TOPLEVEL, data.Expansion(), stopon)
     ]
 
+    if parsestyle == PARSESTYLE_DEFINE:
+        definecount = 1
+        linebegin = True
+
     i = startat
     while i < len(d):
         stacktop = stack[-1]
         c = d[i]
 
         # print "i=%i c=%c parsestate=%i len(d)=%i" % (i, c, stacktop.parsestate, len(d))
 
-        if c == '#' and not iscommand:
+        if parsestyle == PARSESTYLE_DEFINE and linebegin:
+            linebegin = False
+            if d[i] != '\t':
+                # look for endef/define
+                j = skipwhitespace(d, i)
+                kword, j = getkeyword(d, j)
+                if kword == 'define':
+                    print "incrementing definecount at %s" % d.getloc(j)
+                    definecount += 1
+                elif kword == 'endef':
+                    if not d[j] in ('#', None):
+                        raise SyntaxError("Extraneous text after endef directive.",
+                                          d.getloc(j))
+
+
+                    print "decrementing definecount at %s" % d.getloc(j)
+                    definecount -= 1
+                    if definecount == 0:
+                        stacktop.expansion.trimlastnewline()
+                        break
+
+        if c == '#' and parsestyle == PARSESTYLE_MAKEFILE:
             # we need to keep reading lines until there are no more continuations
             while i < len(d):
                 if d[i] == '\\':
                     if d[i+1] == '\\':
                         i += 2
                         continue
                     elif d[i+1] == '\n':
                         i += 2
@@ -536,53 +575,61 @@ def parsemakesyntax(d, startat, stopon, 
                 i += 1
             break
         elif c == '\\':
             # in makefile syntax, backslashes can escape # specially, but nothing else. Fun, huh?
             if d[i+1] is None:
                 stacktop.expansion.append('\\')
                 i += 1
                 break
-            elif d[i+1] == '#' and not iscommand:
+            elif d[i+1] == '#' and parsestyle == PARSESTYLE_MAKEFILE:
                 stacktop.expansion.append('#')
                 i += 2
                 continue
             elif d[i+1:i+3] == '\\#':
                 # This is an edge case that I discovered. It's undocumented, and
                 # totally absolutely weird. See escape-chars.mk VARAWFUL
                 stacktop.expansion.append('\\')
                 i += 2
                 continue
-            elif d[i+1] == '\\' and not iscommand:
+            elif d[i+1] == '\\' and parsestyle == PARSESTYLE_MAKEFILE:
                 stacktop.expansion.append('\\\\')
                 i += 2
                 continue
             elif d[i+1] == '\n':
                 i += 2
                 assert i == len(d), "newline isn't last character?"
 
                 d.readline()
 
-                if iscommand:
+                if parsestyle == PARSESTYLE_COMMAND:
                     stacktop.expansion.append('\\\n')
                     if d[i] == '\t':
                         i += 1
                 else:
                     stacktop.expansion.rstrip()
                     stacktop.expansion.append(' ')
                     i = skipwhitespace(d, i)
                 continue
             else:
                 stacktop.expansion.append(c)
                 i += 1
                 continue
         elif c == '\n':
-            i += 1
-            assert i == len(d), "newline isn't last character?"
-            break
+            assert i + 1 == len(d), "newline isn't last character? i = %i d = %r" % (i, d.data)
+            if parsestyle == PARSESTYLE_DEFINE:
+                stacktop.expansion.append(c)
+                i += 1
+
+                d.readline()
+                linebegin = True
+                continue
+            else:
+                i += 1
+                break
         elif c == '$':
             loc = d.getloc(i)
             i += 1
             c = d[i]
 
             if c == '$':
                 stacktop.expansion.append('$')
             elif c == '(':