Bug 1474369 - Part 3: Add generic type parsing support to xpidl, r=mccr8
authorNika Layzell <nika@thelayzells.com>
Tue, 10 Jul 2018 21:15:16 -0400
changeset 429472 ff1530e9970a964991b9e9960d54dde48ed0c62f
parent 429471 aaadb94f144ded2ae6350701de6f8fd3edef5c39
child 429473 4c78a885c77da5eb250e8fdeeda5b7270730a169
push id105907
push usernika@thelayzells.com
push dateTue, 31 Jul 2018 21:53:40 +0000
treeherdermozilla-inbound@69e4230dc38a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmccr8
bugs1474369
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1474369 - Part 3: Add generic type parsing support to xpidl, r=mccr8 Summary: This patch allows parsing generic types, such as Sequence<T>, in XPIDL. It does this by introducing a new type, TypeId, which contains both the name string and an optional list of generic parameters. Various places which use the xpidl.py library had to be updated to construct one of these TypeId objects, as TypeId and `str` are not compatible types. Depends On D2106 Reviewers: mccr8! Tags: #secure-revision Bug #: 1474369 Differential Revision: https://phabricator.services.mozilla.com/D2109
accessible/xpcom/AccEventGen.py
xpcom/idl-parser/xpidl/runtests.py
xpcom/idl-parser/xpidl/xpidl.py
--- a/accessible/xpcom/AccEventGen.py
+++ b/accessible/xpcom/AccEventGen.py
@@ -192,17 +192,17 @@ def writeAttributeGetter(fd, classname, 
     fd.write("  return NS_OK;\n")
     fd.write("}\n\n")
 
 
 def interfaces(iface):
     interfaces = []
     while iface.base:
         interfaces.append(iface)
-        iface = iface.idl.getName(iface.base, iface.location)
+        iface = iface.idl.getName(xpidl.TypeId(iface.base), iface.location)
     interfaces.append(iface)
     interfaces.reverse()
     return interfaces
 
 
 def allAttributes(iface):
     attributes = []
     for i in interfaces(iface):
--- a/xpcom/idl-parser/xpidl/runtests.py
+++ b/xpcom/idl-parser/xpidl/runtests.py
@@ -55,51 +55,51 @@ class TestParser(unittest.TestCase):
 void bar();
 };""", filename='f')
         self.assertTrue(isinstance(i, xpidl.IDL))
         self.assertTrue(isinstance(i.productions[0], xpidl.Interface))
         iface = i.productions[0]
         m = iface.members[0]
         self.assertTrue(isinstance(m, xpidl.Method))
         self.assertEqual("bar", m.name)
-        self.assertEqual("void", m.type)
+        self.assertEqual(xpidl.TypeId("void"), m.type)
 
     def testMethodParams(self):
         i = self.p.parse("""[uuid(abc)] interface foo {
 long bar(in long a, in float b, [array] in long c);
 };""", filename='f')
         i.resolve([], self.p, {})
         self.assertTrue(isinstance(i, xpidl.IDL))
         self.assertTrue(isinstance(i.productions[0], xpidl.Interface))
         iface = i.productions[0]
         m = iface.members[0]
         self.assertTrue(isinstance(m, xpidl.Method))
         self.assertEqual("bar", m.name)
-        self.assertEqual("long", m.type)
+        self.assertEqual(xpidl.TypeId("long"), m.type)
         self.assertEqual(3, len(m.params))
-        self.assertEqual("long", m.params[0].type)
+        self.assertEqual(xpidl.TypeId("long"), m.params[0].type)
         self.assertEqual("in", m.params[0].paramtype)
-        self.assertEqual("float", m.params[1].type)
+        self.assertEqual(xpidl.TypeId("float"), m.params[1].type)
         self.assertEqual("in", m.params[1].paramtype)
-        self.assertEqual("long", m.params[2].type)
+        self.assertEqual(xpidl.TypeId("long"), m.params[2].type)
         self.assertEqual("in", m.params[2].paramtype)
         self.assertTrue(isinstance(m.params[2].realtype, xpidl.Array))
         self.assertEqual("long", m.params[2].realtype.type.name)
 
     def testAttribute(self):
         i = self.p.parse("""[uuid(abc)] interface foo {
 attribute long bar;
 };""", filename='f')
         self.assertTrue(isinstance(i, xpidl.IDL))
         self.assertTrue(isinstance(i.productions[0], xpidl.Interface))
         iface = i.productions[0]
         a = iface.members[0]
         self.assertTrue(isinstance(a, xpidl.Attribute))
         self.assertEqual("bar", a.name)
-        self.assertEqual("long", a.type)
+        self.assertEqual(xpidl.TypeId("long"), a.type)
 
     def testOverloadedVirtual(self):
         i = self.p.parse("""[uuid(abc)] interface foo {
 attribute long bar;
 void getBar();
 };""", filename='f')
         self.assertTrue(isinstance(i, xpidl.IDL))
         i.resolve([], self.p, {})
--- a/xpcom/idl-parser/xpidl/xpidl.py
+++ b/xpcom/idl-parser/xpidl/xpidl.py
@@ -7,16 +7,17 @@
 
 """A parser for cross-platform IDL (XPIDL) files."""
 
 import sys
 import os.path
 import re
 from ply import lex
 from ply import yacc
+from collections import namedtuple
 
 """A type conforms to the following pattern:
 
     def isScriptable(self):
         'returns True or False'
 
     def nativeType(self, calltype):
         'returns a string representation of the native type
@@ -323,20 +324,23 @@ class IDL(object):
     def __init__(self, productions):
         self.productions = productions
         self.deps = []
 
     def setName(self, object):
         self.namemap.set(object)
 
     def getName(self, id, location):
+        if id.params is not None:
+            raise IDLError("Generic type '%s' unrecognized" % id.name, location)
+
         try:
-            return self.namemap[id]
+            return self.namemap[id.name]
         except KeyError:
-            raise IDLError("type '%s' not found" % id, location)
+            raise IDLError("type '%s' not found" % id.name, location)
 
     def hasName(self, id):
         return id in self.namemap
 
     def getNames(self):
         return iter(self.namemap)
 
     def __str__(self):
@@ -658,32 +662,32 @@ class Interface(object):
 
         # Hack alert: if an identifier is already present, libIDL assigns
         # doc comments incorrectly. This is quirks-mode extraordinaire!
         if parent.hasName(self.name):
             for member in self.members:
                 if hasattr(member, 'doccomments'):
                     member.doccomments[0:0] = self.doccomments
                     break
-            self.doccomments = parent.getName(self.name, None).doccomments
+            self.doccomments = parent.getName(TypeId(self.name), None).doccomments
 
         if self.attributes.function:
             has_method = False
             for member in self.members:
                 if member.kind is 'method':
                     if has_method:
                         raise IDLError(
                             "interface '%s' has multiple methods, but marked 'function'" %
                             self.name, self.location)
                     else:
                         has_method = True
 
         parent.setName(self)
         if self.base is not None:
-            realbase = parent.getName(self.base, self.location)
+            realbase = parent.getName(TypeId(self.base), self.location)
             if realbase.kind != 'interface':
                 raise IDLError("interface '%s' inherits from non-interface type '%s'" %
                                (self.name, self.base), self.location)
 
             if self.attributes.scriptable and not realbase.attributes.scriptable:
                 raise IDLError("interface '%s' is scriptable but derives from "
                                "non-scriptable '%s'" %
                                (self.name, self.base), self.location, warning=True)
@@ -732,38 +736,38 @@ class Interface(object):
             for m in self.members:
                 l.append(str(m))
         return "".join(l)
 
     def getConst(self, name, location):
         # The constant may be in a base class
         iface = self
         while name not in iface.namemap and iface is not None:
-            iface = self.idl.getName(self.base, self.location)
+            iface = self.idl.getName(TypeId(self.base), self.location)
         if iface is None:
             raise IDLError("cannot find symbol '%s'" % name)
         c = iface.namemap.get(name, location)
         if c.kind != 'const':
             raise IDLError("symbol '%s' is not a constant", c.location)
 
         return c.getValue()
 
     def needsJSTypes(self):
         for m in self.members:
-            if m.kind == "attribute" and m.type == "jsval":
+            if m.kind == "attribute" and m.type == TypeId("jsval"):
                 return True
             if m.kind == "method" and m.needsJSTypes():
                 return True
         return False
 
     def countEntries(self):
         ''' Returns the number of entries in the vtable for this interface. '''
         total = sum(member.count() for member in self.members)
         if self.base is not None:
-            realbase = self.idl.getName(self.base, self.location)
+            realbase = self.idl.getName(TypeId(self.base), self.location)
             total += realbase.countEntries()
         return total
 
 
 class InterfaceAttributes(object):
     uuid = None
     scriptable = False
     builtinclass = False
@@ -1079,17 +1083,17 @@ class Method(object):
                                     self.name,
                                     ", ".join([p.toIDL()
                                                for p in self.params]),
                                     raises)
 
     def needsJSTypes(self):
         if self.implicit_jscontext:
             return True
-        if self.type == "jsval":
+        if self.type == TypeId("jsval"):
             return True
         for p in self.params:
             t = p.realtype
             if isinstance(t, Native) and t.specialtype == "jsval":
                 return True
         return False
 
     def count(self):
@@ -1229,16 +1233,30 @@ class Array(object):
                             '*' if 'out' in calltype else '')
 
     def rustType(self, calltype, const=False):
         return "%s%s%s" % ('*mut ' if 'out' in calltype else '',
                            '*const ' if const else '*mut ',
                            self.type.rustType('element'))
 
 
+TypeId = namedtuple('TypeId', 'name params')
+
+
+# Make str(TypeId) produce a nicer value
+TypeId.__str__ = lambda self: \
+    "%s<%s>" % (self.name, ', '.join(str(p) for p in self.params)) \
+    if self.params is not None \
+    else self.name
+
+
+# Allow skipping 'params' in TypeId(..)
+TypeId.__new__.__defaults__ = (None,)
+
+
 class IDLParser(object):
     keywords = {
         'const': 'CONST',
         'interface': 'INTERFACE',
         'in': 'IN',
         'inout': 'INOUT',
         'out': 'OUT',
         'attribute': 'ATTRIBUTE',
@@ -1269,17 +1287,17 @@ class IDLParser(object):
 
     hexchar = r'[a-fA-F0-9]'
 
     t_NUMBER = r'-?\d+'
     t_HEXNUM = r'0x%s+' % hexchar
     t_LSHIFT = r'<<'
     t_RSHIFT = r'>>'
 
-    literals = '"(){}[],;:=|+-*'
+    literals = '"(){}[]<>,;:=|+-*'
 
     t_ignore = ' \t'
 
     def t_multilinecomment(self, t):
         r'/\*(?s).*?\*/'
         t.lexer.lineno += t.value.count('\n')
         if t.value.startswith("/**"):
             self._doccomments.append(t.value)
@@ -1362,17 +1380,17 @@ class IDLParser(object):
         """productions : interface productions
                        | typedef productions
                        | native productions
                        | webidl productions"""
         p[0] = list(p[2])
         p[0].insert(0, p[1])
 
     def p_typedef(self, p):
-        """typedef : TYPEDEF IDENTIFIER IDENTIFIER ';'"""
+        """typedef : TYPEDEF type IDENTIFIER ';'"""
         p[0] = Typedef(type=p[2],
                        name=p[3],
                        location=self.getLocation(p, 1),
                        doccomments=p.slice[1].doccomments)
 
     def p_native(self, p):
         """native : attributes NATIVE IDENTIFIER afternativeid '(' NATIVEID ')' ';'"""
         p[0] = Native(name=p[3],
@@ -1475,17 +1493,17 @@ class IDLParser(object):
         p[0] = list(p[2])
         p[0].insert(0, p[1])
 
     def p_member_cdata(self, p):
         """member : CDATA"""
         p[0] = CDATA(p[1], self.getLocation(p, 1))
 
     def p_member_const(self, p):
-        """member : CONST IDENTIFIER IDENTIFIER '=' number ';' """
+        """member : CONST type IDENTIFIER '=' number ';' """
         p[0] = ConstMember(type=p[2], name=p[3],
                            value=p[5], location=self.getLocation(p, 1),
                            doccomments=p.slice[1].doccomments)
 
 # All "number" products return a function(interface)
 
     def p_number_decimal(self, p):
         """number : NUMBER"""
@@ -1537,33 +1555,33 @@ class IDLParser(object):
 
     def p_number_bitor(self, p):
         """number : number '|' number"""
         n1 = p[1]
         n2 = p[3]
         p[0] = lambda i: n1(i) | n2(i)
 
     def p_member_att(self, p):
-        """member : attributes optreadonly ATTRIBUTE IDENTIFIER IDENTIFIER ';'"""
+        """member : attributes optreadonly ATTRIBUTE type IDENTIFIER ';'"""
         if 'doccomments' in p[1]:
             doccomments = p[1]['doccomments']
         elif p[2] is not None:
             doccomments = p[2]
         else:
             doccomments = p.slice[3].doccomments
 
         p[0] = Attribute(type=p[4],
                          name=p[5],
                          attlist=p[1]['attlist'],
                          readonly=p[2] is not None,
                          location=self.getLocation(p, 3),
                          doccomments=doccomments)
 
     def p_member_method(self, p):
-        """member : attributes IDENTIFIER IDENTIFIER '(' paramlist ')' raises ';'"""
+        """member : attributes type IDENTIFIER '(' paramlist ')' raises ';'"""
         if 'doccomments' in p[1]:
             doccomments = p[1]['doccomments']
         else:
             doccomments = p.slice[2].doccomments
 
         p[0] = Method(type=p[2],
                       name=p[3],
                       attlist=p[1]['attlist'],
@@ -1586,17 +1604,17 @@ class IDLParser(object):
         p[0] = []
 
     def p_moreparams_continue(self, p):
         """moreparams : ',' param moreparams"""
         p[0] = list(p[3])
         p[0].insert(0, p[2])
 
     def p_param(self, p):
-        """param : attributes paramtype IDENTIFIER IDENTIFIER"""
+        """param : attributes paramtype type IDENTIFIER"""
         p[0] = Param(paramtype=p[2],
                      type=p[3],
                      name=p[4],
                      attlist=p[1]['attlist'],
                      location=self.getLocation(p, 3))
 
     def p_paramtype(self, p):
         """paramtype : IN
@@ -1624,16 +1642,35 @@ class IDLParser(object):
         """idlist : IDENTIFIER"""
         p[0] = [p[1]]
 
     def p_idlist_continue(self, p):
         """idlist : IDENTIFIER ',' idlist"""
         p[0] = list(p[3])
         p[0].insert(0, p[1])
 
+    def p_type_id(self, p):
+        """type : IDENTIFIER"""
+        p[0] = TypeId(name=p[1])
+        p.slice[0].doccomments = p.slice[1].doccomments
+
+    def p_type_generic(self, p):
+        """type : IDENTIFIER '<' typelist '>'"""
+        p[0] = TypeId(name=p[1], params=p[3])
+        p.slice[0].doccomments = p.slice[1].doccomments
+
+    def p_typelist(self, p):
+        """typelist : type"""
+        p[0] = [p[1]]
+
+    def p_typelist_continue(self, p):
+        """typelist : type ',' typelist"""
+        p[0] = list(p[3])
+        p[0].insert(0, p[1])
+
     def p_error(self, t):
         if not t:
             raise IDLError(
                 "Syntax Error at end of file. Possibly due to missing semicolon(;), braces(}) "
                 "or both", None)
         else:
             location = Location(self.lexer, t.lineno, t.lexpos)
             raise IDLError("invalid syntax", location)