Bug 458936: Replace xpidlc's typelib generation with a python equivalent. r=ted
authorKyle Huey <khuey@kylehuey.com>
Tue, 09 Aug 2011 20:48:41 -0400
changeset 74180 1a9b3ace8c68e0c53589900f34e0a60cb178d564
parent 74179 4aa92c4a5f135cf1ebbc5f327d0a3e884482f896
child 74181 b8ed3bb885a7b1a807550c7c4cd949bb97b65ca9
push idunknown
push userunknown
push dateunknown
reviewersted
bugs458936
milestone8.0a1
Bug 458936: Replace xpidlc's typelib generation with a python equivalent. r=ted
config/rules.mk
js/src/config/rules.mk
xpcom/idl-parser/typelib.py
xpcom/idl-parser/xpidl.py
xpcom/typelib/xpt/tools/xpt.py
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -1527,34 +1527,39 @@ endif
 	$(MKDIR) -p $(XPIDL_GEN_DIR)
 	@$(TOUCH) $@
 
 # don't depend on $(XPIDL_GEN_DIR), because the modification date changes
 # with any addition to the directory, regenerating all .h files -> everything.
 
 XPIDL_DEPS = \
   $(topsrcdir)/xpcom/idl-parser/header.py \
+  $(topsrcdir)/xpcom/idl-parser/typelib.py \
   $(topsrcdir)/xpcom/idl-parser/xpidl.py \
   $(NULL)
 
 $(XPIDL_GEN_DIR)/%.h: %.idl $(XPIDL_DEPS) $(XPIDL_GEN_DIR)/.done
 	$(REPORT_BUILD)
 	$(PYTHON_PATH) \
 	  -I$(topsrcdir)/other-licenses/ply \
 	  -I$(topsrcdir)/xpcom/idl-parser \
 	  $(topsrcdir)/xpcom/idl-parser/header.py --cachedir=$(topsrcdir)/xpcom/idl-parser $(XPIDL_FLAGS) $(_VPATH_SRCS) -d $(MDDEPDIR)/$(@F).pp -o $@
 	@if test -n "$(findstring $*.h, $(EXPORTS))"; \
 	  then echo "*** WARNING: file $*.h generated from $*.idl overrides $(srcdir)/$*.h"; else true; fi
 
 ifndef NO_GEN_XPT
 # generate intermediate .xpt files into $(XPIDL_GEN_DIR), then link
 # into $(XPIDL_MODULE).xpt and export it to $(FINAL_TARGET)/components.
-$(XPIDL_GEN_DIR)/%.xpt: %.idl $(XPIDL_COMPILE) $(XPIDL_GEN_DIR)/.done
+$(XPIDL_GEN_DIR)/%.xpt: %.idl $(XPIDL_DEPS) $(XPIDL_GEN_DIR)/.done
 	$(REPORT_BUILD)
-	$(ELOG) $(XPIDL_COMPILE) -m typelib -w $(XPIDL_FLAGS) -e $@ -d $(MDDEPDIR)/$(@F).pp $(_VPATH_SRCS)
+	$(PYTHON_PATH) \
+	  -I$(topsrcdir)/other-licenses/ply \
+	  -I$(topsrcdir)/xpcom/idl-parser \
+	  -I$(topsrcdir)/xpcom/typelib/xpt/tools \
+	  $(topsrcdir)/xpcom/idl-parser/typelib.py --cachedir=$(topsrcdir)/xpcom/idl-parser $(XPIDL_FLAGS) $(_VPATH_SRCS) -d $(MDDEPDIR)/$(@F).pp -o $@
 
 # no need to link together if XPIDLSRCS contains only XPIDL_MODULE
 ifneq ($(XPIDL_MODULE).idl,$(strip $(XPIDLSRCS)))
 $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt: $(patsubst %.idl,$(XPIDL_GEN_DIR)/%.xpt,$(XPIDLSRCS)) $(GLOBAL_DEPS)
 	$(XPIDL_LINK) $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt $(patsubst %.idl,$(XPIDL_GEN_DIR)/%.xpt,$(XPIDLSRCS))
 endif # XPIDL_MODULE.xpt != XPIDLSRCS
 
 libs:: $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt
--- a/js/src/config/rules.mk
+++ b/js/src/config/rules.mk
@@ -1527,34 +1527,39 @@ endif
 	$(MKDIR) -p $(XPIDL_GEN_DIR)
 	@$(TOUCH) $@
 
 # don't depend on $(XPIDL_GEN_DIR), because the modification date changes
 # with any addition to the directory, regenerating all .h files -> everything.
 
 XPIDL_DEPS = \
   $(topsrcdir)/xpcom/idl-parser/header.py \
+  $(topsrcdir)/xpcom/idl-parser/typelib.py \
   $(topsrcdir)/xpcom/idl-parser/xpidl.py \
   $(NULL)
 
 $(XPIDL_GEN_DIR)/%.h: %.idl $(XPIDL_DEPS) $(XPIDL_GEN_DIR)/.done
 	$(REPORT_BUILD)
 	$(PYTHON_PATH) \
 	  -I$(topsrcdir)/other-licenses/ply \
 	  -I$(topsrcdir)/xpcom/idl-parser \
 	  $(topsrcdir)/xpcom/idl-parser/header.py --cachedir=$(topsrcdir)/xpcom/idl-parser $(XPIDL_FLAGS) $(_VPATH_SRCS) -d $(MDDEPDIR)/$(@F).pp -o $@
 	@if test -n "$(findstring $*.h, $(EXPORTS))"; \
 	  then echo "*** WARNING: file $*.h generated from $*.idl overrides $(srcdir)/$*.h"; else true; fi
 
 ifndef NO_GEN_XPT
 # generate intermediate .xpt files into $(XPIDL_GEN_DIR), then link
 # into $(XPIDL_MODULE).xpt and export it to $(FINAL_TARGET)/components.
-$(XPIDL_GEN_DIR)/%.xpt: %.idl $(XPIDL_COMPILE) $(XPIDL_GEN_DIR)/.done
+$(XPIDL_GEN_DIR)/%.xpt: %.idl $(XPIDL_DEPS) $(XPIDL_GEN_DIR)/.done
 	$(REPORT_BUILD)
-	$(ELOG) $(XPIDL_COMPILE) -m typelib -w $(XPIDL_FLAGS) -e $@ -d $(MDDEPDIR)/$(@F).pp $(_VPATH_SRCS)
+	$(PYTHON_PATH) \
+	  -I$(topsrcdir)/other-licenses/ply \
+	  -I$(topsrcdir)/xpcom/idl-parser \
+	  -I$(topsrcdir)/xpcom/typelib/xpt/tools \
+	  $(topsrcdir)/xpcom/idl-parser/typelib.py --cachedir=$(topsrcdir)/xpcom/idl-parser $(XPIDL_FLAGS) $(_VPATH_SRCS) -d $(MDDEPDIR)/$(@F).pp -o $@
 
 # no need to link together if XPIDLSRCS contains only XPIDL_MODULE
 ifneq ($(XPIDL_MODULE).idl,$(strip $(XPIDLSRCS)))
 $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt: $(patsubst %.idl,$(XPIDL_GEN_DIR)/%.xpt,$(XPIDLSRCS)) $(GLOBAL_DEPS)
 	$(XPIDL_LINK) $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt $(patsubst %.idl,$(XPIDL_GEN_DIR)/%.xpt,$(XPIDLSRCS))
 endif # XPIDL_MODULE.xpt != XPIDLSRCS
 
 libs:: $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt
new file mode 100644
--- /dev/null
+++ b/xpcom/idl-parser/typelib.py
@@ -0,0 +1,317 @@
+#!/usr/bin/env python
+# typelib.py - Generate XPCOM typelib files from IDL.
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is pyxpidl.
+#
+# The Initial Developer of the Original Code is the Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Kyle Huey <khuey@kylehuey.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either of the GNU General Public License Version 2 or later (the "GPL"),
+# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""Generate an XPIDL typelib for the IDL files specified on the command line"""
+
+import xpidl, xpt
+
+# A map of xpidl.py types to xpt.py types
+TypeMap = {
+    # nsresult is not strictly an xpidl.py type, but it's useful here
+    'nsresult':           xpt.Type.Tags.uint32,
+    # builtins
+    'boolean':            xpt.Type.Tags.boolean,
+    'void':               xpt.Type.Tags.void,
+    'octet':              xpt.Type.Tags.uint8,
+    'short':              xpt.Type.Tags.int16,
+    'long':               xpt.Type.Tags.int32,
+    'long long':          xpt.Type.Tags.int64,
+    'unsigned short':     xpt.Type.Tags.uint16,
+    'unsigned long':      xpt.Type.Tags.uint32,
+    'unsigned long long': xpt.Type.Tags.uint64,
+    'float':              xpt.Type.Tags.float,
+    'double':             xpt.Type.Tags.double,
+    'char':               xpt.Type.Tags.char,
+    'string':             xpt.Type.Tags.char_ptr,
+    'wchar':              xpt.Type.Tags.wchar_t,
+    'wstring':            xpt.Type.Tags.wchar_t_ptr,
+    # special types
+    'nsid':               xpt.Type.Tags.nsIID,
+    'domstring':          xpt.Type.Tags.DOMString,
+    'astring':            xpt.Type.Tags.AString,
+    'utf8string':         xpt.Type.Tags.UTF8String,
+    'cstring':            xpt.Type.Tags.CString,
+    'jsval':              xpt.Type.Tags.jsval
+}
+
+# XXXkhuey dipper types should go away (bug 677784)
+def isDipperType(type):
+    return type == xpt.Type.Tags.DOMString or type == xpt.Type.Tags.AString or type == xpt.Type.Tags.CString or type == xpt.Type.Tags.UTF8String
+
+def build_interface(iface, ifaces):
+    def get_type(type, calltype, iid_is=None, size_is=None):
+        """ Return the appropriate xpt.Type object for this param """
+
+        if isinstance(type, xpidl.Typedef):
+            type = type.realtype
+
+        if isinstance(type, xpidl.Builtin):
+            if type.name == 'string' and size_is != None:
+                  return xpt.StringWithSizeType(size_is, size_is)
+            elif type.name == 'wstring' and size_is != None:
+                  return xpt.WideStringWithSizeType(size_is, size_is)
+            else:
+                  tag = TypeMap[type.name]
+                  isPtr = (tag == xpt.Type.Tags.char_ptr or tag == xpt.Type.Tags.wchar_t_ptr)
+                  return xpt.SimpleType(tag,
+                                        pointer=isPtr,
+                                        #XXXkhuey unique_pointer is completely unused (bug 677787.)
+                                        reference=False)
+
+        if isinstance(type, xpidl.Array):
+            return xpt.ArrayType(get_type(type.type, calltype), size_is,
+                                 #XXXkhuey length_is duplicates size_is (bug 677788),
+                                 size_is)
+
+        if isinstance(type, xpidl.Interface) or isinstance(type, xpidl.Forward):
+            xptiface = None
+            for i in ifaces:
+                if i.name == type.name:
+                    xptiface = i
+
+            if not xptiface:
+                xptiface = xpt.Interface(name=type.name)
+                ifaces.append(xptiface)
+
+            return xpt.InterfaceType(xptiface)
+
+        if isinstance(type, xpidl.Native):
+            if type.specialtype:
+                # XXXkhuey jsval is marked differently in the typelib and in the headers :-(
+                isPtr = (type.isPtr(calltype) or type.isRef(calltype)) and not type.specialtype == 'jsval'
+                isRef = type.isRef(calltype) and not type.specialtype == 'jsval'
+                return xpt.SimpleType(TypeMap[type.specialtype],
+                                      pointer=isPtr,
+                                      #XXXkhuey unique_pointer is completely unused
+                                      reference=isRef)
+            elif iid_is != None:
+                return xpt.InterfaceIsType(iid_is)
+            else:
+                # void ptr
+                return xpt.SimpleType(TypeMap['void'],
+                                      pointer=True,
+                                      #XXXkhuey unique_pointer is completely unused
+                                      reference=False)
+
+        raise Exception("Unknown type!")
+
+    def get_nsresult():
+        return xpt.SimpleType(TypeMap['nsresult'])
+
+    def build_nsresult_param():
+        return xpt.Param(get_nsresult())
+
+    def get_result_type(m):
+        if not m.notxpcom:
+            return get_nsresult()
+
+        return get_type(m.realtype, '')
+
+    def build_result_param(m):
+        return xpt.Param(get_result_type(m))
+
+    def build_retval_param(m):
+        type = get_type(m.realtype, 'out')
+        if isDipperType(type.tag):
+            # NB: The retval bit needs to be set here, contrary to what the
+            # xpt spec says.
+            return xpt.Param(type, in_=True, retval=True, dipper=True)
+        return xpt.Param(type, in_=False, out=True, retval=True)
+
+    def build_attr_param(a, getter=False, setter=False):
+        if not (getter or setter):
+            raise Exception("Attribute param must be for a getter or a setter!")
+
+        type = get_type(a.realtype, getter and 'out' or 'in')
+        if set:
+            return xpt.Param(type)
+        else:
+            if isDipperType(type.tag):
+                # NB: The retval bit needs to be set here, contrary to what the
+                # xpt spec says.
+                return xpt.Param(type, in_=True, retval=True, dipper=True)
+            return xpt.Param(type, in_=False, out=True, retval=True)
+
+    if iface.namemap is None:
+        raise Exception("Interface was not resolved.")
+
+    consts = []
+    methods = []
+
+    def build_const(c):
+        consts.append(xpt.Constant(c.name, get_type(c.basetype, ''), c.getValue()))
+
+    def build_method(m):
+        params = []
+
+        def build_param(p):
+            def findattr(p, attr):
+                if hasattr(p, attr) and getattr(p, attr):
+                    for i, param in enumerate(m.params):
+                        if param.name == getattr(p, attr):
+                            return i
+                    return None
+
+            iid_is = findattr(p, 'iid_is')
+            size_is = findattr(p, 'size_is')
+
+            in_ = p.paramtype.count("in")
+            out = p.paramtype.count("out")
+            dipper = False
+            type = get_type(p.realtype, p.paramtype, iid_is=iid_is, size_is=size_is)
+            if out and isDipperType(type.tag):
+                out = False
+                dipper = True
+
+            return xpt.Param(type, in_, out, p.retval, p.shared, dipper, p.optional)
+
+        for p in m.params:
+            params.append(build_param(p))
+
+        if not m.notxpcom and m.realtype.name != 'void':
+            params.append(build_retval_param(m))
+
+        methods.append(xpt.Method(m.name, build_result_param(m), params,
+                                  getter=False, setter=False, notxpcom=m.notxpcom,
+                                  constructor=False, hidden=m.noscript,
+                                  optargc=m.optional_argc,
+                                  implicit_jscontext=m.implicit_jscontext))
+
+    def build_attr(a):
+        # Write the getter
+        methods.append(xpt.Method(a.name, build_nsresult_param(),
+                                  [build_attr_param(a, get=True)],
+                                  getter=True, setter=False, notxpcom=a.notxpcom,
+                                  constructor=False, hidden=a.noscript,
+                                  optargc=False,
+                                  implict_jscontext=a.implicit_jscontext))
+
+        # And maybe the setter
+        if not a.readonly:
+            methods.append(xpt.Method(a.name, build_nsresult_param(),
+                                      [build_attr_param(a, set=True)],
+                                      getter=False, setter=True, notxpcom=a.notxpcom,
+                                      constructor=False, hidden=a.noscript,
+                                      optargc=False,
+                                      implicit_jscontext=a.implicit_jscontext))
+
+    for member in iface.members:
+        if isinstance(member, xpidl.ConstMember):
+            build_const(member)
+        elif isinstance(member, xpidl.Attribute):
+            build_attr(member)
+        elif isinstance(member, xpidl.Method):
+            build_method(member)
+        elif isinstance(member, xpidl.CDATA):
+            pass
+        else:
+            raise Exception("Unexpected interface member: %s" % member)
+
+    parent = None
+    if iface.base:
+        for i in ifaces:
+            if i.name == iface.base:
+                parent = i
+        if not parent:
+            parent = xpt.Interface(name=iface.base)
+            ifaces.append(parent)
+
+    return xpt.Interface(iface.name, iface.attributes.uuid, methods=methods,
+                         constants=consts, resolved=True, parent=parent,
+                         scriptable=iface.attributes.scriptable,
+                         function=iface.attributes.function,
+                         builtinclass=iface.attributes.builtinclass)
+
+def write_typelib(idl, fd, filename):
+    """ Generate the typelib. """
+
+    # We only care about interfaces
+    ifaces = []
+    for p in idl.productions:
+        if p.kind == 'interface':
+            ifaces.append(build_interface(p, ifaces))
+
+    typelib = xpt.Typelib(interfaces=ifaces)
+    typelib.writefd(fd)
+
+def main(*args):
+    from optparse import OptionParser
+    o = OptionParser()
+    o.add_option('-I', action='append', dest='incdirs', default=['.'],
+                 help="Directory to search for imported files")
+    o.add_option('--cachedir', dest='cachedir', default=None,
+                 help="Directory in which to cache lex/parse tables.")
+    o.add_option('-o', dest='outfile', default=None,
+                 help="Output file")
+    o.add_option('-d', dest='depfile', default=None,
+                 help="Generate a make dependency file")
+    options, args = o.parse_args()
+    file, = args
+
+    if options.cachedir is not None:
+        if not os.path.isdir(options.cachedir):
+            os.mkdir(options.cachedir)
+        sys.path.append(options.cachedir)
+
+    if options.depfile is not None and options.outfile is None:
+        print >>sys.stderr, "-d requires -o"
+        sys.exit(1)
+
+    if options.outfile is not None:
+        outfd = open(options.outfile, 'wb')
+        closeoutfd = True
+    else:
+        raise "typelib generation requires an output file"
+
+    p = xpidl.IDLParser(outputdir=options.cachedir)
+    idl = p.parse(open(file).read(), filename=file)
+    idl.resolve(options.incdirs, p)
+    write_typelib(idl, outfd, file)
+
+    if closeoutfd:
+        outfd.close()
+
+    if options.depfile is not None:
+        depfd = open(options.depfile, 'w')
+        deps = [dep.replace('\\', '/') for dep in idl.deps]
+
+        print >>depfd, "%s: %s" % (options.outfile, " ".join(deps))
+
+if __name__ == '__main__':
+    main()
--- a/xpcom/idl-parser/xpidl.py
+++ b/xpcom/idl-parser/xpidl.py
@@ -445,33 +445,35 @@ class Native(object):
         if self.specialtype is None:
             return False
 
         if self.specialtype == 'nsid':
             return self.modifier is not None
 
         return self.modifier == 'ref'
 
+    def isPtr(self, calltype):
+        return self.modifier == 'ptr' or (self.modifier == 'ref' and self.specialtype == 'jsval' and calltype == 'out')
+
+    def isRef(self, calltype):
+        return self.modifier == 'ref' and not (self.specialtype == 'jsval' and calltype == 'out')
+
     def nativeType(self, calltype, const=False, shared=False):
         if shared:
             if calltype != 'out':
                 raise IDLError("[shared] only applies to out parameters.")
             const = True
 
         if self.specialtype is not None and calltype == 'in':
             const = True
 
-        if self.modifier == 'ptr':
-            m = '*' + (calltype != 'in' and '*' or '')
-        elif self.modifier == 'ref':
-            # jsval outparams are odd, for compatibility with existing code
-            if self.specialtype == 'jsval' and calltype == 'out':
-                m = '*'
-            else:
-                m = '& '
+        if self.isRef(calltype):
+            m = '& '
+        elif self.isPtr(calltype):
+            m = '*' + ((self.modifier == 'ptr' and calltype != 'in') and '*' or '')
         else:
             m = calltype != 'in' and '*' or ''
         return "%s%s %s" % (const and 'const ' or '', self.nativename, m)
 
     def __str__(self):
         return "native %s(%s)\n" % (self.name, self.nativename)
 
 class Interface(object):
--- a/xpcom/typelib/xpt/tools/xpt.py
+++ b/xpcom/typelib/xpt/tools/xpt.py
@@ -140,16 +140,18 @@ class Type(object):
         'AString',
         'jsval',
         )
 
     def __init__(self, pointer=False, unique_pointer=False, reference=False):
         self.pointer = pointer
         self.unique_pointer = unique_pointer
         self.reference = reference
+        if reference and not pointer:
+            raise Exception("If reference is True pointer must be True too")
 
     @staticmethod
     def decodeflags(byte):
         """
         Given |byte|, an unsigned uint8 containing flag bits,
         decode the flag bits as described in
         http://www.mozilla.org/scriptable/typelib_file.html#TypeDescriptor
         and return a dict of flagname: (True|False) suitable
@@ -512,16 +514,17 @@ class Param(object):
 
     def __init__(self, type, in_=True, out=False, retval=False,
                  shared=False, dipper=False, optional=False):
         """
         Construct a Param object with the specified |type| and
         flags. Params default to "in".
 
         """
+
         self.type = type
         self.in_ = in_
         self.out = out
         self.retval = retval
         self.shared = shared
         self.dipper = dipper
         self.optional = optional
 
@@ -637,16 +640,18 @@ class Method(object):
         self.getter = getter
         self.setter = setter
         self.notxpcom = notxpcom
         self.constructor = constructor
         self.hidden = hidden
         self.optargc = optargc
         self.implicit_jscontext = implicit_jscontext
         self.params = list(params)
+        if result and not isinstance(result, Param):
+            raise Exception("result must be a Param!")
         self.result = result
 
     def read_params(self, typelib, map, data_pool, offset, num_args):
         """
         Read |num_args| ParamDescriptors representing this Method's arguments
         from the mmaped file |map| with data pool at the offset |data_pool|,
         starting at |offset| into self.params. Returns the offset
         suitable for reading the data following the ParamDescriptor array.