Bug 458936: Replace xpidlc's typelib generation with a python equivalent. r=ted
authorKyle Huey <khuey@kylehuey.com>
Tue, 09 Aug 2011 20:48:41 -0400
changeset 75198 1a9b3ace8c68e0c53589900f34e0a60cb178d564
parent 75197 4aa92c4a5f135cf1ebbc5f327d0a3e884482f896
child 75199 b8ed3bb885a7b1a807550c7c4cd949bb97b65ca9
push id67
push userclegnitto@mozilla.com
push dateFri, 04 Nov 2011 22:39:41 +0000
treeherdermozilla-release@04778346a3b0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs458936
milestone8.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 458936: Replace xpidlc's typelib generation with a python equivalent. r=ted
config/rules.mk
js/src/config/rules.mk
xpcom/idl-parser/typelib.py
xpcom/idl-parser/xpidl.py
xpcom/typelib/xpt/tools/xpt.py
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -1527,34 +1527,39 @@ endif
 	$(MKDIR) -p $(XPIDL_GEN_DIR)
 	@$(TOUCH) $@
 
 # don't depend on $(XPIDL_GEN_DIR), because the modification date changes
 # with any addition to the directory, regenerating all .h files -> everything.
 
 XPIDL_DEPS = \
   $(topsrcdir)/xpcom/idl-parser/header.py \
+  $(topsrcdir)/xpcom/idl-parser/typelib.py \
   $(topsrcdir)/xpcom/idl-parser/xpidl.py \
   $(NULL)
 
 $(XPIDL_GEN_DIR)/%.h: %.idl $(XPIDL_DEPS) $(XPIDL_GEN_DIR)/.done
 	$(REPORT_BUILD)
 	$(PYTHON_PATH) \
 	  -I$(topsrcdir)/other-licenses/ply \
 	  -I$(topsrcdir)/xpcom/idl-parser \
 	  $(topsrcdir)/xpcom/idl-parser/header.py --cachedir=$(topsrcdir)/xpcom/idl-parser $(XPIDL_FLAGS) $(_VPATH_SRCS) -d $(MDDEPDIR)/$(@F).pp -o $@
 	@if test -n "$(findstring $*.h, $(EXPORTS))"; \
 	  then echo "*** WARNING: file $*.h generated from $*.idl overrides $(srcdir)/$*.h"; else true; fi
 
 ifndef NO_GEN_XPT
 # generate intermediate .xpt files into $(XPIDL_GEN_DIR), then link
 # into $(XPIDL_MODULE).xpt and export it to $(FINAL_TARGET)/components.
-$(XPIDL_GEN_DIR)/%.xpt: %.idl $(XPIDL_COMPILE) $(XPIDL_GEN_DIR)/.done
+$(XPIDL_GEN_DIR)/%.xpt: %.idl $(XPIDL_DEPS) $(XPIDL_GEN_DIR)/.done
 	$(REPORT_BUILD)
-	$(ELOG) $(XPIDL_COMPILE) -m typelib -w $(XPIDL_FLAGS) -e $@ -d $(MDDEPDIR)/$(@F).pp $(_VPATH_SRCS)
+	$(PYTHON_PATH) \
+	  -I$(topsrcdir)/other-licenses/ply \
+	  -I$(topsrcdir)/xpcom/idl-parser \
+	  -I$(topsrcdir)/xpcom/typelib/xpt/tools \
+	  $(topsrcdir)/xpcom/idl-parser/typelib.py --cachedir=$(topsrcdir)/xpcom/idl-parser $(XPIDL_FLAGS) $(_VPATH_SRCS) -d $(MDDEPDIR)/$(@F).pp -o $@
 
 # no need to link together if XPIDLSRCS contains only XPIDL_MODULE
 ifneq ($(XPIDL_MODULE).idl,$(strip $(XPIDLSRCS)))
 $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt: $(patsubst %.idl,$(XPIDL_GEN_DIR)/%.xpt,$(XPIDLSRCS)) $(GLOBAL_DEPS)
 	$(XPIDL_LINK) $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt $(patsubst %.idl,$(XPIDL_GEN_DIR)/%.xpt,$(XPIDLSRCS))
 endif # XPIDL_MODULE.xpt != XPIDLSRCS
 
 libs:: $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt
--- a/js/src/config/rules.mk
+++ b/js/src/config/rules.mk
@@ -1527,34 +1527,39 @@ endif
 	$(MKDIR) -p $(XPIDL_GEN_DIR)
 	@$(TOUCH) $@
 
 # don't depend on $(XPIDL_GEN_DIR), because the modification date changes
 # with any addition to the directory, regenerating all .h files -> everything.
 
 XPIDL_DEPS = \
   $(topsrcdir)/xpcom/idl-parser/header.py \
+  $(topsrcdir)/xpcom/idl-parser/typelib.py \
   $(topsrcdir)/xpcom/idl-parser/xpidl.py \
   $(NULL)
 
 $(XPIDL_GEN_DIR)/%.h: %.idl $(XPIDL_DEPS) $(XPIDL_GEN_DIR)/.done
 	$(REPORT_BUILD)
 	$(PYTHON_PATH) \
 	  -I$(topsrcdir)/other-licenses/ply \
 	  -I$(topsrcdir)/xpcom/idl-parser \
 	  $(topsrcdir)/xpcom/idl-parser/header.py --cachedir=$(topsrcdir)/xpcom/idl-parser $(XPIDL_FLAGS) $(_VPATH_SRCS) -d $(MDDEPDIR)/$(@F).pp -o $@
 	@if test -n "$(findstring $*.h, $(EXPORTS))"; \
 	  then echo "*** WARNING: file $*.h generated from $*.idl overrides $(srcdir)/$*.h"; else true; fi
 
 ifndef NO_GEN_XPT
 # generate intermediate .xpt files into $(XPIDL_GEN_DIR), then link
 # into $(XPIDL_MODULE).xpt and export it to $(FINAL_TARGET)/components.
-$(XPIDL_GEN_DIR)/%.xpt: %.idl $(XPIDL_COMPILE) $(XPIDL_GEN_DIR)/.done
+$(XPIDL_GEN_DIR)/%.xpt: %.idl $(XPIDL_DEPS) $(XPIDL_GEN_DIR)/.done
 	$(REPORT_BUILD)
-	$(ELOG) $(XPIDL_COMPILE) -m typelib -w $(XPIDL_FLAGS) -e $@ -d $(MDDEPDIR)/$(@F).pp $(_VPATH_SRCS)
+	$(PYTHON_PATH) \
+	  -I$(topsrcdir)/other-licenses/ply \
+	  -I$(topsrcdir)/xpcom/idl-parser \
+	  -I$(topsrcdir)/xpcom/typelib/xpt/tools \
+	  $(topsrcdir)/xpcom/idl-parser/typelib.py --cachedir=$(topsrcdir)/xpcom/idl-parser $(XPIDL_FLAGS) $(_VPATH_SRCS) -d $(MDDEPDIR)/$(@F).pp -o $@
 
 # no need to link together if XPIDLSRCS contains only XPIDL_MODULE
 ifneq ($(XPIDL_MODULE).idl,$(strip $(XPIDLSRCS)))
 $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt: $(patsubst %.idl,$(XPIDL_GEN_DIR)/%.xpt,$(XPIDLSRCS)) $(GLOBAL_DEPS)
 	$(XPIDL_LINK) $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt $(patsubst %.idl,$(XPIDL_GEN_DIR)/%.xpt,$(XPIDLSRCS))
 endif # XPIDL_MODULE.xpt != XPIDLSRCS
 
 libs:: $(XPIDL_GEN_DIR)/$(XPIDL_MODULE).xpt
new file mode 100644
--- /dev/null
+++ b/xpcom/idl-parser/typelib.py
@@ -0,0 +1,317 @@
+#!/usr/bin/env python
+# typelib.py - Generate XPCOM typelib files from IDL.
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is pyxpidl.
+#
+# The Initial Developer of the Original Code is the Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2011
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Kyle Huey <khuey@kylehuey.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either of the GNU General Public License Version 2 or later (the "GPL"),
+# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""Generate an XPIDL typelib for the IDL files specified on the command line"""
+
+import xpidl, xpt
+
+# A map of xpidl.py types to xpt.py types
+TypeMap = {
+    # nsresult is not strictly an xpidl.py type, but it's useful here
+    'nsresult':           xpt.Type.Tags.uint32,
+    # builtins
+    'boolean':            xpt.Type.Tags.boolean,
+    'void':               xpt.Type.Tags.void,
+    'octet':              xpt.Type.Tags.uint8,
+    'short':              xpt.Type.Tags.int16,
+    'long':               xpt.Type.Tags.int32,
+    'long long':          xpt.Type.Tags.int64,
+    'unsigned short':     xpt.Type.Tags.uint16,
+    'unsigned long':      xpt.Type.Tags.uint32,
+    'unsigned long long': xpt.Type.Tags.uint64,
+    'float':              xpt.Type.Tags.float,
+    'double':             xpt.Type.Tags.double,
+    'char':               xpt.Type.Tags.char,
+    'string':             xpt.Type.Tags.char_ptr,
+    'wchar':              xpt.Type.Tags.wchar_t,
+    'wstring':            xpt.Type.Tags.wchar_t_ptr,
+    # special types
+    'nsid':               xpt.Type.Tags.nsIID,
+    'domstring':          xpt.Type.Tags.DOMString,
+    'astring':            xpt.Type.Tags.AString,
+    'utf8string':         xpt.Type.Tags.UTF8String,
+    'cstring':            xpt.Type.Tags.CString,
+    'jsval':              xpt.Type.Tags.jsval
+}
+
+# XXXkhuey dipper types should go away (bug 677784)
+def isDipperType(type):
+    return type == xpt.Type.Tags.DOMString or type == xpt.Type.Tags.AString or type == xpt.Type.Tags.CString or type == xpt.Type.Tags.UTF8String
+
+def build_interface(iface, ifaces):
+    def get_type(type, calltype, iid_is=None, size_is=None):
+        """ Return the appropriate xpt.Type object for this param """
+
+        if isinstance(type, xpidl.Typedef):
+            type = type.realtype
+
+        if isinstance(type, xpidl.Builtin):
+            if type.name == 'string' and size_is != None:
+                  return xpt.StringWithSizeType(size_is, size_is)
+            elif type.name == 'wstring' and size_is != None:
+                  return xpt.WideStringWithSizeType(size_is, size_is)
+            else:
+                  tag = TypeMap[type.name]
+                  isPtr = (tag == xpt.Type.Tags.char_ptr or tag == xpt.Type.Tags.wchar_t_ptr)
+                  return xpt.SimpleType(tag,
+                                        pointer=isPtr,
+                                        #XXXkhuey unique_pointer is completely unused (bug 677787.)
+                                        reference=False)
+
+        if isinstance(type, xpidl.Array):
+            return xpt.ArrayType(get_type(type.type, calltype), size_is,
+                                 #XXXkhuey length_is duplicates size_is (bug 677788),
+                                 size_is)
+
+        if isinstance(type, xpidl.Interface) or isinstance(type, xpidl.Forward):
+            xptiface = None
+            for i in ifaces:
+                if i.name == type.name:
+                    xptiface = i
+
+            if not xptiface:
+                xptiface = xpt.Interface(name=type.name)
+                ifaces.append(xptiface)
+
+            return xpt.InterfaceType(xptiface)
+
+        if isinstance(type, xpidl.Native):
+            if type.specialtype:
+                # XXXkhuey jsval is marked differently in the typelib and in the headers :-(
+                isPtr = (type.isPtr(calltype) or type.isRef(calltype)) and not type.specialtype == 'jsval'
+                isRef = type.isRef(calltype) and not type.specialtype == 'jsval'
+                return xpt.SimpleType(TypeMap[type.specialtype],
+                                      pointer=isPtr,
+                                      #XXXkhuey unique_pointer is completely unused
+                                      reference=isRef)
+            elif iid_is != None:
+                return xpt.InterfaceIsType(iid_is)
+            else:
+                # void ptr
+                return xpt.SimpleType(TypeMap['void'],
+                                      pointer=True,
+                                      #XXXkhuey unique_pointer is completely unused
+                                      reference=False)
+
+        raise Exception("Unknown type!")
+
+    def get_nsresult():
+        return xpt.SimpleType(TypeMap['nsresult'])
+
+    def build_nsresult_param():
+        return xpt.Param(get_nsresult())
+
+    def get_result_type(m):
+        if not m.notxpcom:
+            return get_nsresult()
+
+        return get_type(m.realtype, '')
+
+    def build_result_param(m):
+        return xpt.Param(get_result_type(m))
+
+    def build_retval_param(m):
+        type = get_type(m.realtype, 'out')
+        if isDipperType(type.tag):
+            # NB: The retval bit needs to be set here, contrary to what the
+            # xpt spec says.
+            return xpt.Param(type, in_=True, retval=True, dipper=True)
+        return xpt.Param(type, in_=False, out=True, retval=True)
+
+    def build_attr_param(a, getter=False, setter=False):
+        if not (getter or setter):
+            raise Exception("Attribute param must be for a getter or a setter!")
+
+        type = get_type(a.realtype, getter and 'out' or 'in')
+        if set:
+            return xpt.Param(type)
+        else:
+            if isDipperType(type.tag):
+                # NB: The retval bit needs to be set here, contrary to what the
+                # xpt spec says.
+                return xpt.Param(type, in_=True, retval=True, dipper=True)
+            return xpt.Param(type, in_=False, out=True, retval=True)
+
+    if iface.namemap is None:
+        raise Exception("Interface was not resolved.")
+
+    consts = []
+    methods = []
+
+    def build_const(c):
+        consts.append(xpt.Constant(c.name, get_type(c.basetype, ''), c.getValue()))
+
+    def build_method(m):
+        params = []
+
+        def build_param(p):
+            def findattr(p, attr):
+                if hasattr(p, attr) and getattr(p, attr):
+                    for i, param in enumerate(m.params):
+                        if param.name == getattr(p, attr):
+                            return i
+                    return None
+
+            iid_is = findattr(p, 'iid_is')
+            size_is = findattr(p, 'size_is')
+
+            in_ = p.paramtype.count("in")
+            out = p.paramtype.count("out")
+            dipper = False
+            type = get_type(p.realtype, p.paramtype, iid_is=iid_is, size_is=size_is)
+            if out and isDipperType(type.tag):
+                out = False
+                dipper = True
+
+            return xpt.Param(type, in_, out, p.retval, p.shared, dipper, p.optional)
+
+        for p in m.params:
+            params.append(build_param(p))
+
+        if not m.notxpcom and m.realtype.name != 'void':
+            params.append(build_retval_param(m))
+
+        methods.append(xpt.Method(m.name, build_result_param(m), params,
+                                  getter=False, setter=False, notxpcom=m.notxpcom,
+                                  constructor=False, hidden=m.noscript,
+                                  optargc=m.optional_argc,
+                                  implicit_jscontext=m.implicit_jscontext))
+
+    def build_attr(a):
+        # Write the getter
+        methods.append(xpt.Method(a.name, build_nsresult_param(),
+                                  [build_attr_param(a, get=True)],
+                                  getter=True, setter=False, notxpcom=a.notxpcom,
+                                  constructor=False, hidden=a.noscript,
+                                  optargc=False,
+                                  implict_jscontext=a.implicit_jscontext))
+
+        # And maybe the setter
+        if not a.readonly:
+            methods.append(xpt.Method(a.name, build_nsresult_param(),
+                                      [build_attr_param(a, set=True)],
+                                      getter=False, setter=True, notxpcom=a.notxpcom,
+                                      constructor=False, hidden=a.noscript,
+                                      optargc=False,
+                                      implicit_jscontext=a.implicit_jscontext))
+
+    for member in iface.members:
+        if isinstance(member, xpidl.ConstMember):
+            build_const(member)
+        elif isinstance(member, xpidl.Attribute):
+            build_attr(member)
+        elif isinstance(member, xpidl.Method):
+            build_method(member)
+        elif isinstance(member, xpidl.CDATA):
+            pass
+        else:
+            raise Exception("Unexpected interface member: %s" % member)
+
+    parent = None
+    if iface.base:
+        for i in ifaces:
+            if i.name == iface.base:
+                parent = i
+        if not parent:
+            parent = xpt.Interface(name=iface.base)
+            ifaces.append(parent)
+
+    return xpt.Interface(iface.name, iface.attributes.uuid, methods=methods,
+                         constants=consts, resolved=True, parent=parent,
+                         scriptable=iface.attributes.scriptable,
+                         function=iface.attributes.function,
+                         builtinclass=iface.attributes.builtinclass)
+
+def write_typelib(idl, fd, filename):
+    """ Generate the typelib. """
+
+    # We only care about interfaces
+    ifaces = []
+    for p in idl.productions:
+        if p.kind == 'interface':
+            ifaces.append(build_interface(p, ifaces))
+
+    typelib = xpt.Typelib(interfaces=ifaces)
+    typelib.writefd(fd)
+
+def main(*args):
+    from optparse import OptionParser
+    o = OptionParser()
+    o.add_option('-I', action='append', dest='incdirs', default=['.'],
+                 help="Directory to search for imported files")
+    o.add_option('--cachedir', dest='cachedir', default=None,
+                 help="Directory in which to cache lex/parse tables.")
+    o.add_option('-o', dest='outfile', default=None,
+                 help="Output file")
+    o.add_option('-d', dest='depfile', default=None,
+                 help="Generate a make dependency file")
+    options, args = o.parse_args()
+    file, = args
+
+    if options.cachedir is not None:
+        if not os.path.isdir(options.cachedir):
+            os.mkdir(options.cachedir)
+        sys.path.append(options.cachedir)
+
+    if options.depfile is not None and options.outfile is None:
+        print >>sys.stderr, "-d requires -o"
+        sys.exit(1)
+
+    if options.outfile is not None:
+        outfd = open(options.outfile, 'wb')
+        closeoutfd = True
+    else:
+        raise "typelib generation requires an output file"
+
+    p = xpidl.IDLParser(outputdir=options.cachedir)
+    idl = p.parse(open(file).read(), filename=file)
+    idl.resolve(options.incdirs, p)
+    write_typelib(idl, outfd, file)
+
+    if closeoutfd:
+        outfd.close()
+
+    if options.depfile is not None:
+        depfd = open(options.depfile, 'w')
+        deps = [dep.replace('\\', '/') for dep in idl.deps]
+
+        print >>depfd, "%s: %s" % (options.outfile, " ".join(deps))
+
+if __name__ == '__main__':
+    main()
--- a/xpcom/idl-parser/xpidl.py
+++ b/xpcom/idl-parser/xpidl.py
@@ -445,33 +445,35 @@ class Native(object):
         if self.specialtype is None:
             return False
 
         if self.specialtype == 'nsid':
             return self.modifier is not None
 
         return self.modifier == 'ref'
 
+    def isPtr(self, calltype):
+        return self.modifier == 'ptr' or (self.modifier == 'ref' and self.specialtype == 'jsval' and calltype == 'out')
+
+    def isRef(self, calltype):
+        return self.modifier == 'ref' and not (self.specialtype == 'jsval' and calltype == 'out')
+
     def nativeType(self, calltype, const=False, shared=False):
         if shared:
             if calltype != 'out':
                 raise IDLError("[shared] only applies to out parameters.")
             const = True
 
         if self.specialtype is not None and calltype == 'in':
             const = True
 
-        if self.modifier == 'ptr':
-            m = '*' + (calltype != 'in' and '*' or '')
-        elif self.modifier == 'ref':
-            # jsval outparams are odd, for compatibility with existing code
-            if self.specialtype == 'jsval' and calltype == 'out':
-                m = '*'
-            else:
-                m = '& '
+        if self.isRef(calltype):
+            m = '& '
+        elif self.isPtr(calltype):
+            m = '*' + ((self.modifier == 'ptr' and calltype != 'in') and '*' or '')
         else:
             m = calltype != 'in' and '*' or ''
         return "%s%s %s" % (const and 'const ' or '', self.nativename, m)
 
     def __str__(self):
         return "native %s(%s)\n" % (self.name, self.nativename)
 
 class Interface(object):
--- a/xpcom/typelib/xpt/tools/xpt.py
+++ b/xpcom/typelib/xpt/tools/xpt.py
@@ -140,16 +140,18 @@ class Type(object):
         'AString',
         'jsval',
         )
 
     def __init__(self, pointer=False, unique_pointer=False, reference=False):
         self.pointer = pointer
         self.unique_pointer = unique_pointer
         self.reference = reference
+        if reference and not pointer:
+            raise Exception("If reference is True pointer must be True too")
 
     @staticmethod
     def decodeflags(byte):
         """
         Given |byte|, an unsigned uint8 containing flag bits,
         decode the flag bits as described in
         http://www.mozilla.org/scriptable/typelib_file.html#TypeDescriptor
         and return a dict of flagname: (True|False) suitable
@@ -512,16 +514,17 @@ class Param(object):
 
     def __init__(self, type, in_=True, out=False, retval=False,
                  shared=False, dipper=False, optional=False):
         """
         Construct a Param object with the specified |type| and
         flags. Params default to "in".
 
         """
+
         self.type = type
         self.in_ = in_
         self.out = out
         self.retval = retval
         self.shared = shared
         self.dipper = dipper
         self.optional = optional
 
@@ -637,16 +640,18 @@ class Method(object):
         self.getter = getter
         self.setter = setter
         self.notxpcom = notxpcom
         self.constructor = constructor
         self.hidden = hidden
         self.optargc = optargc
         self.implicit_jscontext = implicit_jscontext
         self.params = list(params)
+        if result and not isinstance(result, Param):
+            raise Exception("result must be a Param!")
         self.result = result
 
     def read_params(self, typelib, map, data_pool, offset, num_args):
         """
         Read |num_args| ParamDescriptors representing this Method's arguments
         from the mmaped file |map| with data pool at the offset |data_pool|,
         starting at |offset| into self.params. Returns the offset
         suitable for reading the data following the ParamDescriptor array.