bug 516798: fix busted actor-as-return-value handling in IPDL
authorChris Jones <jones.chris.g@gmail.com>
Wed, 16 Sep 2009 21:49:41 -0500
changeset 35922 cb6ed6132f0e95bdd92dfa4517d5a604f61e2ab5
parent 35921 9bccfe04db0b6b5e71d58b99532fe3e066746216
child 35923 975893837c9bab44054a23ca116738370ef76656
push idunknown
push userunknown
push dateunknown
bugs516798
milestone1.9.3a1pre
bug 516798: fix busted actor-as-return-value handling in IPDL
ipc/glue/SyncChannel.cpp
ipc/ipdl/ipdl/lower.py
--- a/ipc/glue/SyncChannel.cpp
+++ b/ipc/glue/SyncChannel.cpp
@@ -89,17 +89,17 @@ SyncChannel::Send(Message* msg, Message*
 }
 
 void
 SyncChannel::OnDispatchMessage(const Message& msg)
 {
     NS_ABORT_IF_FALSE(msg.is_sync(), "only sync messages here");
     NS_ABORT_IF_FALSE(!msg.is_reply(), "wasn't awaiting reply");
 
-    Message* reply;
+    Message* reply = 0;
 
     mProcessingSyncMessage = true;
     Result rv =
         static_cast<SyncListener*>(mListener)->OnMessageReceived(msg, reply);
     mProcessingSyncMessage = false;
 
     switch (rv) {
     case MsgProcessed:
--- a/ipc/ipdl/ipdl/lower.py
+++ b/ipc/ipdl/ipdl/lower.py
@@ -804,29 +804,30 @@ class GenerateProtocolHeader(Visitor):
         md._cxx.params = [ makeCxxDecl(d) for d in md.inParams ]
         md._cxx.returns = [ makeCxxDecl(d) for d in md.outParams ]
 
         # generate C++ interface to message sending/handling
         method = cxx.MethodDecl(
             name=md.decl.progname,
             params=[ ],
             ret=cxx.Type('nsresult'),
-            virtual=True)
+            virtual=1)
         for param in md._cxx.params:
             pcopy = deepcopy(param)
-            pcopy.type.const = True
+            pcopy.type.const = 1
             if not param.type.actor:
-                pcopy.type.ref = True
+                pcopy.type.ref = 1
             method.params.append(pcopy)
         for ret in md._cxx.returns:
             rcopy = deepcopy(ret)
             if rcopy.type.actor:
-                rcopy.type.ptrptr = True
+                rcopy.type.ptr = 0
+                rcopy.type.ptrptr = 1
             else:
-                rcopy.type.ptr = True
+                rcopy.type.ptr = 1
             method.params.append(rcopy)
         md._cxx.method = method
 
         # the ID is used by the IPC layer only
         md._cxx.id = 'Msg_%s'% (md.decl.progname)
         md._cxx.nsid = '%s::%s'% (self.pname, md._cxx.id)
         if md.decl.type.hasReply():
             md._cxx.replyid = 'Reply_%s'% (md.decl.progname)
@@ -1669,21 +1670,32 @@ class GenerateProtocolActorHeader(Visito
                 impl.addstmt(cxx.StmtDecl(cxx.Decl(cxx.Type('Message'),
                                                    '__reply')))
                 replyvar = cxx.ExprVar('__reply')
 
             impl.addstmt(cxx.StmtDecl(cxx.Decl(cxx.Type(md._cxx.nsid, ptr=1),
                                                '__msg')))
             msgvar = cxx.ExprVar('__msg')
 
+            impl.addstmt(cxx.Whitespace.NL)
+
             # special case for explicit actor params: they need to be
             # converted to Handle's before sending
             for param in mdecl.params:
                 if not param.type.actor:
                     continue
+
+                # an actor outparam.  check that it's non-null
+                if param.type.ptrptr:
+                    checknull = cxx.StmtIf(
+                        cxx.ExprPrefixUnop(cxx.ExprVar(param.name), '!'))
+                    checknull.addifstmt(cxx.StmtReturn(valueerrcode))
+                    impl.addstmt(checknull)
+                    continue
+
                 pavar = cxx.ExprVar(param.name)
                 pahvar = cxx.ExprVar(param.name +'__ah')
                 impl.addstmt(cxx.StmtDecl(cxx.Decl(
                     cxx.Type('mozilla::ipc::ActorHandle'), pahvar.name)))
                 impl.addstmts(_actorToActorHandle(actor=pavar,
                                                   handle=pahvar,
                                                   failcode=valueerrcode))
             impl.addstmt(cxx.Whitespace.NL)
@@ -1850,32 +1862,26 @@ class GenerateProtocolActorHeader(Visito
             if hasreply:
                 replyerror = cxx.ExprCall(
                     cxx.ExprSelect(replyvar, '.', 'is_reply_error'))
                 failif2 = cxx.StmtIf(replyerror)
                 failif2.ifb.addstmt(cxx.StmtReturn(valueerrcode))
                 impl.addstmt(failif2)
 
                 # if this message has explicit actor returns, we need
-                # to convert them from Handle before returning to C++
+                # to convert them from Handle before returning to C++.
+                # first declare the ActorHandle variables in which
+                # we store the deserialized, intermediate representation
                 for ret in md._cxx.returns:
                     if not ret.type.actor: continue
-                    
-                    ravar = cxx.ExprVar(ret.name)
-                    rahvar = cxx.ExprVar(ret.name +'__ah')
-                    actortype = deepcopy(ret.type)
-                    actortype.name = _actorName(actortype.name, self.myside)
-                    actortype.ptrptr = 0; actortype.ptr = 1
-                    
+
                     impl.addstmt(cxx.StmtDecl(cxx.Decl(
-                        cxx.Type('mozilla::ipc::ActorHandle'), rahvar.name)))
-                    impl.addstmts(_actorHandleToActor(handle=rahvar,
-                                                      actor=ravar,
-                                                      actortype=actortype,
-                                                      failcode=valueerrcode))
+                        cxx.Type('mozilla::ipc::ActorHandle'),
+                        ret.name +'__ah')))
+
                 impl.addstmt(cxx.Whitespace.NL)
 
                 unpackargs = [ cxx.ExprAddrOf(replyvar) ]
                 for ret in md._cxx.returns:
                     rname = ret.name
                     if ret.type.actor:
                         unpackargs.append(cxx.ExprAddrOf(cxx.ExprVar(rname +'__ah')))
                     else:
@@ -1884,16 +1890,34 @@ class GenerateProtocolActorHeader(Visito
                     unpackargs.append(cxx.ExprAddrOf(ahvar))
 
                 unpack = cxx.ExprCall(cxx.ExprVar(md._cxx.nsreplyid +'::Read'),
                                       unpackargs)
                 errhandle = cxx.StmtIf(cxx.ExprPrefixUnop(unpack, '!'))
                 errhandle.ifb.addstmt(cxx.StmtReturn(valueerrcode))
                 impl.addstmt(errhandle)
 
+                impl.addstmt(cxx.Whitespace.NL)
+
+                # now look up the actor's ID
+                for ret in md._cxx.returns:
+                    if not ret.type.actor: continue
+
+                    ravar = cxx.ExprVar(ret.name)
+                    rahvar = cxx.ExprVar(ret.name +'__ah')
+                    actortype = deepcopy(ret.type)
+                    actortype.name = _actorName(actortype.name, self.myside)
+                    actortype.ptrptr = 0; actortype.ptr = 1
+
+                    impl.addstmts(_actorHandleToActor(
+                        handle=rahvar,
+                        actor=cxx.ExprDeref(ravar),
+                        actortype=actortype,
+                        failcode=valueerrcode))
+
                 # see if we need to extract an actor out of a union
                 # and re-convert it into an actor pointer
                 for r in md._cxx.returns:
                     if (r.type._union is None
                         or not r.type._union.hasActor):
                         continue
                     u = r.type._union
                     for t, n in u.components:
@@ -2182,16 +2206,30 @@ class GenerateProtocolActorHeader(Visito
                         cxx.ExprAssn(routevar, cxx.ExprLiteral.ZERO)))
 
             if md.decl.type.hasReply():
                 if not md.decl.type.hasReply():
                     block.addstmt(cxx.StmtDecl(
                             cxx.Decl(cxx.Type('Message', ptr=1), 'reply')))
                 replyvar = cxx.ExprVar('reply')
 
+                # pack actor returns into Handles to be shipped across
+                # the wire
+                for ret in md._cxx.returns:
+                    if not ret.type.actor:
+                        continue
+
+                    retavar = cxx.ExprVar(ret.name)
+                    retahvar = cxx.ExprVar(ret.name +'__ah')
+                    block.addstmts(_actorToActorHandle(
+                        actor=retavar, handle=retahvar,
+                        failcode=cxx.ExprVar('MsgValueError')))
+
+                block.addstmt(cxx.Whitespace.NL)
+
                 # FIXME/cjones: yet more copy-n-paste
                 #
                 # special case for IPDL union types: if this union
                 # contains an actor type, we secretly convert
                 # it into a serializable representation
                 ruid = 0
                 switches = [ ]
                 for r in md._cxx.returns:
@@ -2244,16 +2282,18 @@ class GenerateProtocolActorHeader(Visito
                     switches.append(switch)
 
                 for switch in switches:
                     block.addstmt(switch)
 
                 ctorparams = [ ]
                 for i, r in enumerate(md._cxx.returns):
                     rvar = cxx.ExprVar(r.name)
+                    if r.type.actor:
+                        rvar.name += '__ah'
                     ctorparams.append(rvar)
 
                 replymsgctor = cxx.ExprNew(cxx.Type(md._cxx.nsreplyid),
                                            args=ctorparams)
                 if md.decl.type.hasImplicitActorParam():
                     replymsgctor.args.append(ahvar)
                 block.addstmt(cxx.StmtExpr(cxx.ExprAssn(replyvar,
                                                         replymsgctor)))