Bug 1348591 - Support custom default segment buffer list size. r=billm
authorHenry Chang <hchang@mozilla.com>
Wed, 21 Jun 2017 17:55:13 +0800
changeset 417150 e3a6a933eee5107d2605e9f9f0b1695fa924f58a
parent 417149 a6625c09d7772f1b047c8401f92e113e069aea6b
child 417151 aaacb772bada5bda82f692f805a6cbd3daff0ae7
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbillm
bugs1348591
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1348591 - Support custom default segment buffer list size. r=billm MozReview-Commit-ID: 2Nkj6RPx62f
ipc/chromium/src/base/pickle.cc
ipc/chromium/src/base/pickle.h
ipc/chromium/src/chrome/common/ipc_message.cc
ipc/chromium/src/chrome/common/ipc_message.h
ipc/glue/ProtocolUtils.cpp
ipc/glue/Shmem.cpp
ipc/ipdl/Makefile.in
ipc/ipdl/ipdl.py
ipc/ipdl/ipdl/__init__.py
ipc/ipdl/ipdl/lower.py
ipc/ipdl/message-metadata.ini
--- a/ipc/chromium/src/base/pickle.cc
+++ b/ipc/chromium/src/base/pickle.cc
@@ -121,18 +121,20 @@ void Pickle::UpdateIter(PickleIterator* 
   // Make sure we don't get into trouble where AlignInt(bytes) == 0.
   MOZ_RELEASE_ASSERT(bytes < 64);
 
   iter->iter_.Advance(buffers_, AlignInt(bytes));
 }
 
 // Payload is sizeof(Pickle::memberAlignmentType) aligned.
 
-Pickle::Pickle(uint32_t header_size)
-    : buffers_(AlignInt(header_size), kHeaderSegmentCapacity, kDefaultSegmentCapacity),
+Pickle::Pickle(uint32_t header_size, size_t segment_capacity)
+    : buffers_(AlignInt(header_size),
+               segment_capacity ? segment_capacity : kHeaderSegmentCapacity,
+               segment_capacity ? segment_capacity : kDefaultSegmentCapacity),
       header_(nullptr),
       header_size_(AlignInt(header_size)) {
   DCHECK(static_cast<memberAlignmentType>(header_size) >= sizeof(Header));
   DCHECK(header_size_ <= kHeaderSegmentCapacity);
   header_ = reinterpret_cast<Header*>(buffers_.Start());
   header_->payload_size = 0;
 }
 
--- a/ipc/chromium/src/base/pickle.h
+++ b/ipc/chromium/src/base/pickle.h
@@ -64,17 +64,17 @@ class Pickle {
  public:
   ~Pickle();
 
   Pickle() = delete;
 
   // Initialize a Pickle object with the specified header size in bytes, which
   // must be greater-than-or-equal-to sizeof(Pickle::Header).  The header size
   // will be rounded up to ensure that the header size is 32bit-aligned.
-  explicit Pickle(uint32_t header_size);
+  explicit Pickle(uint32_t header_size, size_t segment_capacity = 0);
 
   Pickle(uint32_t header_size, const char* data, uint32_t length);
 
   Pickle(const Pickle& other) = delete;
 
   Pickle(Pickle&& other);
 
   // Performs a deep copy.
--- a/ipc/chromium/src/chrome/common/ipc_message.cc
+++ b/ipc/chromium/src/chrome/common/ipc_message.cc
@@ -49,19 +49,25 @@ Message::Message()
     GetCurTraceInfo(&_header->source_event_id,
                     &_header->parent_task_id,
                     &_header->source_event_type);
   }
 #endif
   InitLoggingVariables();
 }
 
-Message::Message(int32_t routing_id, msgid_t type, NestedLevel nestedLevel, PriorityValue priority,
-                 MessageCompression compression, const char* const aName, bool recordWriteLatency)
-    : Pickle(MSG_HEADER_SZ) {
+Message::Message(int32_t routing_id,
+                 msgid_t type,
+                 uint32_t segment_capacity,
+                 NestedLevel nestedLevel,
+                 PriorityValue priority,
+                 MessageCompression compression,
+                 const char* const aName,
+                 bool recordWriteLatency)
+    : Pickle(MSG_HEADER_SZ, segment_capacity) {
   MOZ_COUNT_CTOR(IPC::Message);
   header()->routing = routing_id;
   header()->type = type;
   header()->flags = nestedLevel;
   if (priority == HIGH_PRIORITY)
     header()->flags |= PRIO_BIT;
   if (compression == COMPRESSION_ENABLED)
     header()->flags |= COMPRESS_BIT;
--- a/ipc/chromium/src/chrome/common/ipc_message.h
+++ b/ipc/chromium/src/chrome/common/ipc_message.h
@@ -62,16 +62,17 @@ class Message : public Pickle {
 
   // Initialize a message with a user-defined type, priority value, and
   // destination WebView ID.
   //
   // NOTE: `recordWriteLatency` is only passed by IPDL generated message code,
   // and is used to trigger the IPC_WRITE_LATENCY_MS telemetry.
   Message(int32_t routing_id,
           msgid_t type,
+          uint32_t segmentCapacity = 0, // 0 for the default capacity.
           NestedLevel nestedLevel = NOT_NESTED,
           PriorityValue priority = NORMAL_PRIORITY,
           MessageCompression compression = COMPRESSION_NONE,
           const char* const name="???",
           bool recordWriteLatency=false);
 
   Message(const char* data, int data_len);
 
--- a/ipc/glue/ProtocolUtils.cpp
+++ b/ipc/glue/ProtocolUtils.cpp
@@ -71,16 +71,17 @@ class ChannelOpened : public IPC::Messag
 {
 public:
   ChannelOpened(TransportDescriptor aDescriptor,
                 ProcessId aOtherProcess,
                 ProtocolId aProtocol,
                 NestedLevel aNestedLevel = NOT_NESTED)
     : IPC::Message(MSG_ROUTING_CONTROL, // these only go to top-level actors
                    CHANNEL_OPENED_MESSAGE_TYPE,
+                   0,
                    aNestedLevel)
   {
     IPC::WriteParam(this, aDescriptor);
     IPC::WriteParam(this, aOtherProcess);
     IPC::WriteParam(this, static_cast<uint32_t>(aProtocol));
   }
 
   static bool Read(const IPC::Message& aMsg,
--- a/ipc/glue/Shmem.cpp
+++ b/ipc/glue/Shmem.cpp
@@ -20,17 +20,17 @@ class ShmemCreated : public IPC::Message
 private:
   typedef Shmem::id_t id_t;
 
 public:
   ShmemCreated(int32_t routingId,
                id_t aIPDLId,
                size_t aSize,
                SharedMemory::SharedMemoryType aType) :
-    IPC::Message(routingId, SHMEM_CREATED_MESSAGE_TYPE, NESTED_INSIDE_CPOW)
+    IPC::Message(routingId, SHMEM_CREATED_MESSAGE_TYPE, 0, NESTED_INSIDE_CPOW)
   {
     IPC::WriteParam(this, aIPDLId);
     IPC::WriteParam(this, aSize);
     IPC::WriteParam(this, int32_t(aType));
   }
 
   static bool
   ReadInfo(const Message* msg, PickleIterator* iter,
--- a/ipc/ipdl/Makefile.in
+++ b/ipc/ipdl/Makefile.in
@@ -15,16 +15,17 @@ include $(topsrcdir)/config/rules.mk
 
 # NB: the IPDL compiler manages .ipdl-->.h/.cpp dependencies itself,
 # which is why we don't have explicit .h/.cpp targets here
 ipdl: $(ALL_IPDLSRCS)
 	$(PYTHON) $(topsrcdir)/config/pythonpath.py \
 	  $(PLY_INCLUDE) \
 	  $(srcdir)/ipdl.py \
 	  --sync-msg-list=$(srcdir)/sync-messages.ini \
+	  --msg-metadata=$(srcdir)/message-metadata.ini \
 	  --outheaders-dir=_ipdlheaders \
 	  --outcpp-dir=. \
 	  $(IPDLDIRS:%=-I%) \
 	  $^
 
 .PHONY: ipdl
 
 export:: ipdl
--- a/ipc/ipdl/ipdl.py
+++ b/ipc/ipdl/ipdl.py
@@ -18,16 +18,18 @@ def log(minv, fmt, *args):
 # process command line
 
 op = optparse.OptionParser(usage='ipdl.py [options] IPDLfiles...')
 op.add_option('-I', '--include', dest='includedirs', default=[ ],
               action='append',
               help='Additional directory to search for included protocol specifications')
 op.add_option('-s', '--sync-msg-list', dest='syncMsgList', default='sync-messages.ini',
               help="Config file listing allowed sync messages")
+op.add_option('-m', '--msg-metadata', dest='msgMetadata', default='message-metadata.ini',
+              help="Predicted message sizes for reducing serialization malloc overhead.")
 op.add_option('-v', '--verbose', dest='verbosity', default=1, action='count',
               help='Verbose logging (specify -vv or -vvv for very verbose logging)')
 op.add_option('-q', '--quiet', dest='verbosity', action='store_const', const=0,
               help="Suppress logging output")
 op.add_option('-d', '--outheaders-dir', dest='headersdir', default='.',
               help="""Directory into which C++ headers will be generated.
 A protocol Foo in the namespace bar will cause the headers
   dir/bar/Foo.h, dir/bar/FooParent.h, and dir/bar/FooParent.h
@@ -36,16 +38,17 @@ op.add_option('-o', '--outcpp-dir', dest
               help="""Directory into which C++ sources will be generated
 A protocol Foo in the namespace bar will cause the sources
   cppdir/FooParent.cpp, cppdir/FooChild.cpp
 to be generated""")
 
 options, files = op.parse_args()
 _verbosity = options.verbosity
 syncMsgList = options.syncMsgList
+msgMetadata = options.msgMetadata
 headersdir = options.headersdir
 cppdir = options.cppdir
 includedirs = [ os.path.abspath(incdir) for incdir in options.includedirs ]
 
 if not len(files):
     op.error("No IPDL files specified")
 
 ipcmessagestartpath = os.path.join(headersdir, 'IPCMessageStart.h')
@@ -104,28 +107,41 @@ earliestoutputmod = min(outputModTime(f)
 
 if latestipdlmod < earliestoutputmod:
     sys.exit(0)
 
 log(2, 'Generated C++ headers will be generated relative to "%s"', headersdir)
 log(2, 'Generated C++ sources will be generated in "%s"', cppdir)
 
 allmessages = {}
+allmessageprognames = []
 allprotocols = []
 
 def normalizedFilename(f):
     if f == '-':
         return '<stdin>'
     return f
 
 log(2, 'Reading sync message list')
 parser = RawConfigParser()
 parser.readfp(open(options.syncMsgList))
 syncMsgList = parser.sections()
 
+# Read message metadata. Right now we only have 'segment_capacity'
+# for the standard segment size used for serialization.
+log(2, 'Reading message metadata...')
+msgMetadataConfig = RawConfigParser()
+msgMetadataConfig.readfp(open(options.msgMetadata))
+
+segmentCapacityDict = {}
+for msgName in msgMetadataConfig.sections():
+    if msgMetadataConfig.has_option(msgName, 'segment_capacity'):
+        capacity = msgMetadataConfig.get(msgName, 'segment_capacity')
+        segmentCapacityDict[msgName] = capacity
+
 # First pass: parse and type-check all protocols
 for f in files:
     log(2, os.path.basename(f))
     filename = normalizedFilename(f)
     if f == '-':
         fd = sys.stdin
     else:
         fd = open(f)
@@ -155,24 +171,35 @@ if not ipdl.checkFixedSyncMessages(parse
     # Errors have alraedy been printed to stderr, just exit
     sys.exit(1)
 
 # Second pass: generate code
 for f in files:
     # Read from parser cache
     filename = normalizedFilename(f)
     ast = ipdl.parse(None, filename, includedirs=includedirs)
-    ipdl.gencxx(filename, ast, headersdir, cppdir)
+    ipdl.gencxx(filename, ast, headersdir, cppdir, segmentCapacityDict)
 
     if ast.protocol:
         allmessages[ast.protocol.name] = ipdl.genmsgenum(ast)
         allprotocols.append('%sMsgStart' % ast.protocol.name)
+        # e.g. PContent::RequestMemoryReport (not prefixed or suffixed.)
+        for md in ast.protocol.messageDecls:
+            allmessageprognames.append('%s::%s' % (md.namespace, md.decl.progname))
 
 allprotocols.sort()
 
+# Check if we have undefined message names in segmentCapacityDict.
+# This is a fool-proof of the 'message-metadata.ini' file.
+undefinedMessages = set(segmentCapacityDict.keys()) - set(allmessageprognames)
+if len(undefinedMessages) > 0:
+    print >>sys.stderr, 'Error: Undefined message names in message-metadata.ini:'
+    print >>sys.stderr, undefinedMessages
+    sys.exit(1)
+
 ipcmsgstart = StringIO()
 
 print >>ipcmsgstart, """
 // CODE GENERATED by ipdl.py. Do not edit.
 
 #ifndef IPCMessageStart_h
 #define IPCMessageStart_h
 
--- a/ipc/ipdl/ipdl/__init__.py
+++ b/ipc/ipdl/ipdl/__init__.py
@@ -36,18 +36,18 @@ def parse(specstring, filename='/stdin',
         return None
 
 def typecheck(ast, errout=sys.stderr):
     '''Return True iff |ast| is well typed.  Print errors to |errout| if
     it is not.'''
     return TypeCheck().check(ast, errout)
 
 
-def gencxx(ipdlfilename, ast, outheadersdir, outcppdir):
-    headers, cpps = LowerToCxx().lower(ast)
+def gencxx(ipdlfilename, ast, outheadersdir, outcppdir, segmentcapacitydict):
+    headers, cpps = LowerToCxx().lower(ast, segmentcapacitydict)
 
     def resolveHeader(hdr):
         return [
             hdr,
             os.path.join(
                 outheadersdir,
                 *([ns.name for ns in ast.namespaces] + [hdr.name]))
         ]
--- a/ipc/ipdl/ipdl/lower.py
+++ b/ipc/ipdl/ipdl/lower.py
@@ -10,28 +10,28 @@ import ipdl.ast
 import ipdl.builtin
 from ipdl.cxx.ast import *
 from ipdl.type import ActorType, TypeVisitor, builtinHeaderIncludes
 
 ##-----------------------------------------------------------------------------
 ## "Public" interface to lowering
 ##
 class LowerToCxx:
-    def lower(self, tu):
+    def lower(self, tu, segmentcapacitydict):
         '''returns |[ header: File ], [ cpp : File ]| representing the
 lowered form of |tu|'''
         # annotate the AST with IPDL/C++ IR-type stuff used later
         tu.accept(_DecorateWithCxxStuff())
 
         # Any modifications to the filename scheme here need corresponding
         # modifications in the ipdl.py driver script.
         name = tu.name
         pheader, pcpp = File(name +'.h'), File(name +'.cpp')
 
-        _GenerateProtocolCode().lower(tu, pheader, pcpp)
+        _GenerateProtocolCode().lower(tu, pheader, pcpp, segmentcapacitydict)
         headers = [ pheader ]
         cpps = [ pcpp ]
 
         if tu.protocol:
             pname = tu.protocol.name
 
             parentheader, parentcpp = File(pname +'Parent.h'), File(pname +'Parent.cpp')
             _GenerateProtocolParentCode().lower(
@@ -1412,20 +1412,21 @@ class _GenerateProtocolCode(ipdl.ast.Vis
     def __init__(self):
         self.protocol = None     # protocol we're generating a class for
         self.hdrfile = None      # what will become Protocol.h
         self.cppfile = None      # what will become Protocol.cpp
         self.cppIncludeHeaders = []
         self.structUnionDefns = []
         self.funcDefns = []
 
-    def lower(self, tu, cxxHeaderFile, cxxFile):
+    def lower(self, tu, cxxHeaderFile, cxxFile, segmentcapacitydict):
         self.protocol = tu.protocol
         self.hdrfile = cxxHeaderFile
         self.cppfile = cxxFile
+        self.segmentcapacitydict = segmentcapacitydict
         tu.accept(self)
 
     def visitTranslationUnit(self, tu):
         hf = self.hdrfile
 
         hf.addthing(_DISCLAIMER)
         hf.addthings(_includeGuardStart(hf))
         hf.addthing(Whitespace.NL)
@@ -1572,29 +1573,37 @@ class _GenerateProtocolCode(ipdl.ast.Vis
 
         tfDecl, tfDefn = _splitFuncDeclDefn(self.genTransitionFunc())
         ns.addstmts([ tfDecl, Whitespace.NL ])
         self.funcDefns.append(tfDefn)
 
         for md in p.messageDecls:
             decls = []
 
+            # Look up the segment capacity used for serializing this
+            # message. If the capacity is not specified, use '0' for
+            # the default capacity (defined in ipc_message.cc)
+            name = '%s::%s' % (md.namespace, md.decl.progname)
+            segmentcapacity = self.segmentcapacitydict.get(name, 0)
+
             mfDecl, mfDefn = _splitFuncDeclDefn(
                 _generateMessageConstructor(md.msgCtorFunc(), md.msgId(),
+                                            segmentcapacity,
                                             md.decl.type.nested,
                                             md.decl.type.prio,
                                             md.prettyMsgName(p.name+'::'),
                                             md.decl.type.compress))
             decls.append(mfDecl)
             self.funcDefns.append(mfDefn)
 
             if md.hasReply():
                 rfDecl, rfDefn = _splitFuncDeclDefn(
                     _generateMessageConstructor(
                         md.replyCtorFunc(), md.replyId(),
+                        0,
                         md.decl.type.nested,
                         md.decl.type.prio,
                         md.prettyReplyName(p.name+'::'),
                         md.decl.type.compress))
                 decls.append(rfDecl)
                 self.funcDefns.append(rfDefn)
 
             decls.append(Whitespace.NL)
@@ -1687,17 +1696,17 @@ class _GenerateProtocolCode(ipdl.ast.Vis
         fromswitch.addcase(DefaultLabel(), unreachedblock)
 
         transitionfunc.addstmt(fromswitch)
 
         return transitionfunc
 
 ##--------------------------------------------------
 
-def _generateMessageConstructor(clsname, msgid, nested, prio, prettyName, compress):
+def _generateMessageConstructor(clsname, msgid, segmentSize, nested, prio, prettyName, compress):
     routingId = ExprVar('routingId')
 
     func = FunctionDefn(FunctionDecl(
         clsname,
         params=[ Decl(Type('int32_t'), routingId.name) ],
         ret=Type('IPC::Message', ptr=1)))
 
     if compress == 'compress':
@@ -1721,16 +1730,17 @@ def _generateMessageConstructor(clsname,
     else:
         assert prio == ipdl.ast.HIGH_PRIORITY
         prioEnum = 'IPC::Message::HIGH_PRIORITY'
 
     func.addstmt(
         StmtReturn(ExprNew(Type('IPC::Message'),
                            args=[ routingId,
                                   ExprVar(msgid),
+                                  ExprLiteral.Int(int(segmentSize)),
                                   ExprVar(nestedEnum),
                                   ExprVar(prioEnum),
                                   compression,
                                   ExprLiteral.String(prettyName),
                                   # Pass `true` to recordWriteLatency to collect telemetry
                                   ExprLiteral.TRUE ])))
 
     return func
new file mode 100644
--- /dev/null
+++ b/ipc/ipdl/message-metadata.ini
@@ -0,0 +1,31 @@
+#############################################################
+#                                                           #
+# Any changes to this file must be reviewed by an IPC peer. #
+#                                                           #
+#############################################################
+
+[PContent::AccumulateChildKeyedHistograms]
+segment_capacity = 16384
+[PContent::AccumulateChildHistograms]
+segment_capacity = 16384
+[PLayerTransaction::Update]
+segment_capacity = 8192
+[PContent::StoreAndBroadcastBlobURLRegistration]
+segment_capacity = 8192
+[PHttpChannel::OnStartRequest]
+segment_capacity = 8192
+[PHttpChannel::Redirect1Begin]
+segment_capacity = 8192
+[PHttpBackgroundChannel::OnTransportAndData]
+segment_capacity = 8192
+[PNecko::PHttpChannelConstructor]
+segment_capacity = 8192
+[PWyciwygChannel::WriteToCacheEntry]
+segment_capacity = 8192
+[PWyciwygChannel::SetSecurityInfo]
+segment_capacity = 8192
+[PMessagePort::PostMessages]
+segment_capacity = 12288
+[PMessagePort::ReceiveData]
+segment_capacity = 12288
+