Bug 599059: Always store length at the end of shared memory segments as a 32-bit value. Don't use "sizeof(size_t)" because that differs between i386 and x86_64 and causes crashes when running i386 plugins from a x86_64 host. r=cjones a=blocking-b7
authorJosh Aas <joshmoz@gmail.com>
Fri, 24 Sep 2010 02:31:47 -0400
changeset 54612 c7ed283dda27f26e020bed851a77ab34ecdce41b
parent 54611 78c90846f8c76e57d7ef25216bc33e12b0e822d0
child 54613 32f406acbb456d0390b950aa2cbe4e2ce7becad6
push idunknown
push userunknown
push dateunknown
reviewerscjones, blocking-b7
bugs599059
milestone2.0b7pre
Bug 599059: Always store length at the end of shared memory segments as a 32-bit value. Don't use "sizeof(size_t)" because that differs between i386 and x86_64 and causes crashes when running i386 plugins from a x86_64 host. r=cjones a=blocking-b7
ipc/glue/Shmem.cpp
ipc/glue/Shmem.h
--- a/ipc/glue/Shmem.cpp
+++ b/ipc/glue/Shmem.cpp
@@ -365,16 +365,18 @@ Shmem::RevokeRights(IHadBetterBeIPDLCode
 
 // static
 Shmem::SharedMemory*
 Shmem::Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
              size_t aNBytes,
              SharedMemoryType aType,
              bool aProtect)
 {
+  NS_ASSERTION(aNBytes <= PR_UINT32_MAX, "Will truncate shmem segment size!");
+
   size_t pageSize = SharedMemory::SystemPageSize();
   SharedMemory* segment = nsnull;
   // |2*pageSize| is for the front and back sentinel
   size_t segmentSize = PageAlignedSize(aNBytes + 2*pageSize);
 
   if (aType == SharedMemory::TYPE_BASIC)
     segment = CreateSegment(segmentSize, SharedMemoryBasic::NULLHandle());
 #ifdef MOZ_HAVE_SHAREDMEMORYSYSV
@@ -390,17 +392,16 @@ Shmem::Alloc(IHadBetterBeIPDLCodeCalling
   char *frontSentinel;
   char *data;
   char *backSentinel;
   GetSections(segment, &frontSentinel, &data, &backSentinel);
 
   // initialize the segment with Shmem-internal information
   Header* header = reinterpret_cast<Header*>(frontSentinel);
   memcpy(header->mMagic, sMagic, sizeof(sMagic));
-  NS_ASSERTION(aNBytes <= PR_UINT32_MAX, "Will truncate shmem segment size!");
   header->mSize = static_cast<uint32>(aNBytes);
 
   if (aProtect)
     Protect(segment);
 
   return segment;
 }
 
@@ -488,31 +489,31 @@ Shmem::SharedMemory*
 Shmem::Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
              size_t aNBytes, 
              SharedMemoryType aType,
              bool /*unused*/)
 {
   SharedMemory *segment = nsnull;
 
   if (aType == SharedMemory::TYPE_BASIC)
-    segment = CreateSegment(PageAlignedSize(aNBytes + sizeof(size_t)),
+    segment = CreateSegment(PageAlignedSize(aNBytes + sizeof(uint32)),
                             SharedMemoryBasic::NULLHandle());
 #ifdef MOZ_HAVE_SHAREDMEMORYSYSV
   else if (aType == SharedMemory::TYPE_SYSV)
-    segment = CreateSegment(PageAlignedSize(aNBytes + sizeof(size_t)),
+    segment = CreateSegment(PageAlignedSize(aNBytes + sizeof(uint32)),
                             SharedMemorySysV::NULLHandle());
 #endif
   else
     // Unhandled!!
     NS_ABORT();
 
   if (!segment)
     return 0;
 
-  *PtrToSize(segment) = aNBytes;
+  *PtrToSize(segment) = static_cast<uint32>(aNBytes);
 
   return segment;
 }
 
 // static
 Shmem::SharedMemory*
 Shmem::OpenExisting(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
                     const IPC::Message& aDescriptor,
@@ -555,17 +556,17 @@ Shmem::OpenExisting(IHadBetterBeIPDLCode
   else {
     NS_RUNTIMEABORT("unknown shmem type");
   }
 
   if (!segment)
     return 0;
 
   // this is the only validity check done OPT builds
-  if (size != *PtrToSize(segment))
+  if (size != static_cast<size_t>(*PtrToSize(segment)))
     NS_RUNTIMEABORT("Alloc() segment size disagrees with OpenExisting()'s");
 
   return segment;
 }
 
 // static
 void
 Shmem::Dealloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
--- a/ipc/glue/Shmem.h
+++ b/ipc/glue/Shmem.h
@@ -117,17 +117,17 @@ public:
 #if !defined(DEBUG)
   Shmem(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
         SharedMemory* aSegment, id_t aId) :
     mSegment(aSegment),
     mData(aSegment->memory()),
     mSize(0),
     mId(aId)
   {
-    mSize = *PtrToSize(mSegment);
+    mSize = static_cast<size_t>(*PtrToSize(mSegment));
   }
 #else
   Shmem(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
         SharedMemory* aSegment, id_t aId);
 #endif
 
   ~Shmem()
   {
@@ -269,22 +269,22 @@ private:
     if (0 != (mSize % sizeof(T)))
       NS_RUNTIMEABORT("shmem is not T-aligned");
   }
 
 #if !defined(DEBUG)
   void AssertInvariants() const
   { }
 
-  static size_t*
+  static uint32*
   PtrToSize(SharedMemory* aSegment)
   {
     char* endOfSegment =
       reinterpret_cast<char*>(aSegment->memory()) + aSegment->Size();
-    return reinterpret_cast<size_t*>(endOfSegment - sizeof(size_t));
+    return reinterpret_cast<uint32*>(endOfSegment - sizeof(uint32));
   }
 
 #else
   void AssertInvariants() const;
 #endif
 
   SharedMemory* mSegment;
   void* mData;