Bug 598866, part 1: Add library support for an "unsafe" Shmem variant. r=joe

ipc/glue/Shmem.cpp

@@ -216,14 +216,16 @@ static const char sMagic[] =
 
 
 struct Header {
-  // Don't use size_t here because the data type's length depends
-  // on the architecture.
+  // Don't use size_t or bool here because their size depends on the
+  // architecture.
   uint32 mSize;
+  uint32 mUnsafe;
   char mMagic[sizeof(sMagic)];
 };
 
 static void
 GetSections(Shmem::SharedMemory* aSegment,
+            Header** aHeader,
             char** aFrontSentinel,
             char** aData,
             char** aBackSentinel)
@@ -234,12 +236,23 @@ GetSections(Shmem::SharedMemory* aSegment,
   *aFrontSentinel = reinterpret_cast<char*>(aSegment->memory());
   NS_ABORT_IF_FALSE(*aFrontSentinel, "NULL memory()");
 
+  *aHeader = reinterpret_cast<Header*>(*aFrontSentinel);
+
   size_t pageSize = Shmem::SharedMemory::SystemPageSize();
   *aData = *aFrontSentinel + pageSize;
 
   *aBackSentinel = *aFrontSentinel + aSegment->Size() - pageSize;
 }
 
+static Header*
+GetHeader(Shmem::SharedMemory* aSegment)
+{
+  Header* header;
+  char* dontcare;
+  GetSections(aSegment, &header, &dontcare, &dontcare, &dontcare);
+  return header;
+}
+
 static void
 Protect(SharedMemory* aSegment)
 {
@@ -318,16 +331,16 @@ Shmem::Shmem(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
 
   Unprotect(mSegment);
 
+  Header* header;
   char* frontSentinel;
   char* data;
   char* backSentinel;
-  GetSections(aSegment, &frontSentinel, &data, &backSentinel);
+  GetSections(aSegment, &header, &frontSentinel, &data, &backSentinel);
 
   // do a quick validity check to avoid weird-looking crashes in libc
   char check = *frontSentinel;
   (void)check;
 
-  Header* header = reinterpret_cast<Header*>(frontSentinel);
   NS_ABORT_IF_FALSE(!strncmp(header->mMagic, sMagic, sizeof(sMagic)),
                       "invalid segment");
   mSize = static_cast<size_t>(header->mSize);
@@ -360,7 +373,18 @@ void
 Shmem::RevokeRights(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead)
 {
   AssertInvariants();
-  Protect(mSegment);
+
+  size_t pageSize = SharedMemory::SystemPageSize();
+  Header* header = GetHeader(mSegment);
+
+  // Open this up for reading temporarily
+  mSegment->Protect(reinterpret_cast<char*>(header), pageSize, RightsRead);
+
+  if (!header->mUnsafe) {
+    Protect(mSegment);
+  } else {
+    mSegment->Protect(reinterpret_cast<char*>(header), pageSize, RightsNone);
+  }
 }
 
 // static
@@ -368,9 +392,11 @@ Shmem::SharedMemory*
 Shmem::Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
              size_t aNBytes,
              SharedMemoryType aType,
+             bool aUnsafe,
              bool aProtect)
 {
   NS_ASSERTION(aNBytes <= PR_UINT32_MAX, "Will truncate shmem segment size!");
+  NS_ABORT_IF_FALSE(!aProtect || !aUnsafe, "protect => !unsafe");
 
   size_t pageSize = SharedMemory::SystemPageSize();
   SharedMemory* segment = nsnull;
@@ -389,15 +415,22 @@ Shmem::Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
   if (!segment)
     return 0;
 
+  Header* header;
   char *frontSentinel;
   char *data;
   char *backSentinel;
-  GetSections(segment, &frontSentinel, &data, &backSentinel);
+  GetSections(segment, &header, &frontSentinel, &data, &backSentinel);
 
   // initialize the segment with Shmem-internal information
-  Header* header = reinterpret_cast<Header*>(frontSentinel);
+
+  // NB: this can't be a static assert because technically pageSize
+  // isn't known at compile time, event though in practice it's always
+  // going to be 4KiB
+  NS_ABORT_IF_FALSE(sizeof(Header) <= pageSize,
+                    "Shmem::Header has gotten too big");
   memcpy(header->mMagic, sMagic, sizeof(sMagic));
   header->mSize = static_cast<uint32>(aNBytes);
+  header->mUnsafe = aUnsafe;
 
   if (aProtect)
     Protect(segment);
@@ -453,7 +486,10 @@ Shmem::OpenExisting(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
   if (!segment)
     return 0;
 
-  if (aProtect)
+  // The caller of this function may not know whether the segment is
+  // unsafe or not
+  Header* header = GetHeader(segment);
+  if (!header->mUnsafe && aProtect)
     Protect(segment);
 
   return segment;
@@ -468,15 +504,16 @@ Shmem::Dealloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
     return;
 
   size_t pageSize = SharedMemory::SystemPageSize();
+  Header* header;
   char *frontSentinel;
   char *data;
   char *backSentinel;
-  GetSections(aSegment, &frontSentinel, &data, &backSentinel);
+  GetSections(aSegment, &header, &frontSentinel, &data, &backSentinel);
 
   aSegment->Protect(frontSentinel, pageSize, RightsWrite | RightsRead);
-  Header* header = reinterpret_cast<Header*>(frontSentinel);
   memset(header->mMagic, 0, sizeof(sMagic));
   header->mSize = 0;
+  header->mUnsafe = false;          // make it "safe" so as to catch errors
 
   DestroySegment(aSegment);
 }
@@ -489,6 +526,7 @@ Shmem::SharedMemory*
 Shmem::Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
              size_t aNBytes, 
              SharedMemoryType aType,
+             bool /*unused*/,
              bool /*unused*/)
 {
   SharedMemory *segment = nsnull;

ipc/glue/Shmem.h

@@ -228,6 +228,7 @@ public:
   Alloc(IHadBetterBeIPDLCodeCallingThis_OtherwiseIAmADoodyhead,
         size_t aNBytes,
         SharedMemoryType aType,
+        bool aUnsafe,
         bool aProtect=false);
 
   // Prepare this to be shared with |aProcess|.  Return an IPC message

ipc/ipdl/ipdl/lower.py

@@ -191,7 +191,7 @@ def _shmemSegment(shmemexpr):
 def _shmemAlloc(size, type):
     # starts out UNprotected
     return ExprCall(ExprVar('Shmem::Alloc'),
-                    args=[ _shmemBackstagePass(), size, type ])
+                    args=[ _shmemBackstagePass(), size, type, ExprLiteral.FALSE ])
 
 def _shmemDealloc(rawmemvar):
     return ExprCall(ExprVar('Shmem::Dealloc'),