Bug 1072877 - Ensure OOP transactions can't use memory backed tile locks. r=bjacob

gfx/layers/TiledLayerBuffer.h

@@ -222,8 +222,11 @@ public:
    * ReadLock state, so that the locks can be adopted when recreating a
    * ClientTiledLayerBuffer locally. This lock will be retained until the buffer
    * has completed uploading.
+   *
+   * Returns false if a deserialization error happened, in which case we will
+   * have to kill the child process.
    */
-  virtual void UseTiledLayerBuffer(ISurfaceAllocator* aAllocator,
+  virtual bool UseTiledLayerBuffer(ISurfaceAllocator* aAllocator,
                                    const SurfaceDescriptorTiles& aTiledDescriptor) = 0;
 
   /**

gfx/layers/composite/TiledContentHost.cpp

@@ -30,7 +30,7 @@ class Layer;
 TiledLayerBufferComposite::TiledLayerBufferComposite()
   : mFrameResolution(1.0)
   , mHasDoubleBufferedTiles(false)
-  , mUninitialized(true)
+  , mIsValid(false)
 {}
 
 /* static */ void
@@ -43,7 +43,7 @@ TiledLayerBufferComposite::TiledLayerBufferComposite(ISurfaceAllocator* aAllocat
                                                      const SurfaceDescriptorTiles& aDescriptor,
                                                      const nsIntRegion& aOldPaintedRegion)
 {
-  mUninitialized = false;
+  mIsValid = true;
   mHasDoubleBufferedTiles = false;
   mValidRegion = aDescriptor.validRegion();
   mPaintedRegion = aDescriptor.paintedRegion();
@@ -57,6 +57,8 @@ TiledLayerBufferComposite::TiledLayerBufferComposite(ISurfaceAllocator* aAllocat
   oldPaintedRegion.And(oldPaintedRegion, mValidRegion);
   mPaintedRegion.Or(mPaintedRegion, oldPaintedRegion);
 
+  bool isSameProcess = aAllocator->IsSameProcess();
+
   const InfallibleTArray<TileDescriptor>& tiles = aDescriptor.tiles();
   for(size_t i = 0; i < tiles.Length(); i++) {
     RefPtr<TextureHost> texture;
@@ -74,6 +76,17 @@ TiledLayerBufferComposite::TiledLayerBufferComposite(ISurfaceAllocator* aAllocat
         if (ipcLock.type() == TileLock::TShmemSection) {
           sharedLock = gfxShmSharedReadLock::Open(aAllocator, ipcLock.get_ShmemSection());
         } else {
+          if (!isSameProcess) {
+            // Trying to use a memory based lock instead of a shmem based one in
+            // the cross-process case is a bad security violation.
+            NS_ERROR("A client process may be trying to peek at the host's address space!");
+            // This tells the TiledContentHost that deserialization failed so that
+            // it can propagate the error.
+            mIsValid = false;
+
+            mRetainedTiles.Clear();
+            return;
+          }
           sharedLock = reinterpret_cast<gfxMemorySharedReadLock*>(ipcLock.get_uintptr_t());
           if (sharedLock) {
             // The corresponding AddRef is in TiledClient::GetTileDescriptor
@@ -287,7 +300,7 @@ TiledContentHost::Detach(Layer* aLayer,
   CompositableHost::Detach(aLayer,aFlags);
 }
 
-void
+bool
 TiledContentHost::UseTiledLayerBuffer(ISurfaceAllocator* aAllocator,
                                       const SurfaceDescriptorTiles& aTiledDescriptor)
 {
@@ -310,6 +323,14 @@ TiledContentHost::UseTiledLayerBuffer(ISurfaceAllocator* aAllocator,
     mLowPrecisionTiledBuffer =
       TiledLayerBufferComposite(aAllocator, aTiledDescriptor,
                                 mLowPrecisionTiledBuffer.GetPaintedRegion());
+    if (!mLowPrecisionTiledBuffer.IsValid()) {
+      // Something bad happened. Stop here, return false (kills the child process),
+      // and do as little work as possible on the received data as it appears
+      // to be corrupted.
+      mPendingLowPrecisionUpload = false;
+      mPendingUpload = false;
+      return false;
+    }
   } else {
     if (mPendingUpload) {
       mTiledBuffer.ReadUnlock();
@@ -322,7 +343,16 @@ TiledContentHost::UseTiledLayerBuffer(ISurfaceAllocator* aAllocator,
     }
     mTiledBuffer = TiledLayerBufferComposite(aAllocator, aTiledDescriptor,
                                              mTiledBuffer.GetPaintedRegion());
+    if (!mTiledBuffer.IsValid()) {
+      // Something bad happened. Stop here, return false (kills the child process),
+      // and do as little work as possible on the received data as it appears
+      // to be corrupted.
+      mPendingLowPrecisionUpload = false;
+      mPendingUpload = false;
+      return false;
+    }
   }
+  return true;
 }
 
 void

gfx/layers/composite/TiledContentHost.h

@@ -137,7 +137,7 @@ public:
 
   bool HasDoubleBufferedTiles() { return mHasDoubleBufferedTiles; }
 
-  bool IsValid() const { return !mUninitialized; }
+  bool IsValid() const { return mIsValid; }
 
 #if defined(MOZ_WIDGET_GONK) && ANDROID_VERSION >= 17
   virtual void SetReleaseFence(const android::sp<android::Fence>& aReleaseFence);
@@ -162,7 +162,7 @@ protected:
 private:
   CSSToParentLayerScale mFrameResolution;
   bool mHasDoubleBufferedTiles;
-  bool mUninitialized;
+  bool mIsValid;
 };
 
 /**
@@ -215,8 +215,8 @@ public:
     return mLowPrecisionTiledBuffer.GetValidRegion();
   }
 
-  void UseTiledLayerBuffer(ISurfaceAllocator* aAllocator,
-                           const SurfaceDescriptorTiles& aTiledDescriptor);
+  virtual bool UseTiledLayerBuffer(ISurfaceAllocator* aAllocator,
+                                   const SurfaceDescriptorTiles& aTiledDescriptor) MOZ_OVERRIDE;
 
   void Composite(EffectChain& aEffectChain,
                  float aOpacity,

gfx/layers/ipc/CompositableTransactionParent.cpp

@@ -148,7 +148,10 @@ CompositableParentManager::ReceiveCompositableUpdate(const CompositableOperation
       NS_ASSERTION(tileComposer, "compositable is not a tile composer");
 
       const SurfaceDescriptorTiles& tileDesc = op.tileLayerDescriptor();
-      tileComposer->UseTiledLayerBuffer(this, tileDesc);
+      bool success = tileComposer->UseTiledLayerBuffer(this, tileDesc);
+      if (!success) {
+        return false;
+      }
       break;
     }
     case CompositableOperation::TOpRemoveTexture: {