Bug 1162358 - Defer processing GMP DecodingComplete() calls until intr shmem allocs are finished. r=jesup

dom/media/gmp/GMPContentChild.cpp

@@ -78,13 +78,15 @@ GMPContentChild::DeallocPGMPDecryptorChild(PGMPDecryptorChild* aActor)
 PGMPVideoDecoderChild*
 GMPContentChild::AllocPGMPVideoDecoderChild()
 {
-  return new GMPVideoDecoderChild(this);
+  GMPVideoDecoderChild* actor = new GMPVideoDecoderChild(this);
+  actor->AddRef();
+  return actor;
 }
 
 bool
 GMPContentChild::DeallocPGMPVideoDecoderChild(PGMPVideoDecoderChild* aActor)
 {
-  delete aActor;
+  static_cast<GMPVideoDecoderChild*>(aActor)->Release();
   return true;
 }
 

dom/media/gmp/GMPVideoDecoderChild.cpp

@@ -9,21 +9,25 @@
 #include <stdio.h>
 #include "mozilla/unused.h"
 #include "GMPVideoEncodedFrameImpl.h"
+#include "runnable_utils.h"
 
 namespace mozilla {
 namespace gmp {
 
 GMPVideoDecoderChild::GMPVideoDecoderChild(GMPContentChild* aPlugin)
-: GMPSharedMemManager(aPlugin),
+  : GMPSharedMemManager(aPlugin)
-  mPlugin(aPlugin),
+  , mPlugin(aPlugin)
-  mVideoDecoder(nullptr),
+  , mVideoDecoder(nullptr)
-  mVideoHost(this)
+  , mVideoHost(this)
+  , mNeedShmemIntrCount(0)
+  , mPendingDecodeComplete(false)
 {
   MOZ_ASSERT(mPlugin);
 }
 
 GMPVideoDecoderChild::~GMPVideoDecoderChild()
 {
+  MOZ_ASSERT(!mNeedShmemIntrCount);
 }
 
 void
@@ -184,6 +188,16 @@ GMPVideoDecoderChild::RecvDrain()
 bool
 GMPVideoDecoderChild::RecvDecodingComplete()
 {
+  MOZ_ASSERT(mPlugin->GMPMessageLoop() == MessageLoop::current());
+
+  if (mNeedShmemIntrCount) {
+    // There's a GMP blocked in Alloc() waiting for the CallNeedShem() to
+    // return a frame they can use. Don't call the GMP's DecodingComplete()
+    // now and don't delete the GMPVideoDecoderChild, defer processing the
+    // DecodingComplete() until once the Alloc() finishes.
+    mPendingDecodeComplete = true;
+    return true;
+  }
   if (mVideoDecoder) {
     // Ignore any return code. It is OK for this to fail without killing the process.
     mVideoDecoder->DecodingComplete();
@@ -199,5 +213,41 @@ GMPVideoDecoderChild::RecvDecodingComplete()
   return true;
 }
 
+bool
+GMPVideoDecoderChild::Alloc(size_t aSize,
+                            Shmem::SharedMemory::SharedMemoryType aType,
+                            Shmem* aMem)
+{
+  MOZ_ASSERT(mPlugin->GMPMessageLoop() == MessageLoop::current());
+
+  bool rv;
+#ifndef SHMEM_ALLOC_IN_CHILD
+  ++mNeedShmemIntrCount;
+  rv = CallNeedShmem(aSize, aMem);
+  --mNeedShmemIntrCount;
+  if (mPendingDecodeComplete) {
+    auto t = NewRunnableMethod(this, &GMPVideoDecoderChild::RecvDecodingComplete);
+    mPlugin->GMPMessageLoop()->PostTask(FROM_HERE, t);
+  }
+#else
+#ifdef GMP_SAFE_SHMEM
+  rv = AllocShmem(aSize, aType, aMem);
+#else
+  rv = AllocUnsafeShmem(aSize, aType, aMem);
+#endif
+#endif
+  return rv;
+}
+
+void
+GMPVideoDecoderChild::Dealloc(Shmem& aMem)
+{
+#ifndef SHMEM_ALLOC_IN_CHILD
+  SendParentShmemForPool(aMem);
+#else
+  DeallocShmem(aMem);
+#endif
+}
+
 } // namespace gmp
 } // namespace mozilla

dom/media/gmp/GMPVideoDecoderChild.h

@@ -23,8 +23,9 @@ class GMPVideoDecoderChild : public PGMPVideoDecoderChild,
                              public GMPSharedMemManager
 {
 public:
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GMPVideoDecoderChild);
+
   explicit GMPVideoDecoderChild(GMPContentChild* aPlugin);
-  virtual ~GMPVideoDecoderChild();
 
   void Init(GMPVideoDecoder* aDecoder);
   GMPVideoHostImpl& Host();
@@ -39,28 +40,12 @@ public:
   virtual void Error(GMPErr aError) override;
 
   // GMPSharedMemManager
-  virtual bool Alloc(size_t aSize, Shmem::SharedMemory::SharedMemoryType aType, Shmem* aMem) override
+  virtual bool Alloc(size_t aSize, Shmem::SharedMemory::SharedMemoryType aType, Shmem* aMem) override;
-  {
+  virtual void Dealloc(Shmem& aMem) override;
-#ifndef SHMEM_ALLOC_IN_CHILD
-    return CallNeedShmem(aSize, aMem);
-#else
-#ifdef GMP_SAFE_SHMEM
-    return AllocShmem(aSize, aType, aMem);
-#else
-    return AllocUnsafeShmem(aSize, aType, aMem);
-#endif
-#endif
-  }
-  virtual void Dealloc(Shmem& aMem) override
-  {
-#ifndef SHMEM_ALLOC_IN_CHILD
-    SendParentShmemForPool(aMem);
-#else
-    DeallocShmem(aMem);
-#endif
-  }
 
 private:
+  virtual ~GMPVideoDecoderChild();
+
   // PGMPVideoDecoderChild
   virtual bool RecvInitDecode(const GMPVideoCodec& aCodecSettings,
                               InfallibleTArray<uint8_t>&& aCodecSpecific,
@@ -77,6 +62,11 @@ private:
   GMPContentChild* mPlugin;
   GMPVideoDecoder* mVideoDecoder;
   GMPVideoHostImpl mVideoHost;
+
+  // Non-zero when a GMP is blocked spinning the IPC message loop while
+  // waiting on an NeedShmem to complete.
+  int mNeedShmemIntrCount;
+  bool mPendingDecodeComplete;
 };
 
 } // namespace gmp