Bug 1186406 - Copy input to ClearKey's decoder, so we can return its containing shmem earlier. r=gerald

We're failing in the "Very rough kill-switch" case in
GMPVideoDecoderParent::Decode() we find that too many shmems are in use when we
come to send a "Decode" message to the GMP, and that causes an error which
percolates up to cause the test failure.

This patch changes gmp-clearkey to copy the input encrypted and compressed
sample and immediately return the shmem to the parent process. We are
copying the data anyway when we decrypt, so we can rejigg things so that we
don't actually end up doing a second copy.

media/gmp-clearkey/0.1/AudioDecoder.cpp

@@ -119,7 +119,7 @@ AudioDecoder::DecodeTask(GMPAudioSamples* aInput)
     // Plugin host should have set up its decryptor/key sessions
     // before trying to decode!
     GMPErr rv =
-      ClearKeyDecryptionManager::Get()->Decrypt(&buffer[0], buffer.size(), crypto);
+      ClearKeyDecryptionManager::Get()->Decrypt(buffer, CryptoMetaData(crypto));
 
     if (GMP_FAILED(rv)) {
       CK_LOGE("Failed to decrypt with key id %08x...", *(uint32_t*)crypto->KeyId());

media/gmp-clearkey/0.1/ClearKeyDecryptionManager.cpp

@@ -30,7 +30,7 @@ public:
   bool HasKey() const { return !!mKey.size(); }
 
   GMPErr Decrypt(uint8_t* aBuffer, uint32_t aBufferSize,
-                 const GMPEncryptedBufferMetadata* aMetadata);
+                 const CryptoMetaData& aMetadata);
 
   const Key& DecryptionKey() const { return mKey; }
 
@@ -130,18 +130,23 @@ ClearKeyDecryptionManager::ReleaseKeyId(KeyId aKeyId)
   }
 }
 
+GMPErr
+ClearKeyDecryptionManager::Decrypt(std::vector<uint8_t>& aBuffer,
+                                   const CryptoMetaData& aMetadata)
+{
+  return Decrypt(&aBuffer[0], aBuffer.size(), aMetadata);
+}
+
 GMPErr
 ClearKeyDecryptionManager::Decrypt(uint8_t* aBuffer, uint32_t aBufferSize,
-                                   const GMPEncryptedBufferMetadata* aMetadata)
+                                   const CryptoMetaData& aMetadata)
 {
   CK_LOGD("ClearKeyDecryptionManager::Decrypt");
-  KeyId keyId(aMetadata->KeyId(), aMetadata->KeyId() + aMetadata->KeyIdSize());
-
-  if (!HasKeyForKeyId(keyId)) {
+  if (!HasKeyForKeyId(aMetadata.mKeyId)) {
     return GMPNoKeyErr;
   }
 
-  return mDecryptors[keyId]->Decrypt(aBuffer, aBufferSize, aMetadata);
+  return mDecryptors[aMetadata.mKeyId]->Decrypt(aBuffer, aBufferSize, aMetadata);
 }
 
 ClearKeyDecryptor::ClearKeyDecryptor()
@@ -162,21 +167,21 @@ ClearKeyDecryptor::InitKey(const Key& aKey)
 
 GMPErr
 ClearKeyDecryptor::Decrypt(uint8_t* aBuffer, uint32_t aBufferSize,
-                           const GMPEncryptedBufferMetadata* aMetadata)
+                           const CryptoMetaData& aMetadata)
 {
   CK_LOGD("ClearKeyDecryptor::Decrypt");
   // If the sample is split up into multiple encrypted subsamples, we need to
   // stitch them into one continuous buffer for decryption.
   std::vector<uint8_t> tmp(aBufferSize);
 
-  if (aMetadata->NumSubsamples()) {
+  if (aMetadata.NumSubsamples()) {
     // Take all encrypted parts of subsamples and stitch them into one
     // continuous encrypted buffer.
-    unsigned char* data = aBuffer;
-    unsigned char* iter = &tmp[0];
-    for (size_t i = 0; i < aMetadata->NumSubsamples(); i++) {
-      data += aMetadata->ClearBytes()[i];
-      uint32_t cipherBytes = aMetadata->CipherBytes()[i];
+    uint8_t* data = aBuffer;
+    uint8_t* iter = &tmp[0];
+    for (size_t i = 0; i < aMetadata.NumSubsamples(); i++) {
+      data += aMetadata.mClearBytes[i];
+      uint32_t cipherBytes = aMetadata.mCipherBytes[i];
 
       memcpy(iter, data, cipherBytes);
 
@@ -189,20 +194,20 @@ ClearKeyDecryptor::Decrypt(uint8_t* aBuffer, uint32_t aBufferSize,
     memcpy(&tmp[0], aBuffer, aBufferSize);
   }
 
-  assert(aMetadata->IVSize() == 8 || aMetadata->IVSize() == 16);
-  std::vector<uint8_t> iv(aMetadata->IV(), aMetadata->IV() + aMetadata->IVSize());
-  iv.insert(iv.end(), CLEARKEY_KEY_LEN - aMetadata->IVSize(), 0);
+  assert(aMetadata.mIV.size() == 8 || aMetadata.mIV.size() == 16);
+  std::vector<uint8_t> iv(aMetadata.mIV);
+  iv.insert(iv.end(), CLEARKEY_KEY_LEN - aMetadata.mIV.size(), 0);
 
   ClearKeyUtils::DecryptAES(mKey, tmp, iv);
 
-  if (aMetadata->NumSubsamples()) {
+  if (aMetadata.NumSubsamples()) {
     // Take the decrypted buffer, split up into subsamples, and insert those
     // subsamples back into their original position in the original buffer.
-    unsigned char* data = aBuffer;
-    unsigned char* iter = &tmp[0];
-    for (size_t i = 0; i < aMetadata->NumSubsamples(); i++) {
-      data += aMetadata->ClearBytes()[i];
-      uint32_t cipherBytes = aMetadata->CipherBytes()[i];
+    uint8_t* data = aBuffer;
+    uint8_t* iter = &tmp[0];
+    for (size_t i = 0; i < aMetadata.NumSubsamples(); i++) {
+      data += aMetadata.mClearBytes[i];
+      uint32_t cipherBytes = aMetadata.mCipherBytes[i];
 
       memcpy(data, iter, cipherBytes);
 

media/gmp-clearkey/0.1/ClearKeyDecryptionManager.h

@@ -24,6 +24,45 @@
 
 class ClearKeyDecryptor;
 
+class CryptoMetaData {
+public:
+  CryptoMetaData() {}
+
+  explicit CryptoMetaData(const GMPEncryptedBufferMetadata* aCrypto)
+  {
+    Init(aCrypto);
+  }
+
+  void Init(const GMPEncryptedBufferMetadata* aCrypto)
+  {
+    if (!aCrypto) {
+      assert(!IsValid());
+      return;
+    }
+    Assign(mKeyId, aCrypto->KeyId(), aCrypto->KeyIdSize());
+    Assign(mIV, aCrypto->IV(), aCrypto->IVSize());
+    Assign(mClearBytes, aCrypto->ClearBytes(), aCrypto->NumSubsamples());
+    Assign(mCipherBytes, aCrypto->CipherBytes(), aCrypto->NumSubsamples());
+  }
+
+  bool IsValid() const {
+    return !mKeyId.empty() &&
+           !mIV.empty() &&
+           !mCipherBytes.empty() &&
+           !mClearBytes.empty();
+  }
+
+  size_t NumSubsamples() const {
+    assert(mClearBytes.size() == mCipherBytes.size());
+    return mClearBytes.size();
+  }
+
+  std::vector<uint8_t> mKeyId;
+  std::vector<uint8_t> mIV;
+  std::vector<uint16_t> mClearBytes;
+  std::vector<uint32_t> mCipherBytes;
+};
+
 class ClearKeyDecryptionManager : public RefCounted
 {
 private:
@@ -45,8 +84,11 @@ public:
   void ExpectKeyId(KeyId aKeyId);
   void ReleaseKeyId(KeyId aKeyId);
 
+  // Decrypts buffer *in place*.
   GMPErr Decrypt(uint8_t* aBuffer, uint32_t aBufferSize,
-                 const GMPEncryptedBufferMetadata* aMetadata);
+                 const CryptoMetaData& aMetadata);
+  GMPErr Decrypt(std::vector<uint8_t>& aBuffer,
+                 const CryptoMetaData& aMetadata);
 
   void Shutdown();
 

media/gmp-clearkey/0.1/ClearKeySessionManager.cpp

@@ -400,7 +400,7 @@ ClearKeySessionManager::DoDecrypt(GMPBuffer* aBuffer,
   CK_LOGD("ClearKeySessionManager::DoDecrypt");
 
   GMPErr rv = mDecryptionManager->Decrypt(aBuffer->Data(), aBuffer->Size(),
-                                              aMetadata);
+                                          CryptoMetaData(aMetadata));
   CK_LOGD("DeDecrypt finished with code %x\n", rv);
   mCallback->Decrypted(aBuffer, rv);
 }

media/gmp-clearkey/0.1/ClearKeyUtils.h

@@ -103,4 +103,11 @@ private:
 
 GMPMutex* GMPCreateMutex();
 
+template<typename T>
+inline void
+Assign(std::vector<T>& aVec, const T* aData, size_t aLength)
+{
+  aVec.assign(aData, aData + aLength);
+}
+
 #endif // __ClearKeyUtils_h__

media/gmp-clearkey/0.1/VideoDecoder.cpp

@@ -115,30 +115,32 @@ VideoDecoder::Decode(GMPVideoEncodedFrame* aInputFrame,
   // Note: we don't need the codec specific info on a per-frame basis.
   // It's mostly useful for WebRTC use cases.
 
+  // Make a copy of the data, so we can release aInputFrame ASAP,
+  // to avoid too many shmem handles being held by the GMP process.
+  // If the GMP process holds on to too many shmem handles, the Gecko
+  // side can fail to allocate a shmem to send more input. This is
+  // particularly a problem in Gecko mochitests, which can open multiple
+  // actors at once which share the same pool of shmems.
+  DecodeData* data = new DecodeData();
+  Assign(data->mBuffer, aInputFrame->Buffer(), aInputFrame->Size());
+  data->mTimestamp = aInputFrame->TimeStamp();
+  data->mDuration = aInputFrame->Duration();
+  data->mIsKeyframe = (aInputFrame->FrameType() == kGMPKeyFrame);
+  const GMPEncryptedBufferMetadata* crypto = aInputFrame->GetDecryptionData();
+  if (crypto) {
+    data->mCrypto.Init(crypto);
+  }
+  aInputFrame->Destroy();
   mWorkerThread->Post(WrapTaskRefCounted(this,
                                          &VideoDecoder::DecodeTask,
-                                         aInputFrame));
+                                         data));
 }
 
-class AutoReleaseVideoFrame {
-public:
-  AutoReleaseVideoFrame(GMPVideoEncodedFrame* aFrame)
-    : mFrame(aFrame)
-  {
-  }
-  ~AutoReleaseVideoFrame()
-  {
-    GetPlatform()->runonmainthread(WrapTask(mFrame, &GMPVideoEncodedFrame::Destroy));
-  }
-private:
-  GMPVideoEncodedFrame* mFrame;
-};
-
 void
-VideoDecoder::DecodeTask(GMPVideoEncodedFrame* aInput)
+VideoDecoder::DecodeTask(DecodeData* aData)
 {
   CK_LOGD("VideoDecoder::DecodeTask");
-  AutoReleaseVideoFrame ensureFrameReleased(aInput);
+  AutoPtr<DecodeData> d(aData);
   HRESULT hr;
 
   {
@@ -152,24 +154,17 @@ VideoDecoder::DecodeTask(GMPVideoEncodedFrame* aInput)
     return;
   }
 
-  if (!aInput || !mHostAPI || !mDecoder) {
+  if (!aData || !mHostAPI || !mDecoder) {
     CK_LOGE("Decode job not set up correctly!");
     return;
   }
 
-  const uint8_t* inBuffer = aInput->Buffer();
-  if (!inBuffer) {
-    CK_LOGE("No buffer for encoded frame!\n");
-    return;
-  }
-
-  const GMPEncryptedBufferMetadata* crypto = aInput->GetDecryptionData();
-  std::vector<uint8_t> buffer(inBuffer, inBuffer + aInput->Size());
-  if (crypto) {
+  std::vector<uint8_t>& buffer = aData->mBuffer;
+  if (aData->mCrypto.IsValid()) {
     // Plugin host should have set up its decryptor/key sessions
     // before trying to decode!
     GMPErr rv =
-      ClearKeyDecryptionManager::Get()->Decrypt(&buffer[0], buffer.size(), crypto);
+      ClearKeyDecryptionManager::Get()->Decrypt(buffer, aData->mCrypto);
 
     if (GMP_FAILED(rv)) {
       MaybeRunOnMainThread(WrapTask(mCallback, &GMPVideoDecoderCallback::Error, rv));
@@ -179,7 +174,7 @@ VideoDecoder::DecodeTask(GMPVideoEncodedFrame* aInput)
 
   AnnexB::ConvertFrameInPlace(buffer);
 
-  if (aInput->FrameType() == kGMPKeyFrame) {
+  if (aData->mIsKeyframe) {
     // We must send the SPS and PPS to Windows Media Foundation's decoder.
     // Note: We do this *after* decryption, otherwise the subsample info
     // would be incorrect.
@@ -188,8 +183,8 @@ VideoDecoder::DecodeTask(GMPVideoEncodedFrame* aInput)
 
   hr = mDecoder->Input(buffer.data(),
                        buffer.size(),
-                       aInput->TimeStamp(),
-                       aInput->Duration());
+                       aData->mTimestamp,
+                       aData->mDuration);
 
   CK_LOGD("VideoDecoder::DecodeTask() Input ret hr=0x%x\n", hr);
   if (FAILED(hr)) {

media/gmp-clearkey/0.1/VideoDecoder.h

@@ -60,7 +60,20 @@ private:
 
   void DrainTask();
 
-  void DecodeTask(GMPVideoEncodedFrame* aInputFrame);
+  struct DecodeData {
+    DecodeData()
+      : mTimestamp(0)
+      , mDuration(0)
+      , mIsKeyframe(false)
+    {}
+    std::vector<uint8_t> mBuffer;
+    uint64_t mTimestamp;
+    uint64_t mDuration;
+    bool mIsKeyframe;
+    CryptoMetaData mCrypto;
+  };
+
+  void DecodeTask(DecodeData* aData);
 
   void ResetCompleteTask();