Bug 1030007 - Throttle updating the preview window when CPU low and/or encoder falls behind. r=mikeh, r=cpearce

content/media/omx/MediaOmxReader.cpp

@@ -66,15 +66,22 @@ nsresult MediaOmxReader::Init(MediaDecoderReader* aCloneDonor)
   return NS_OK;
 }
 
-void MediaOmxReader::Shutdown()
+void MediaOmxReader::ReleaseDecoder()
 {
-  ReleaseMediaResources();
   if (mOmxDecoder.get()) {
     mOmxDecoder->ReleaseDecoder();
   }
   mOmxDecoder.clear();
 }
 
+void MediaOmxReader::Shutdown()
+{
+  ReleaseMediaResources();
+  nsCOMPtr<nsIRunnable> event =
+    NS_NewRunnableMethod(this, &MediaOmxReader::ReleaseDecoder);
+  NS_DispatchToMainThread(event);
+}
+
 bool MediaOmxReader::IsWaitingMediaResources()
 {
   if (!mOmxDecoder.get()) {

content/media/omx/MediaOmxReader.h

@@ -104,6 +104,8 @@ public:
   // ANDROID_VERSION < 19
   void CheckAudioOffload();
 #endif
+
+  void ReleaseDecoder();
 };
 
 } // namespace mozilla

dom/camera/CameraControlImpl.cpp

@@ -248,6 +248,20 @@ CameraControlImpl::OnPreviewStateChange(CameraControlListener::PreviewState aNew
   }
 }
 
+void
+CameraControlImpl::OnRateLimitPreview(bool aLimit)
+{
+  // This function runs on neither the Main Thread nor the Camera Thread.
+  RwLockAutoEnterRead lock(mListenerLock);
+
+  DOM_CAMERA_LOGI("OnRateLimitPreview: %d\n", aLimit);
+
+  for (uint32_t i = 0; i < mListeners.Length(); ++i) {
+    CameraControlListener* l = mListeners[i];
+    l->OnRateLimitPreview(aLimit);
+  }
+}
+
 bool
 CameraControlImpl::OnNewPreviewFrame(layers::Image* aImage, uint32_t aWidth, uint32_t aHeight)
 {

dom/camera/CameraControlImpl.h

@@ -67,6 +67,7 @@ protected:
   void OnFacesDetected(const nsTArray<Face>& aFaces);
   void OnTakePictureComplete(uint8_t* aData, uint32_t aLength, const nsAString& aMimeType);
 
+  void OnRateLimitPreview(bool aLimit);
   bool OnNewPreviewFrame(layers::Image* aImage, uint32_t aWidth, uint32_t aHeight);
   void OnRecorderStateChange(CameraControlListener::RecorderState aState,
                              int32_t aStatus = -1, int32_t aTrackNumber = -1);

dom/camera/CameraControlListener.h

@@ -64,6 +64,7 @@ public:
   virtual void OnRecorderStateChange(RecorderState aState, int32_t aStatus, int32_t aTrackNum) { }
 
   virtual void OnShutter() { }
+  virtual void OnRateLimitPreview(bool aLimit) { }
   virtual bool OnNewPreviewFrame(layers::Image* aFrame, uint32_t aWidth, uint32_t aHeight)
   {
     return false;

dom/camera/CameraPreviewMediaStream.cpp

@@ -4,6 +4,15 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "CameraPreviewMediaStream.h"
+#include "CameraCommon.h"
+
+/**
+ * Maximum number of outstanding invalidates before we start to drop frames;
+ * if we hit this threshold, it is an indicator that the main thread is
+ * either very busy or the device is busy elsewhere (e.g. encoding or
+ * persisting video data).
+ */
+#define MAX_INVALIDATE_PENDING 4
 
 using namespace mozilla::layers;
 using namespace mozilla::dom;
@@ -13,7 +22,9 @@ namespace mozilla {
 CameraPreviewMediaStream::CameraPreviewMediaStream(DOMMediaStream* aWrapper)
   : MediaStream(aWrapper)
   , mMutex("mozilla::camera::CameraPreviewMediaStream")
-  , mFrameCallback(nullptr)
+  , mInvalidatePending(0)
+  , mDiscardedFrames(0)
+  , mRateLimit(false)
 {
   SetGraphImpl(MediaStreamGraph::GetInstance());
   mIsConsumed = false;
@@ -102,23 +113,54 @@ CameraPreviewMediaStream::Destroy()
   DestroyImpl();
 }
 
+void
+CameraPreviewMediaStream::Invalidate()
+{
+  MutexAutoLock lock(mMutex);
+  --mInvalidatePending;
+  for (nsTArray<nsRefPtr<VideoFrameContainer> >::size_type i = 0; i < mVideoOutputs.Length(); ++i) {
+    VideoFrameContainer* output = mVideoOutputs[i];
+    output->Invalidate();
+  }
+}
+
+void
+CameraPreviewMediaStream::RateLimit(bool aLimit)
+{
+  mRateLimit = aLimit;
+}
+
 void
 CameraPreviewMediaStream::SetCurrentFrame(const gfxIntSize& aIntrinsicSize, Image* aImage)
+{
   {
     MutexAutoLock lock(mMutex);
 
-  TimeStamp now = TimeStamp::Now();
+    if (mInvalidatePending > 0) {
-  for (uint32_t i = 0; i < mVideoOutputs.Length(); ++i) {
+      if (mRateLimit || mInvalidatePending > MAX_INVALIDATE_PENDING) {
-    VideoFrameContainer* output = mVideoOutputs[i];
+        ++mDiscardedFrames;
-    output->SetCurrentFrame(aIntrinsicSize, aImage, now);
+        DOM_CAMERA_LOGW("Discard preview frame %d, %d invalidation(s) pending",
-    nsCOMPtr<nsIRunnable> event =
+          mDiscardedFrames, mInvalidatePending);
-      NS_NewRunnableMethod(output, &VideoFrameContainer::Invalidate);
+        return;
-    NS_DispatchToMainThread(event);
       }
 
-  if (mFrameCallback) {
+      DOM_CAMERA_LOGI("Update preview frame, %d invalidation(s) pending",
-    mFrameCallback->OnNewFrame(aIntrinsicSize, aImage);
+        mInvalidatePending);
     }
+    mDiscardedFrames = 0;
+
+    TimeStamp now = TimeStamp::Now();
+    for (nsTArray<nsRefPtr<VideoFrameContainer> >::size_type i = 0; i < mVideoOutputs.Length(); ++i) {
+      VideoFrameContainer* output = mVideoOutputs[i];
+      output->SetCurrentFrame(aIntrinsicSize, aImage, now);
+    }
+
+    ++mInvalidatePending;
+  }
+
+  nsCOMPtr<nsIRunnable> event =
+    NS_NewRunnableMethod(this, &CameraPreviewMediaStream::Invalidate);
+  NS_DispatchToMainThread(event);
 }
 
 void
@@ -126,7 +168,7 @@ CameraPreviewMediaStream::ClearCurrentFrame()
 {
   MutexAutoLock lock(mMutex);
 
-  for (uint32_t i = 0; i < mVideoOutputs.Length(); ++i) {
+  for (nsTArray<nsRefPtr<VideoFrameContainer> >::size_type i = 0; i < mVideoOutputs.Length(); ++i) {
     VideoFrameContainer* output = mVideoOutputs[i];
     output->ClearCurrentFrame();
     nsCOMPtr<nsIRunnable> event =

dom/camera/CameraPreviewMediaStream.h

@@ -11,13 +11,8 @@
 
 namespace mozilla {
 
-class CameraPreviewFrameCallback {
-public:
-  virtual void OnNewFrame(const gfxIntSize& aIntrinsicSize, layers::Image* aImage) = 0;
-};
-
 /**
- * This is a stream for camere preview.
+ * This is a stream for camera preview.
  *
  * XXX It is a temporary fix of SourceMediaStream.
  * A camera preview requests no delay and no buffering stream.
@@ -40,20 +35,21 @@ public:
   virtual void RemoveListener(MediaStreamListener* aListener) MOZ_OVERRIDE;
   virtual void Destroy();
 
+  void Invalidate();
+
   // Call these on any thread.
   void SetCurrentFrame(const gfxIntSize& aIntrinsicSize, Image* aImage);
   void ClearCurrentFrame();
-
+  void RateLimit(bool aLimit);
-  void SetFrameCallback(CameraPreviewFrameCallback* aCallback) {
-    mFrameCallback = aCallback;
-  }
 
 protected:
   // mMutex protects all the class' fields.
   // This class is not registered to MediaStreamGraph.
   // It needs to protect all the fields.
   Mutex mMutex;
-  CameraPreviewFrameCallback* mFrameCallback;
+  int32_t mInvalidatePending;
+  uint32_t mDiscardedFrames;
+  bool mRateLimit;
 };
 
 }

dom/camera/DOMCameraControl.h

@@ -214,7 +214,7 @@ protected:
   DOMCameraControlListener* mListener;
 
   // our viewfinder stream
-  CameraPreviewMediaStream* mInput;
+  nsRefPtr<CameraPreviewMediaStream> mInput;
 
   // set once when this object is created
   nsCOMPtr<nsPIDOMWindow>   mWindow;

dom/camera/DOMCameraControlListener.cpp

@@ -287,6 +287,12 @@ DOMCameraControlListener::OnShutter()
   NS_DispatchToMainThread(new Callback(mDOMCameraControl));
 }
 
+void
+DOMCameraControlListener::OnRateLimitPreview(bool aLimit)
+{
+  mStream->RateLimit(aLimit);
+}
+
 bool
 DOMCameraControlListener::OnNewPreviewFrame(layers::Image* aImage, uint32_t aWidth, uint32_t aHeight)
 {

dom/camera/DOMCameraControlListener.h

@@ -28,6 +28,7 @@ public:
   virtual void OnRecorderStateChange(RecorderState aState, int32_t aStatus, int32_t aTrackNum) MOZ_OVERRIDE;
   virtual void OnConfigurationChange(const CameraListenerConfiguration& aConfiguration) MOZ_OVERRIDE;
   virtual void OnShutter() MOZ_OVERRIDE;
+  virtual void OnRateLimitPreview(bool aLimit) MOZ_OVERRIDE;
   virtual bool OnNewPreviewFrame(layers::Image* aImage, uint32_t aWidth, uint32_t aHeight) MOZ_OVERRIDE;
   virtual void OnUserError(UserContext aContext, nsresult aError) MOZ_OVERRIDE;
 

dom/camera/GonkCameraControl.cpp

@@ -1682,6 +1682,12 @@ nsGonkCameraControl::GetRecorderProfileManagerImpl()
   return profileMgr.forget();
 }
 
+void
+nsGonkCameraControl::OnRateLimitPreview(bool aLimit)
+{
+  CameraControlImpl::OnRateLimitPreview(aLimit);
+}
+
 void
 nsGonkCameraControl::OnNewPreviewFrame(layers::TextureClient* aBuffer)
 {
@@ -1744,6 +1750,12 @@ OnFacesDetected(nsGonkCameraControl* gc, camera_frame_metadata_t* aMetaData)
   gc->OnFacesDetected(aMetaData);
 }
 
+void
+OnRateLimitPreview(nsGonkCameraControl* gc, bool aLimit)
+{
+  gc->OnRateLimitPreview(aLimit);
+}
+
 void
 OnNewPreviewFrame(nsGonkCameraControl* gc, layers::TextureClient* aBuffer)
 {

dom/camera/GonkCameraControl.h

@@ -52,6 +52,7 @@ public:
   void OnFacesDetected(camera_frame_metadata_t* aMetaData);
   void OnTakePictureComplete(uint8_t* aData, uint32_t aLength);
   void OnTakePictureError();
+  void OnRateLimitPreview(bool aLimit);
   void OnNewPreviewFrame(layers::TextureClient* aBuffer);
   void OnRecorderEvent(int msg, int ext1, int ext2);
   void OnSystemError(CameraControlListener::SystemContext aWhere, nsresult aError);
@@ -84,6 +85,7 @@ public:
 protected:
   ~nsGonkCameraControl();
 
+  using CameraControlImpl::OnRateLimitPreview;
   using CameraControlImpl::OnNewPreviewFrame;
   using CameraControlImpl::OnAutoFocusComplete;
   using CameraControlImpl::OnFacesDetected;
@@ -178,6 +180,7 @@ private:
 };
 
 // camera driver callbacks
+void OnRateLimitPreview(nsGonkCameraControl* gc, bool aLimit);
 void OnTakePictureComplete(nsGonkCameraControl* gc, uint8_t* aData, uint32_t aLength);
 void OnTakePictureError(nsGonkCameraControl* gc);
 void OnAutoFocusComplete(nsGonkCameraControl* gc, bool aSuccess);

dom/camera/GonkCameraHwMgr.cpp

@@ -44,6 +44,12 @@ GonkCameraHardware::GonkCameraHardware(mozilla::nsGonkCameraControl* aTarget, ui
   DOM_CAMERA_LOGT("%s:%d : this=%p (aTarget=%p)\n", __func__, __LINE__, (void*)this, (void*)aTarget);
 }
 
+void
+GonkCameraHardware::OnRateLimitPreview(bool aLimit)
+{
+  ::OnRateLimitPreview(mTarget, aLimit);
+}
+
 void
 GonkCameraHardware::OnNewFrame()
 {

dom/camera/GonkCameraHwMgr.h

@@ -55,6 +55,8 @@ public:
   static sp<GonkCameraHardware> Connect(mozilla::nsGonkCameraControl* aTarget, uint32_t aCameraId);
   virtual void Close();
 
+  virtual void OnRateLimitPreview(bool aLimit);
+
   // derived from GonkNativeWindowNewFrameCallback
   virtual void OnNewFrame() MOZ_OVERRIDE;
 

dom/camera/GonkCameraSource.cpp

@@ -170,6 +170,7 @@ GonkCameraSource::GonkCameraSource(
       mStarted(false),
       mNumFramesEncoded(0),
       mTimeBetweenFrameCaptureUs(0),
+      mRateLimit(false),
       mFirstFrameTimeUs(0),
       mNumFramesDropped(0),
       mNumGlitches(0),
@@ -589,6 +590,10 @@ status_t GonkCameraSource::reset() {
         }
     }
     stopCameraRecording();
+    if (mRateLimit) {
+      mRateLimit = false;
+      mCameraHw->OnRateLimitPreview(false);
+    }
     releaseCamera();
 
     if (mCollectStats) {
@@ -692,7 +697,10 @@ status_t GonkCameraSource::read(
 
 void GonkCameraSource::dataCallbackTimestamp(int64_t timestampUs,
         int32_t msgType, const sp<IMemory> &data) {
+    bool rateLimit;
+    bool prevRateLimit;
     CS_LOGV("dataCallbackTimestamp: timestamp %lld us", timestampUs);
+    {
         Mutex::Autolock autoLock(mLock);
         if (!mStarted || (mNumFramesReceived == 0 && timestampUs < mStartTimeUs)) {
             CS_LOGV("Drop frame at %lld/%lld us", timestampUs, mStartTimeUs);
@@ -730,6 +738,12 @@ void GonkCameraSource::dataCallbackTimestamp(int64_t timestampUs,
         }
         ++mNumFramesReceived;
 
+        // If a backlog is building up in the receive queue, we are likely
+        // resource constrained and we need to throttle
+        prevRateLimit = mRateLimit;
+        rateLimit = mFramesReceived.empty();
+        mRateLimit = rateLimit;
+
         CHECK(data != NULL && data->size() > 0);
         mFramesReceived.push_back(data);
         int64_t timeUs = mStartTimeUs + (timestampUs - mFirstFrameTimeUs);
@@ -739,6 +753,11 @@ void GonkCameraSource::dataCallbackTimestamp(int64_t timestampUs,
         mFrameAvailableCondition.signal();
     }
 
+    if(prevRateLimit != rateLimit) {
+        mCameraHw->OnRateLimitPreview(rateLimit);
+    }
+}
+
 bool GonkCameraSource::isMetaDataStoredInVideoBuffers() const {
     CS_LOGV("isMetaDataStoredInVideoBuffers");
     return mIsMetaDataStoredInVideoBuffers;

dom/camera/GonkCameraSource.h

@@ -127,6 +127,7 @@ private:
     List<sp<IMemory> > mFramesReceived;
     List<sp<IMemory> > mFramesBeingEncoded;
     List<int64_t> mFrameTimes;
+    bool mRateLimit;
 
     int64_t mFirstFrameTimeUs;
     int32_t mNumFramesDropped;