Bug 1032848 - Part 2: Implement HTMLCanvasElement::CaptureStream. r=mt, r=jesup, r=jgilbert, r=gwright

dom/canvas/WebGLContext.h

@@ -310,6 +310,8 @@ public:
 
     bool IsPremultAlpha() const { return mOptions.premultipliedAlpha; }
 
+    bool IsPreservingDrawingBuffer() const { return mOptions.preserveDrawingBuffer; }
+
     bool PresentScreenBuffer();
 
     // a number that increments every time we have an event that causes

dom/html/HTMLCanvasElement.cpp

@@ -410,8 +410,47 @@ already_AddRefed<CanvasCaptureMediaStream>
 HTMLCanvasElement::CaptureStream(const Optional<double>& aFrameRate,
                                  ErrorResult& aRv)
 {
-  aRv.Throw(NS_ERROR_NOT_IMPLEMENTED);
-  return nullptr;
+  if (IsWriteOnly()) {
+    aRv.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return nullptr;
+  }
+
+  nsIDOMWindow* window = OwnerDoc()->GetInnerWindow();
+  if (!window) {
+    aRv.Throw(NS_ERROR_FAILURE);
+    return nullptr;
+  }
+
+  if (!mCurrentContext) {
+    aRv.Throw(NS_ERROR_NOT_INITIALIZED);
+    return nullptr;
+  }
+
+  if (mCurrentContextType != CanvasContextType::Canvas2D) {
+    WebGLContext* gl = static_cast<WebGLContext*>(mCurrentContext.get());
+    if (!gl->IsPreservingDrawingBuffer()) {
+      aRv.Throw(NS_ERROR_FAILURE);
+      return nullptr;
+    }
+  }
+
+  nsRefPtr<CanvasCaptureMediaStream> stream =
+    CanvasCaptureMediaStream::CreateSourceStream(window, this);
+  if (!stream) {
+    aRv.Throw(NS_ERROR_FAILURE);
+    return nullptr;
+  }
+
+  nsRefPtr<nsIPrincipal> principal = NodePrincipal();
+  stream->CombineWithPrincipal(principal);
+
+  TrackID videoTrackId = 1;
+  nsresult rv = stream->Init(aFrameRate, videoTrackId);
+  if (NS_FAILED(rv)) {
+    aRv.Throw(rv);
+    return nullptr;
+  }
+  return stream.forget();
 }
 
 nsresult

dom/media/CanvasCaptureMediaStream.cpp

@@ -5,6 +5,7 @@
 
 #include "CanvasCaptureMediaStream.h"
 #include "DOMMediaStream.h"
+#include "gfxPlatform.h"
 #include "ImageContainer.h"
 #include "MediaStreamGraph.h"
 #include "mozilla/Mutex.h"
@@ -21,6 +22,257 @@ using namespace mozilla::gfx;
 namespace mozilla {
 namespace dom {
 
+class OutputStreamDriver::StreamListener : public MediaStreamListener
+{
+public:
+  explicit StreamListener(OutputStreamDriver* aDriver,
+                          SourceMediaStream* aSourceStream)
+    : mSourceStream(aSourceStream)
+    , mMutex("CanvasCaptureMediaStream::OSD::StreamListener")
+    , mDriver(aDriver)
+  {
+    MOZ_ASSERT(mDriver);
+    MOZ_ASSERT(mSourceStream);
+  }
+
+  void Forget() {
+    MOZ_ASSERT(NS_IsMainThread());
+
+    MutexAutoLock lock(mMutex);
+    mDriver = nullptr;
+  }
+
+  virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) override
+  {
+    // Called on the MediaStreamGraph thread.
+
+    MutexAutoLock lock(mMutex);
+    if (mDriver) {
+      mDriver->NotifyPull(aDesiredTime);
+    } else {
+      // The DOM stream is dead, let's end it
+      mSourceStream->EndAllTrackAndFinish();
+    }
+  }
+
+protected:
+  ~StreamListener() { }
+
+private:
+  nsRefPtr<SourceMediaStream> mSourceStream;
+
+  // The below members are protected by mMutex.
+  Mutex mMutex;
+  // This is a raw pointer to avoid a reference cycle with OutputStreamDriver.
+  // Accessed on main and MediaStreamGraph threads, set on main thread.
+  OutputStreamDriver* mDriver;
+};
+
+OutputStreamDriver::OutputStreamDriver(CanvasCaptureMediaStream* aDOMStream,
+                                       const TrackID& aTrackId)
+  : mDOMStream(aDOMStream)
+  , mSourceStream(nullptr)
+  , mStarted(false)
+  , mStreamListener(nullptr)
+  , mTrackId(aTrackId)
+  , mMutex("CanvasCaptureMediaStream::OutputStreamDriver")
+  , mImage(nullptr)
+{
+  MOZ_ASSERT(mDOMStream);
+}
+
+OutputStreamDriver::~OutputStreamDriver()
+{
+  if (mStreamListener) {
+    // MediaStreamGraph will keep the listener alive until it can finish the
+    // stream on the next NotifyPull().
+    mStreamListener->Forget();
+  }
+}
+
+nsresult
+OutputStreamDriver::Start()
+{
+  if (mStarted) {
+    return NS_ERROR_ALREADY_INITIALIZED;
+  }
+
+  MOZ_ASSERT(mDOMStream);
+
+  mDOMStream->CreateDOMTrack(mTrackId, MediaSegment::VIDEO);
+
+  mSourceStream = mDOMStream->GetStream()->AsSourceStream();
+  MOZ_ASSERT(mSourceStream);
+
+  mStreamListener = new StreamListener(this, mSourceStream);
+  mSourceStream->AddListener(mStreamListener);
+  mSourceStream->AddTrack(mTrackId, 0, new VideoSegment());
+  mSourceStream->AdvanceKnownTracksTime(STREAM_TIME_MAX);
+  mSourceStream->SetPullEnabled(true);
+
+  // Run StartInternal() in stable state to allow it to directly capture a frame
+  nsCOMPtr<nsIRunnable> runnable =
+    NS_NewRunnableMethod(this, &OutputStreamDriver::StartInternal);
+  nsCOMPtr<nsIAppShell> appShell = do_GetService(kAppShellCID);
+  appShell->RunInStableState(runnable);
+
+  mStarted = true;
+  return NS_OK;
+}
+
+void
+OutputStreamDriver::ForgetDOMStream()
+{
+  if (mStreamListener) {
+    mStreamListener->Forget();
+  }
+  mDOMStream = nullptr;
+}
+
+void
+OutputStreamDriver::AppendToTrack(StreamTime aDuration)
+{
+  MOZ_ASSERT(mSourceStream);
+
+  MutexAutoLock lock(mMutex);
+
+  nsRefPtr<Image> image = mImage;
+  IntSize size = image ? image->GetSize() : IntSize(0, 0);
+  VideoSegment segment;
+  segment.AppendFrame(image.forget(), aDuration, size);
+
+  mSourceStream->AppendToTrack(mTrackId, &segment);
+}
+
+void
+OutputStreamDriver::NotifyPull(StreamTime aDesiredTime)
+{
+  StreamTime delta = aDesiredTime - mSourceStream->GetEndOfAppendedData(mTrackId);
+  if (delta > 0) {
+    // nullptr images are allowed
+    AppendToTrack(delta);
+  }
+}
+
+void
+OutputStreamDriver::SetImage(Image* aImage)
+{
+  MutexAutoLock lock(mMutex);
+  mImage = aImage;
+}
+
+// ----------------------------------------------------------------------
+
+class TimerDriver : public OutputStreamDriver
+                  , public nsITimerCallback
+{
+public:
+  explicit TimerDriver(CanvasCaptureMediaStream* aDOMStream,
+                       const double& aFPS,
+                       const TrackID& aTrackId)
+    : OutputStreamDriver(aDOMStream, aTrackId)
+    , mFPS(aFPS)
+    , mTimer(nullptr)
+  {
+  }
+
+  void ForgetDOMStream() override
+  {
+    if (mTimer) {
+      mTimer->Cancel();
+      mTimer = nullptr;
+    }
+    OutputStreamDriver::ForgetDOMStream();
+  }
+
+  nsresult
+  TakeSnapshot()
+  {
+    // mDOMStream can't be killed while we're on main thread
+    MOZ_ASSERT(NS_IsMainThread());
+    MOZ_ASSERT(DOMStream());
+
+    if (!DOMStream()->Canvas()) {
+      // DOMStream's canvas pointer was garbage collected. We can abort now.
+      return NS_ERROR_NOT_AVAILABLE;
+    }
+    MOZ_ASSERT(DOMStream()->Canvas());
+
+    if (DOMStream()->Canvas()->IsWriteOnly()) {
+      return NS_ERROR_DOM_SECURITY_ERR;
+    }
+
+    // Pass `nullptr` to force alpha-premult.
+    RefPtr<SourceSurface> snapshot = DOMStream()->Canvas()->GetSurfaceSnapshot(nullptr);
+    if (!snapshot) {
+      return NS_ERROR_FAILURE;
+    }
+
+    RefPtr<SourceSurface> opt = gfxPlatform::GetPlatform()
+      ->ScreenReferenceDrawTarget()->OptimizeSourceSurface(snapshot);
+    if (!opt) {
+      return NS_ERROR_FAILURE;
+    }
+
+    CairoImage::Data imageData;
+    imageData.mSize = opt->GetSize();
+    imageData.mSourceSurface = opt;
+
+    RefPtr<CairoImage> image = new layers::CairoImage();
+    image->SetData(imageData);
+
+    SetImage(image);
+    return NS_OK;
+  }
+
+  NS_IMETHODIMP
+  Notify(nsITimer* aTimer)
+  {
+    nsresult rv = TakeSnapshot();
+    if (NS_FAILED(rv)) {
+      aTimer->Cancel();
+    }
+    return rv;
+  }
+
+  virtual void RequestFrame() override
+  {
+    TakeSnapshot();
+  }
+
+  NS_DECL_ISUPPORTS_INHERITED
+
+protected:
+  virtual ~TimerDriver() {}
+
+  virtual void StartInternal() override
+  {
+    // Always capture at least one frame.
+    DebugOnly<nsresult> rv = TakeSnapshot();
+    MOZ_ASSERT(NS_SUCCEEDED(rv));
+
+    if (mFPS == 0.0) {
+      return;
+    }
+
+    mTimer = do_CreateInstance(NS_TIMER_CONTRACTID);
+    if (!mTimer) {
+      return;
+    }
+    mTimer->InitWithCallback(this, int(1000 / mFPS), nsITimer::TYPE_REPEATING_SLACK);
+  }
+
+private:
+  const double mFPS;
+  nsCOMPtr<nsITimer> mTimer;
+};
+
+NS_IMPL_ADDREF_INHERITED(TimerDriver, OutputStreamDriver)
+NS_IMPL_RELEASE_INHERITED(TimerDriver, OutputStreamDriver)
+NS_IMPL_QUERY_INTERFACE(TimerDriver, nsITimerCallback)
+
+// ----------------------------------------------------------------------
+
 NS_IMPL_CYCLE_COLLECTION_INHERITED(CanvasCaptureMediaStream, DOMMediaStream,
                                    mCanvas)
 
@@ -32,11 +284,15 @@ NS_INTERFACE_MAP_END_INHERITING(DOMMediaStream)
 
 CanvasCaptureMediaStream::CanvasCaptureMediaStream(HTMLCanvasElement* aCanvas)
   : mCanvas(aCanvas)
+  , mOutputStreamDriver(nullptr)
 {
 }
 
 CanvasCaptureMediaStream::~CanvasCaptureMediaStream()
 {
+  if (mOutputStreamDriver) {
+    mOutputStreamDriver->ForgetDOMStream();
+  }
 }
 
 JSObject*
@@ -48,6 +304,9 @@ CanvasCaptureMediaStream::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGive
 void
 CanvasCaptureMediaStream::RequestFrame()
 {
+  if (mOutputStreamDriver) {
+    mOutputStreamDriver->RequestFrame();
+  }
 }
 
 nsresult
@@ -55,11 +314,17 @@ CanvasCaptureMediaStream::Init(const dom::Optional<double>& aFPS,
                                const TrackID& aTrackId)
 {
   if (!aFPS.WasPassed()) {
-    return NS_ERROR_NOT_IMPLEMENTED;
+    // TODO (Bug 1152298): Implement a real AutoDriver.
+    // We use a 30FPS TimerDriver for now.
+    mOutputStreamDriver = new TimerDriver(this, 30.0, aTrackId);
   } else if (aFPS.Value() < 0) {
     return NS_ERROR_ILLEGAL_VALUE;
+  } else {
+    // Cap frame rate to 60 FPS for sanity
+    double fps = std::min(60.0, aFPS.Value());
+    mOutputStreamDriver = new TimerDriver(this, fps, aTrackId);
   }
-  return NS_ERROR_NOT_IMPLEMENTED;
+  return mOutputStreamDriver->Start();
 }
 
 already_AddRefed<CanvasCaptureMediaStream>

dom/media/CanvasCaptureMediaStream.h

@@ -8,10 +8,69 @@
 
 namespace mozilla {
 class DOMMediaStream;
+class MediaStreamListener;
+class SourceMediaStream;
+
+namespace layers {
+class Image;
+}
 
 namespace dom {
+class CanvasCaptureMediaStream;
 class HTMLCanvasElement;
 
+class OutputStreamDriver
+{
+public:
+  OutputStreamDriver(CanvasCaptureMediaStream* aDOMStream,
+                     const TrackID& aTrackId);
+
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(OutputStreamDriver);
+
+  nsresult Start();
+
+  virtual void ForgetDOMStream();
+
+  virtual void RequestFrame() { }
+
+  CanvasCaptureMediaStream* DOMStream() const { return mDOMStream; }
+
+protected:
+  virtual ~OutputStreamDriver();
+  class StreamListener;
+
+  /*
+   * Appends mImage to video track for the desired duration.
+   */
+  void AppendToTrack(StreamTime aDuration);
+  void NotifyPull(StreamTime aDesiredTime);
+
+  /*
+   * Sub classes can SetImage() to update the image being appended to the
+   * output stream. It will be appended on the next NotifyPull from MSG.
+   */
+  void SetImage(layers::Image* aImage);
+
+  /*
+   * Called in main thread stable state to initialize sub classes.
+   */
+  virtual void StartInternal() = 0;
+
+private:
+  // This is a raw pointer to avoid a reference cycle between OutputStreamDriver
+  // and CanvasCaptureMediaStream. ForgetDOMStream() will be called by
+  // ~CanvasCaptureMediaStream() to make sure we don't do anything illegal.
+  CanvasCaptureMediaStream* mDOMStream;
+  nsRefPtr<SourceMediaStream> mSourceStream;
+  bool mStarted;
+  nsRefPtr<StreamListener> mStreamListener;
+  const TrackID mTrackId;
+
+  // The below members are protected by mMutex.
+  Mutex mMutex;
+  nsRefPtr<layers::Image> mImage;
+};
+
 class CanvasCaptureMediaStream: public DOMMediaStream
 {
 public:
@@ -40,6 +99,7 @@ protected:
 
 private:
   nsRefPtr<HTMLCanvasElement> mCanvas;
+  nsRefPtr<OutputStreamDriver> mOutputStreamDriver;
 };
 
 } // namespace dom