Bug 466699 - Fix out of sync video and audio - r+sr=roc

--HG--
extra : rebase_source : ee8eedf72add30627fdbeef7cf22bc70a9415362

content/media/video/public/nsAudioStream.h

@@ -89,6 +89,16 @@ class nsAudioStream
   // Block until buffered audio data has been consumed.
   void Drain();
 
+  // Pause audio playback
+  void Pause();
+
+  // Resume audio playback
+  void Resume();
+
+  // Return the position in seconds of the sample being played by the
+  // audio hardware.
+  float GetPosition();
+
  private:
   double mVolume;
   void* mAudioHandle;

content/media/video/src/nsAudioStream.cpp

@@ -190,7 +190,9 @@ PRInt32 nsAudioStream::Available()
     return FAKE_BUFFER_SIZE;
 
   size_t s = 0; 
-  sa_stream_get_write_size(static_cast<sa_stream_t*>(mAudioHandle), &s);
+  if (sa_stream_get_write_size(static_cast<sa_stream_t*>(mAudioHandle), &s) != SA_SUCCESS)
+    return 0;
+
   return s / sizeof(short);
 }
 
@@ -218,3 +220,35 @@ void nsAudioStream::Drain()
         Shutdown();
   }
 }
+
+void nsAudioStream::Pause()
+{
+  if (!mAudioHandle)
+    return;
+
+  sa_stream_pause(static_cast<sa_stream_t*>(mAudioHandle));
+}
+
+void nsAudioStream::Resume()
+{
+  if (!mAudioHandle)
+    return;
+
+  sa_stream_resume(static_cast<sa_stream_t*>(mAudioHandle));
+}
+
+float nsAudioStream::GetPosition()
+{
+  if (!mAudioHandle)
+    return -1.0;
+
+  PRInt64 position = 0;
+  if (sa_stream_get_position(static_cast<sa_stream_t*>(mAudioHandle),
+                             SA_POSITION_WRITE_SOFTWARE,
+                             &position) == SA_SUCCESS) {
+    return (position / float(mRate) / mChannels / sizeof(short));
+  }
+
+  return -1.0;
+}
+

content/media/video/src/nsOggDecoder.cpp

@@ -157,11 +157,8 @@ public:
     // Write the audio data from the frame to the Audio stream.
     void Write(nsAudioStream* aStream)
     {
-      PRUint32 length = mAudioData.Length();
-      if (length == 0)
-        return;
-
-      aStream->Write(mAudioData.Elements(), length);
+      aStream->Write(mAudioData.Elements(), mAudioData.Length());
+      mAudioData.Clear(); 
     }
 
     void SetVideoHeader(OggPlayDataHeader* aVideoHeader)
@@ -230,6 +227,20 @@ public:
       return !mEmpty && mHead == mTail;
     }
 
+    float ResetTimes(float aPeriod)
+    {
+      float time = 0.0;
+      if (!mEmpty) {
+        PRInt32 current = mHead;
+        do {
+          mQueue[current]->mTime = time;
+          time += aPeriod;
+          current = (current + 1) % OGGPLAY_BUFFER_SIZE;
+        } while (current != mTail);
+      }
+      return time;
+    }
+
   private:
     FrameData* mQueue[OGGPLAY_BUFFER_SIZE];
     PRInt32 mHead;
@@ -292,8 +303,11 @@ public:
   // must be locked when calling this method.
   void PlayVideo(FrameData* aFrame);
 
-  // Play the audio data from the given frame. The decode monitor must
-  // be locked when calling this method.
+  // Plays the audio for the frame, plus any outstanding audio data
+  // buffered by nsAudioStream and not yet written to the
+  // hardware. The audio data for the frame is cleared out so
+  // subsequent calls with the same frame do not re-write the data.
+  // The decode monitor must be locked when calling this method.
   void PlayAudio(FrameData* aFrame);
 
   // Called from the main thread to get the current frame time. The decoder
@@ -365,11 +379,22 @@ protected:
   void StopAudio();
 
   // Start playback of media. Must be called with the decode monitor held.
+  // This opens or re-opens the audio stream for playback to start.
   void StartPlayback();
 
   // Stop playback of media. Must be called with the decode monitor held.
+  // This actually closes the audio stream and releases any OS resources.
   void StopPlayback();
 
+  // Pause playback of media. Must be called with the decode monitor held.
+  // This does not close the OS based audio stream - it suspends it to be
+  // resumed later.
+  void PausePlayback();
+
+  // Resume playback of media. Must be called with the decode monitor held.
+  // This resumes a paused audio stream.
+  void ResumePlayback();
+
   // Update the playback position. This can result in a timeupdate event
   // and an invalidate of the frame being dispatched asynchronously if
   // there is no such event currently queued.
@@ -377,6 +402,11 @@ protected:
   // the decode monitor held.
   void UpdatePlaybackPosition(float aTime);
 
+  // Takes decoded frames from liboggplay's internal buffer and
+  // places them in our frame queue. Must be called with the decode
+  // monitor held.
+  void QueueDecodedFrames();
+
 private:
   // *****
   // The follow fields are only accessed by the decoder thread
@@ -753,7 +783,7 @@ void nsOggDecodeStateMachine::PlayFrame() {
 
   if (mDecoder->GetState() == nsOggDecoder::PLAY_STATE_PLAYING) {
     if (!mPlaying) {
-      StartPlayback();
+      ResumePlayback();
     }
 
     if (!mDecodedFrames.IsEmpty()) {
@@ -769,7 +799,14 @@ void nsOggDecodeStateMachine::PlayFrame() {
 
       double time;
       for (;;) {
-        time = (TimeStamp::Now() - mPlayStartTime - mPauseDuration).ToSeconds();
+        // Even if the frame has had its audio data written we call
+        // PlayAudio to ensure that any data we have buffered in the
+        // nsAudioStream is written to the hardware.
+        PlayAudio(frame);
+        double hwtime = mAudioStream ? mAudioStream->GetPosition() : -1.0;
+        time = hwtime < 0.0 ?
+          (TimeStamp::Now() - mPlayStartTime - mPauseDuration).ToSeconds() :
+          hwtime;
         if (time < frame->mTime) {
           mon.Wait(PR_MillisecondsToInterval(PRInt64((frame->mTime - time)*1000)));
           if (mState == DECODER_STATE_SHUTDOWN)
@@ -780,24 +817,33 @@ void nsOggDecodeStateMachine::PlayFrame() {
       }
 
       mDecodedFrames.Pop();
+      QueueDecodedFrames();
 
       // Skip frames up to the one we should be showing.
       while (!mDecodedFrames.IsEmpty() && time >= mDecodedFrames.Peek()->mTime) {
         LOG(PR_LOG_DEBUG, ("Skipping frame time %f with audio at time %f", mDecodedFrames.Peek()->mTime, time));
+        PlayAudio(frame);
         delete frame;
         frame = mDecodedFrames.Peek();
         mDecodedFrames.Pop();
       }
-      PlayAudio(frame);
-      PlayVideo(frame);
-      mDecoder->mPlaybackPosition = frame->mEndStreamPosition;
-      UpdatePlaybackPosition(frame->mDecodedFrameTime);
-      delete frame;
+      if (time < frame->mTime + mCallbackPeriod) {
+        PlayAudio(frame);
+        PlayVideo(frame);
+        mDecoder->mPlaybackPosition = frame->mEndStreamPosition;
+        UpdatePlaybackPosition(frame->mDecodedFrameTime);
+        delete frame;
+      }
+      else {
+        PlayAudio(frame);
+        delete frame;
+        frame = 0;
+      }
     }
   }
   else {
     if (mPlaying) {
-      StopPlayback();
+      PausePlayback();
     }
 
     if (mState == DECODER_STATE_DECODING) {
@@ -904,16 +950,52 @@ void nsOggDecodeStateMachine::StartPlayback()
     // Null out mPauseStartTime
     mPauseStartTime = TimeStamp();
   }
+  mPlayStartTime = TimeStamp::Now();
+  mPauseDuration = 0;
+
 }
 
 void nsOggDecodeStateMachine::StopPlayback()
 {
   //  NS_ASSERTION(PR_InMonitor(mDecoder->GetMonitor()), "StopPlayback() called without acquiring decoder monitor");
+  mLastFrameTime = mDecodedFrames.ResetTimes(mCallbackPeriod);
   StopAudio();
   mPlaying = PR_FALSE;
   mPauseStartTime = TimeStamp::Now();
 }
 
+void nsOggDecodeStateMachine::PausePlayback()
+{
+  if (!mAudioStream) {
+    StopPlayback();
+    return;
+  }
+
+  mAudioStream->Pause();
+  mPlaying = PR_FALSE;
+  mPauseStartTime = TimeStamp::Now();
+}
+
+void nsOggDecodeStateMachine::ResumePlayback()
+{
+ if (!mAudioStream) {
+    StartPlayback();
+    return;
+ }
+ 
+ mAudioStream->Resume();
+ mPlaying = PR_TRUE;
+
+ // Compute duration spent paused
+ if (!mPauseStartTime.IsNull()) {
+   mPauseDuration += TimeStamp::Now() - mPauseStartTime;
+   // Null out mPauseStartTime
+   mPauseStartTime = TimeStamp();
+ }
+ mPlayStartTime = TimeStamp::Now();
+ mPauseDuration = 0;
+}
+
 void nsOggDecodeStateMachine::UpdatePlaybackPosition(float aTime)
 {
   //  NS_ASSERTION(PR_InMonitor(mDecoder->GetMonitor()), "UpdatePlaybackPosition() called without acquiring decoder monitor");
@@ -926,6 +1008,15 @@ void nsOggDecodeStateMachine::UpdatePlaybackPosition(float aTime)
   }
 }
 
+void nsOggDecodeStateMachine::QueueDecodedFrames()
+{
+  //  NS_ASSERTION(PR_InMonitor(mDecoder->GetMonitor()), "QueueDecodedFrames() called without acquiring decoder monitor");
+  FrameData* frame;
+  while (!mDecodedFrames.IsFull() && (frame = NextFrame())) {
+    mDecodedFrames.Push(frame);
+  }
+}
+
 void nsOggDecodeStateMachine::ClearPositionChangeFlag()
 {
   //  NS_ASSERTION(PR_InMonitor(mDecoder->GetMonitor()), "ClearPositionChangeFlag() called without acquiring decoder monitor");
@@ -1096,10 +1187,7 @@ nsresult nsOggDecodeStateMachine::Run()
           mon.Wait(PR_MillisecondsToInterval(PRInt64(mCallbackPeriod*500)));
           if (mState != DECODER_STATE_DECODING)
             break;
-          FrameData* frame = NextFrame();
-          if (frame) {
-            mDecodedFrames.Push(frame);
-          }
+          QueueDecodedFrames();
         }
 
         if (mState != DECODER_STATE_DECODING)
@@ -1108,11 +1196,11 @@ nsresult nsOggDecodeStateMachine::Run()
         if (mDecodingCompleted) {
           LOG(PR_LOG_DEBUG, ("Changed state from DECODING to COMPLETED"));
           mState = DECODER_STATE_COMPLETED;
-          mStepDecodeThread->Shutdown();
-          mStepDecodeThread = nsnull;
           mDecodingCompleted = PR_FALSE;
           mBufferExhausted = PR_FALSE;
           mon.NotifyAll();
+          mStepDecodeThread->Shutdown();
+          mStepDecodeThread = nsnull;
           continue;
         }
 
@@ -1130,7 +1218,7 @@ nsresult nsOggDecodeStateMachine::Run()
           // more data to load. Let's buffer to make sure we can play a
           // decent amount of video in the future.
           if (mPlaying) {
-            StopPlayback();
+            PausePlayback();
           }
 
           // We need to tell the element that buffering has started.
@@ -1153,7 +1241,7 @@ nsresult nsOggDecodeStateMachine::Run()
               BUFFERING_RATE(playbackRate) * BUFFERING_WAIT;
           mState = DECODER_STATE_BUFFERING;
           if (mPlaying) {
-            StopPlayback();
+            PausePlayback();
           }
           LOG(PR_LOG_DEBUG, ("Changed state from DECODING to BUFFERING"));
         } else {
@@ -1284,7 +1372,7 @@ nsresult nsOggDecodeStateMachine::Run()
           NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
           if (mDecoder->GetState() == nsOggDecoder::PLAY_STATE_PLAYING) {
             if (!mPlaying) {
-              StartPlayback();
+              ResumePlayback();
             }
           }
         }
@@ -1296,23 +1384,18 @@ nsresult nsOggDecodeStateMachine::Run()
       {
         // Get all the remaining decoded frames in the liboggplay buffer and
         // place them in the frame queue.
-        FrameData* frame;
-        do {
-          frame = NextFrame();
-          if (frame) {
-            mDecodedFrames.Push(frame);
-          }
-        } while (frame);
+        QueueDecodedFrames();
 
         // Play the remaining frames in the frame queue
         while (mState == DECODER_STATE_COMPLETED &&
                !mDecodedFrames.IsEmpty()) {
           PlayFrame();
-          if (mState != DECODER_STATE_SHUTDOWN) {
+          if (mState == DECODER_STATE_COMPLETED) {
             // Wait for the time of one frame so we don't tight loop
             // and we need to release the monitor so timeupdate and
             // invalidate's on the main thread can occur.
             mon.Wait(PR_MillisecondsToInterval(PRInt64(mCallbackPeriod*1000)));
+            QueueDecodedFrames();
           }
         }