gecko/content/media/mediasource/MediaSourceDecoder.cpp
Chris Double f432d6c60c Bug 1030539 - Seeking hangs due to time truncation in Media Source Extensions - r=kinetik
--HG--
extra : rebase_source : 169eb8b240894ec82606b0a15d74866ca5b52118
2014-06-26 14:40:55 +12:00

568 lines
17 KiB
C++

/* -*- mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MediaSourceResource.h"
#include "MediaSourceDecoder.h"
#include "AbstractMediaDecoder.h"
#include "MediaDecoderReader.h"
#include "MediaDecoderStateMachine.h"
#include "mozilla/Assertions.h"
#include "mozilla/FloatingPoint.h"
#include "mozilla/dom/HTMLMediaElement.h"
#include "mozilla/dom/TimeRanges.h"
#include "mozilla/mozalloc.h"
#include "nsAutoPtr.h"
#include "nsISupports.h"
#include "nsIThread.h"
#include "prlog.h"
#include "MediaSource.h"
#include "SubBufferDecoder.h"
#include "SourceBufferResource.h"
#include "SourceBufferList.h"
#include "VideoUtils.h"
#ifdef PR_LOGGING
extern PRLogModuleInfo* gMediaSourceLog;
#define MSE_DEBUG(...) PR_LOG(gMediaSourceLog, PR_LOG_DEBUG, (__VA_ARGS__))
#else
#define MSE_DEBUG(...)
#endif
namespace mozilla {
namespace dom {
class TimeRanges;
} // namespace dom
class MediaSourceReader : public MediaDecoderReader
{
public:
MediaSourceReader(MediaSourceDecoder* aDecoder, dom::MediaSource* aSource)
: MediaDecoderReader(aDecoder)
, mTimeThreshold(-1)
, mDropVideoBeforeThreshold(false)
, mActiveVideoDecoder(-1)
, mActiveAudioDecoder(-1)
, mMediaSource(aSource)
{
}
nsresult Init(MediaDecoderReader* aCloneDonor) MOZ_OVERRIDE
{
// Although we technically don't implement anything here, we return NS_OK
// so that when the state machine initializes and calls this function
// we don't return an error code back to the media element.
return NS_OK;
}
bool IsWaitingMediaResources() MOZ_OVERRIDE
{
return mDecoders.IsEmpty() && mPendingDecoders.IsEmpty();
}
void RequestAudioData() MOZ_OVERRIDE
{
if (!GetAudioReader()) {
MSE_DEBUG("%p DecodeAudioFrame called with no audio reader", this);
MOZ_ASSERT(mPendingDecoders.IsEmpty());
GetCallback()->OnDecodeError();
return;
}
GetAudioReader()->RequestAudioData();
}
void OnAudioDecoded(AudioData* aSample)
{
GetCallback()->OnAudioDecoded(aSample);
}
void OnAudioEOS()
{
GetCallback()->OnAudioEOS();
}
void RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) MOZ_OVERRIDE
{
if (!GetVideoReader()) {
MSE_DEBUG("%p DecodeVideoFrame called with no video reader", this);
MOZ_ASSERT(mPendingDecoders.IsEmpty());
GetCallback()->OnDecodeError();
return;
}
mTimeThreshold = aTimeThreshold;
GetVideoReader()->RequestVideoData(aSkipToNextKeyframe, aTimeThreshold);
}
void OnVideoDecoded(VideoData* aSample)
{
if (mDropVideoBeforeThreshold) {
if (aSample->mTime < mTimeThreshold) {
delete aSample;
GetVideoReader()->RequestVideoData(false, mTimeThreshold);
} else {
mDropVideoBeforeThreshold = false;
GetCallback()->OnVideoDecoded(aSample);
}
} else {
GetCallback()->OnVideoDecoded(aSample);
}
}
void OnVideoEOS()
{
// End of stream. See if we can switch to another video decoder.
MSE_DEBUG("%p MSR::DecodeVF %d (%p) returned false (readers=%u)",
this, mActiveVideoDecoder, mDecoders[mActiveVideoDecoder].get(), mDecoders.Length());
if (MaybeSwitchVideoReaders()) {
// Success! Resume decoding with next video decoder.
RequestVideoData(false, mTimeThreshold);
} else {
// End of stream.
MSE_DEBUG("%p MSR::DecodeVF %d (%p) EOS (readers=%u)",
this, mActiveVideoDecoder, mDecoders[mActiveVideoDecoder].get(), mDecoders.Length());
GetCallback()->OnVideoEOS();
}
}
void OnDecodeError() {
GetCallback()->OnDecodeError();
}
bool HasVideo() MOZ_OVERRIDE
{
return mInfo.HasVideo();
}
bool HasAudio() MOZ_OVERRIDE
{
return mInfo.HasAudio();
}
bool IsMediaSeekable() { return true; }
nsresult ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) MOZ_OVERRIDE;
nsresult Seek(int64_t aTime, int64_t aStartTime, int64_t aEndTime,
int64_t aCurrentTime) MOZ_OVERRIDE;
nsresult GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime) MOZ_OVERRIDE;
already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
MediaSourceDecoder* aParentDecoder,
MediaTaskQueue* aTaskQueue);
void Shutdown() MOZ_OVERRIDE {
MediaDecoderReader::Shutdown();
for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
mDecoders[i]->GetReader()->Shutdown();
}
}
virtual void BreakCycles() MOZ_OVERRIDE {
MediaDecoderReader::BreakCycles();
for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
mDecoders[i]->GetReader()->BreakCycles();
}
}
void InitializePendingDecoders();
bool IsShutdown() {
ReentrantMonitorAutoEnter decoderMon(mDecoder->GetReentrantMonitor());
return mDecoder->IsShutdown();
}
private:
// These are read and written on the decode task queue threads.
int64_t mTimeThreshold;
bool mDropVideoBeforeThreshold;
bool MaybeSwitchVideoReaders() {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
MOZ_ASSERT(mActiveVideoDecoder != -1);
InitializePendingDecoders();
for (uint32_t i = mActiveVideoDecoder + 1; i < mDecoders.Length(); ++i) {
if (!mDecoders[i]->GetReader()->GetMediaInfo().HasVideo()) {
continue;
}
if (mTimeThreshold >= mDecoders[i]->GetMediaStartTime()) {
GetVideoReader()->SetIdle();
mActiveVideoDecoder = i;
MSE_DEBUG("%p MSR::DecodeVF switching to %d", this, mActiveVideoDecoder);
return true;
}
}
return false;
}
MediaDecoderReader* GetAudioReader() {
if (mActiveAudioDecoder == -1) {
return nullptr;
}
return mDecoders[mActiveAudioDecoder]->GetReader();
}
MediaDecoderReader* GetVideoReader() {
if (mActiveVideoDecoder == -1) {
return nullptr;
}
return mDecoders[mActiveVideoDecoder]->GetReader();
}
nsTArray<nsRefPtr<SubBufferDecoder>> mPendingDecoders;
nsTArray<nsRefPtr<SubBufferDecoder>> mDecoders;
int32_t mActiveVideoDecoder;
int32_t mActiveAudioDecoder;
dom::MediaSource* mMediaSource;
};
class MediaSourceStateMachine : public MediaDecoderStateMachine
{
public:
MediaSourceStateMachine(MediaDecoder* aDecoder,
MediaDecoderReader* aReader,
bool aRealTime = false)
: MediaDecoderStateMachine(aDecoder, aReader, aRealTime)
{
}
already_AddRefed<SubBufferDecoder> CreateSubDecoder(const nsACString& aType,
MediaSourceDecoder* aParentDecoder) {
if (!mReader) {
return nullptr;
}
return static_cast<MediaSourceReader*>(mReader.get())->CreateSubDecoder(aType, aParentDecoder, mDecodeTaskQueue);
}
nsresult EnqueueDecoderInitialization() {
AssertCurrentThreadInMonitor();
if (!mReader) {
return NS_ERROR_FAILURE;
}
return mDecodeTaskQueue->Dispatch(NS_NewRunnableMethod(this,
&MediaSourceStateMachine::InitializePendingDecoders));
}
private:
void InitializePendingDecoders() {
if (!mReader) {
return;
}
static_cast<MediaSourceReader*>(mReader.get())->InitializePendingDecoders();
}
};
MediaSourceDecoder::MediaSourceDecoder(dom::HTMLMediaElement* aElement)
: mMediaSource(nullptr)
{
Init(aElement);
}
MediaDecoder*
MediaSourceDecoder::Clone()
{
// TODO: Sort out cloning.
return nullptr;
}
MediaDecoderStateMachine*
MediaSourceDecoder::CreateStateMachine()
{
return new MediaSourceStateMachine(this, new MediaSourceReader(this, mMediaSource));
}
nsresult
MediaSourceDecoder::Load(nsIStreamListener**, MediaDecoder*)
{
MOZ_ASSERT(!mDecoderStateMachine);
mDecoderStateMachine = CreateStateMachine();
if (!mDecoderStateMachine) {
NS_WARNING("Failed to create state machine!");
return NS_ERROR_FAILURE;
}
return mDecoderStateMachine->Init(nullptr);
}
nsresult
MediaSourceDecoder::GetSeekable(dom::TimeRanges* aSeekable)
{
double duration = mMediaSource->Duration();
if (IsNaN(duration)) {
// Return empty range.
} else if (duration > 0 && mozilla::IsInfinite(duration)) {
nsRefPtr<dom::TimeRanges> bufferedRanges = new dom::TimeRanges();
GetBuffered(bufferedRanges);
aSeekable->Add(bufferedRanges->GetStartTime(), bufferedRanges->GetEndTime());
} else {
aSeekable->Add(0, duration);
}
return NS_OK;
}
/*static*/
already_AddRefed<MediaResource>
MediaSourceDecoder::CreateResource()
{
return nsRefPtr<MediaResource>(new MediaSourceResource()).forget();
}
void
MediaSourceDecoder::AttachMediaSource(dom::MediaSource* aMediaSource)
{
MOZ_ASSERT(!mMediaSource && !mDecoderStateMachine);
mMediaSource = aMediaSource;
}
void
MediaSourceDecoder::DetachMediaSource()
{
mMediaSource = nullptr;
}
already_AddRefed<SubBufferDecoder>
MediaSourceDecoder::CreateSubDecoder(const nsACString& aType)
{
if (!mDecoderStateMachine) {
return nullptr;
}
return static_cast<MediaSourceStateMachine*>(mDecoderStateMachine.get())->CreateSubDecoder(aType, this);
}
nsresult
MediaSourceDecoder::EnqueueDecoderInitialization()
{
if (!mDecoderStateMachine) {
return NS_ERROR_FAILURE;
}
return static_cast<MediaSourceStateMachine*>(mDecoderStateMachine.get())->EnqueueDecoderInitialization();
}
class ReleaseDecodersTask : public nsRunnable {
public:
ReleaseDecodersTask(nsTArray<nsRefPtr<SubBufferDecoder>>& aDecoders)
{
mDecoders.SwapElements(aDecoders);
}
NS_IMETHOD Run() MOZ_OVERRIDE MOZ_FINAL {
mDecoders.Clear();
return NS_OK;
}
private:
nsTArray<nsRefPtr<SubBufferDecoder>> mDecoders;
};
void
MediaSourceReader::InitializePendingDecoders()
{
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
for (uint32_t i = 0; i < mPendingDecoders.Length(); ++i) {
nsRefPtr<SubBufferDecoder> decoder = mPendingDecoders[i];
MediaDecoderReader* reader = decoder->GetReader();
MSE_DEBUG("%p: Initializating subdecoder %p reader %p", this, decoder.get(), reader);
MediaInfo mi;
nsAutoPtr<MetadataTags> tags; // TODO: Handle metadata.
nsresult rv;
int64_t startTime = 0;
{
ReentrantMonitorAutoExit exitMon(mDecoder->GetReentrantMonitor());
rv = reader->ReadMetadata(&mi, getter_Transfers(tags));
if (NS_SUCCEEDED(rv)) {
reader->FindStartTime(startTime);
}
}
reader->SetIdle();
if (NS_FAILED(rv)) {
// XXX: Need to signal error back to owning SourceBuffer.
MSE_DEBUG("%p: Reader %p failed to initialize, rv=%x", this, reader, rv);
continue;
}
decoder->SetMediaStartTime(startTime);
bool active = false;
if (mi.HasVideo() || mi.HasAudio()) {
MSE_DEBUG("%p: Reader %p has video=%d audio=%d startTime=%lld",
this, reader, mi.HasVideo(), mi.HasAudio(), startTime);
active = true;
}
if (active) {
mDecoders.AppendElement(decoder);
} else {
MSE_DEBUG("%p: Reader %p not activated", this, reader);
}
}
NS_DispatchToMainThread(new ReleaseDecodersTask(mPendingDecoders));
MOZ_ASSERT(mPendingDecoders.IsEmpty());
mDecoder->NotifyWaitingForResourcesStatusChanged();
}
already_AddRefed<SubBufferDecoder>
MediaSourceReader::CreateSubDecoder(const nsACString& aType,
MediaSourceDecoder* aParentDecoder,
MediaTaskQueue* aTaskQueue)
{
// XXX: Why/when is mDecoder null here, since it should be equal to aParentDecoder?!
nsRefPtr<SubBufferDecoder> decoder =
new SubBufferDecoder(new SourceBufferResource(nullptr, aType), aParentDecoder);
nsRefPtr<MediaDecoderReader> reader(DecoderTraits::CreateReader(aType, decoder));
if (!reader) {
return nullptr;
}
// Set a callback on the subreader that forwards calls to this reader.
// This reader will then forward them onto the state machine via this
// reader's callback.
RefPtr<MediaDataDecodedListener<MediaSourceReader>> callback =
new MediaDataDecodedListener<MediaSourceReader>(this, aTaskQueue);
reader->SetCallback(callback);
reader->SetTaskQueue(aTaskQueue);
reader->Init(nullptr);
ReentrantMonitorAutoEnter mon(aParentDecoder->GetReentrantMonitor());
MSE_DEBUG("Registered subdecoder %p subreader %p", decoder.get(), reader.get());
decoder->SetReader(reader);
mPendingDecoders.AppendElement(decoder);
if (NS_FAILED(static_cast<MediaSourceDecoder*>(mDecoder)->EnqueueDecoderInitialization())) {
MSE_DEBUG("%p: Failed to enqueue decoder initialization task", this);
return nullptr;
}
mDecoder->NotifyWaitingForResourcesStatusChanged();
return decoder.forget();
}
namespace {
class ChangeToHaveMetadata : public nsRunnable {
public:
ChangeToHaveMetadata(AbstractMediaDecoder* aDecoder) :
mDecoder(aDecoder)
{
}
NS_IMETHOD Run() MOZ_OVERRIDE MOZ_FINAL {
auto owner = mDecoder->GetOwner();
if (owner) {
owner->UpdateReadyStateForData(MediaDecoderOwner::NEXT_FRAME_WAIT_FOR_MSE_DATA);
}
return NS_OK;
}
private:
nsRefPtr<AbstractMediaDecoder> mDecoder;
};
}
nsresult
MediaSourceReader::Seek(int64_t aTime, int64_t aStartTime, int64_t aEndTime,
int64_t aCurrentTime)
{
double target = static_cast<double>(aTime) / USECS_PER_S;
if (!mMediaSource->ActiveSourceBuffers()->AllContainsTime(target)) {
NS_DispatchToMainThread(new ChangeToHaveMetadata(mDecoder));
}
// Loop until we have the requested time range in the source buffers.
// This is a workaround for our lack of async functionality in the
// MediaDecoderStateMachine. Bug 979104 implements what we need and
// we'll remove this for an async approach based on that in bug XXXXXXX.
while (!mMediaSource->ActiveSourceBuffers()->AllContainsTime(target)
&& !IsShutdown()) {
mMediaSource->WaitForData();
MaybeSwitchVideoReaders();
}
if (IsShutdown()) {
return NS_OK;
}
ResetDecode();
if (GetAudioReader()) {
nsresult rv = GetAudioReader()->Seek(aTime, aStartTime, aEndTime, aCurrentTime);
if (NS_FAILED(rv)) {
return rv;
}
}
if (GetVideoReader()) {
nsresult rv = GetVideoReader()->Seek(aTime, aStartTime, aEndTime, aCurrentTime);
if (NS_FAILED(rv)) {
return rv;
}
}
return NS_OK;
}
nsresult
MediaSourceReader::GetBuffered(dom::TimeRanges* aBuffered, int64_t aStartTime)
{
for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
nsRefPtr<dom::TimeRanges> r = new dom::TimeRanges();
mDecoders[i]->GetBuffered(r);
aBuffered->Add(r->GetStartTime(), r->GetEndTime());
}
aBuffered->Normalize();
return NS_OK;
}
nsresult
MediaSourceReader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags)
{
MSE_DEBUG("%p: MSR::ReadMetadata pending=%u", this, mPendingDecoders.Length());
InitializePendingDecoders();
MSE_DEBUG("%p: MSR::ReadMetadata decoders=%u", this, mDecoders.Length());
// XXX: Make subdecoder setup async, so that use cases like bug 989888 can
// work. This will require teaching the state machine about dynamic track
// changes (and multiple tracks).
// Shorter term, make this block until we've got at least one video track
// and lie about having an audio track, then resample/remix as necessary
// to match any audio track added later to fit the format we lied about
// now. For now we just configure what we've got and cross our fingers.
int64_t maxDuration = -1;
for (uint32_t i = 0; i < mDecoders.Length(); ++i) {
MediaDecoderReader* reader = mDecoders[i]->GetReader();
MediaInfo mi = reader->GetMediaInfo();
if (mi.HasVideo() && !mInfo.HasVideo()) {
MOZ_ASSERT(mActiveVideoDecoder == -1);
mActiveVideoDecoder = i;
mInfo.mVideo = mi.mVideo;
maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
MSE_DEBUG("%p: MSR::ReadMetadata video decoder=%u maxDuration=%lld", this, i, maxDuration);
}
if (mi.HasAudio() && !mInfo.HasAudio()) {
MOZ_ASSERT(mActiveAudioDecoder == -1);
mActiveAudioDecoder = i;
mInfo.mAudio = mi.mAudio;
maxDuration = std::max(maxDuration, mDecoders[i]->GetMediaDuration());
MSE_DEBUG("%p: MSR::ReadMetadata audio decoder=%u maxDuration=%lld", this, i, maxDuration);
}
}
if (maxDuration != -1) {
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->SetMediaDuration(maxDuration);
ErrorResult dummy;
mMediaSource->SetDuration(maxDuration, dummy);
}
*aInfo = mInfo;
*aTags = nullptr; // TODO: Handle metadata.
return NS_OK;
}
} // namespace mozilla