wine/gecko
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-gecko.git synced 2024-09-13 09:24:08 -07:00
Code Issues Packages Projects Releases Wiki Activity

Bug 848954 - Part 10 - Add a MediaStreamGraph driver based on an audio callback. r=roc

Browse Source
This commit is contained in:
Paul Adenot 2014-08-26 17:01:33 +02:00
parent 10e102a0d5
commit 666e6ccb9d
10 changed files with 849 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
content/media/AudioBufferUtils.h
View File

@ -36,14 +36,14 @@ public:
AudioCallbackBufferWrapper()
: mBuffer(nullptr),
mSamples(0),
mSampleWriteOffset(0)
mSampleWriteOffset(1)
{}
/**
* Set the buffer in this wrapper. This is to be called at the beginning of
* the callback.
*/
void SetBuffer(T* aBuffer, uint32_t aFrames) {
MOZ_ASSERT(!mBuffer && !mSamples && mSampleWriteOffset,
MOZ_ASSERT(!mBuffer && !mSamples,
"SetBuffer called twice.");
mBuffer = aBuffer;
mSamples = FramesToSamples(CHANNELS, aFrames);
@ -75,10 +75,12 @@ public:
* instance can be reused.
*/
void BufferFilled() {
MOZ_ASSERT(Available() == 0,
"Audio Buffer is not full by the end of the callback.");
MOZ_ASSERT(mSamples && mSampleWriteOffset && mBuffer,
"Buffer not set.");
// It's okay to have exactly zero samples here, it can happen we have an
// audio callback driver because of a hint on MSG creation, but the
// AudioOutputStream has not been created yet.
NS_WARN_IF_FALSE(Available() == 0 || mSampleWriteOffset == 0,
"Audio Buffer is not full by the end of the callback.");
MOZ_ASSERT(mSamples, "Buffer not set.");
mSamples = 0;
mSampleWriteOffset = 0;
mBuffer = nullptr;

3
content/media/AudioSegment.cpp
View File

@ -149,7 +149,6 @@ void AudioSegment::ResampleChunks(SpeexResamplerState* aResampler, uint32_t aInR
void
AudioSegment::WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aOutputChannels, uint32_t aSampleRate)
{
uint32_t outputChannels = aOutput->GetChannels();
nsAutoTArray<AudioDataValue,AUDIO_PROCESSING_FRAMES*GUESS_AUDIO_CHANNELS> buf;
nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> channelData;
// Offset in the buffer that will end up sent to the AudioStream, in samples.
@ -208,7 +207,7 @@ AudioSegment::WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aOutputChannels
}
if (offset) {
aMixer.Mix(buf.Elements(), aOutputChannels, offset, aSampleRate);
aMixer.Mix(buf.Elements(), aOutputChannels, offset / aOutputChannels, aSampleRate);
}
}

4
content/media/AudioStream.cpp
View File

@ -347,7 +347,7 @@ AudioStream::Init(int32_t aNumChannels, int32_t aRate,
#if defined(__ANDROID__)
#if defined(MOZ_B2G)
mAudioChannel = aAudioChannel;
params.stream_type = ConvertChannelToCubebType(aAudioChannel);
params.stream_type = CubebUtils::ConvertChannelToCubebType(aAudioChannel);
#else
mAudioChannel = dom::AudioChannel::Content;
params.stream_type = CUBEB_STREAM_TYPE_MUSIC;
@ -1008,7 +1008,7 @@ AudioStream::Reset()
params.channels = mOutChannels;
#if defined(__ANDROID__)
#if defined(MOZ_B2G)
params.stream_type = ConvertChannelToCubebType(mAudioChannel);
params.stream_type = CubebUtils::ConvertChannelToCubebType(mAudioChannel);
#else
params.stream_type = CUBEB_STREAM_TYPE_MUSIC;
#endif

4
content/media/CubebUtils.cpp
View File

@ -5,8 +5,10 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include <stdint.h>
#include <algorithm>
#include "mozilla/Preferences.h"
#include "CubebUtils.h"
#include "prdtoa.h"
#define PREF_VOLUME_SCALE "media.volume_scale"
#define PREF_CUBEB_LATENCY "media.cubeb_latency_ms"
@ -145,7 +147,7 @@ bool CubebUtils::sCubebLatencyPrefSet;
}
#if defined(__ANDROID__) && defined(MOZ_B2G)
static cubeb_stream_type ConvertChannelToCubebType(dom::AudioChannel aChannel)
/*static*/ cubeb_stream_type CubebUtils::ConvertChannelToCubebType(dom::AudioChannel aChannel)
{
switch(aChannel) {
case dom::AudioChannel::Normal:

492
content/media/GraphDriver.cpp
View File

@ -4,6 +4,7 @@
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include <MediaStreamGraphImpl.h>
#include "CubebUtils.h"
#ifdef PR_LOGGING
extern PRLogModuleInfo* gMediaStreamGraphLog;
@ -28,34 +29,100 @@ struct AutoProfilerUnregisterThread
};
GraphDriver::GraphDriver(MediaStreamGraphImpl* aGraphImpl)
: mIterationStart(INITIAL_CURRENT_TIME),
mIterationEnd(INITIAL_CURRENT_TIME),
mStateComputedTime(INITIAL_CURRENT_TIME),
: mIterationStart(0),
mIterationEnd(0),
mStateComputedTime(0),
mNextStateComputedTime(0),
mGraphImpl(aGraphImpl),
mWaitState(WAITSTATE_RUNNING),
mNeedAnotherIteration(false),
mMonitor("MediaStreamGraphMonitor")
mCurrentTimeStamp(TimeStamp::Now()),
mPreviousDriver(nullptr),
mNextDriver(nullptr)
{ }
DriverHolder::DriverHolder(MediaStreamGraphImpl* aGraphImpl)
: mGraphImpl(aGraphImpl),
mLastSwitchOffset(0) // init at INITIAL_CURRENT_TIME ?
: mGraphImpl(aGraphImpl)
{ }
GraphTime
DriverHolder::GetCurrentTime()
{
MOZ_ASSERT(mDriver, "Can't get current time without a clock.");
return mLastSwitchOffset + mDriver->GetCurrentTime();
return mDriver->GetCurrentTime();
}
void
DriverHolder::Switch(GraphDriver* aDriver)
{
if (mDriver) {
mLastSwitchOffset = mDriver->GetCurrentTime();
}
MOZ_ASSERT(!mDriver, "This is only for initial switch.");
mDriver = aDriver;
mDriver->Init();
}
void GraphDriver::SetGraphTime(GraphDriver* aPreviousDriver,
GraphTime aLastSwitchNextIterationStart,
GraphTime aLastSwitchNextIterationEnd,
GraphTime aLastSwitchStateComputedTime,
GraphTime aLastSwitchNextStateComputedTime)
{
// We set mIterationEnd here, because the first thing a driver do when it
// does an iteration is to update graph times, so we are in fact setting
// mIterationStart of the next iteration by setting the end of the previous
// iteration.
mIterationStart = aLastSwitchNextIterationStart;
mIterationEnd = aLastSwitchNextIterationEnd;
mStateComputedTime = aLastSwitchStateComputedTime;
mNextStateComputedTime = aLastSwitchNextStateComputedTime;
STREAM_LOG(PR_LOG_DEBUG, ("Setting previous driver: %p (%s)", aPreviousDriver, aPreviousDriver->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver"));
MOZ_ASSERT(!mPreviousDriver);
mPreviousDriver = aPreviousDriver;
}
void GraphDriver::SwitchAtNextIteration(GraphDriver* aNextDriver)
{
STREAM_LOG(PR_LOG_DEBUG, ("Switching to new driver: %p (%s)", aNextDriver, aNextDriver->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver"));
MOZ_ASSERT(!mNextDriver);
mNextDriver = aNextDriver;
}
void GraphDriver::EnsureImmediateWakeUpLocked()
{
mGraphImpl->GetMonitor().AssertCurrentThreadOwns();
mWaitState = WAITSTATE_WAKING_UP;
mGraphImpl->GetMonitor().Notify();
}
void GraphDriver::UpdateStateComputedTime(GraphTime aStateComputedTime)
{
MOZ_ASSERT(aStateComputedTime > mIterationEnd);
// The next state computed time can be the same as the previous, here: it
// means the driver would be have been blocking indefinitly, but the graph has
// been woken up right after having been to sleep.
MOZ_ASSERT(aStateComputedTime >= mStateComputedTime, "State time can't go backward.");
mStateComputedTime = aStateComputedTime;
}
void GraphDriver::EnsureNextIteration()
{
MonitorAutoLock lock(mGraphImpl->GetMonitor());
EnsureNextIterationLocked();
}
void GraphDriver::EnsureNextIterationLocked()
{
mGraphImpl->GetMonitor().AssertCurrentThreadOwns();
if (mNeedAnotherIteration) {
return;
}
mNeedAnotherIteration = true;
if (IsWaitingIndefinitly()) {
WakeUp();
}
}
ThreadedDriver::ThreadedDriver(MediaStreamGraphImpl* aGraphImpl)
@ -63,7 +130,27 @@ ThreadedDriver::ThreadedDriver(MediaStreamGraphImpl* aGraphImpl)
{ }
ThreadedDriver::~ThreadedDriver()
{ }
{
if (mThread) {
mThread->Shutdown();
}
}
class MediaStreamGraphShutdownThreadRunnable : public nsRunnable {
public:
explicit MediaStreamGraphShutdownThreadRunnable(GraphDriver* aDriver)
: mDriver(aDriver)
{
}
NS_IMETHOD Run()
{
MOZ_ASSERT(NS_IsMainThread());
mDriver = nullptr;
return NS_OK;
}
private:
nsRefPtr<GraphDriver> mDriver;
};
class MediaStreamGraphInitThreadRunnable : public nsRunnable {
public:
@ -74,11 +161,17 @@ public:
NS_IMETHOD Run()
{
char aLocal;
STREAM_LOG(PR_LOG_DEBUG, ("Starting system thread"));
profiler_register_thread("MediaStreamGraph", &aLocal);
{
MonitorAutoLock mon(mDriver->GetThreadMonitor());
MonitorAutoLock mon(mDriver->mGraphImpl->GetMonitor());
mDriver->mGraphImpl->SwapMessageQueues();
}
if (mDriver->mPreviousDriver) {
nsCOMPtr<nsIRunnable> event = new MediaStreamGraphShutdownThreadRunnable(mDriver->mPreviousDriver);
mDriver->mPreviousDriver = nullptr;
NS_DispatchToMainThread(event);
}
mDriver->RunThread();
return NS_OK;
}
@ -94,9 +187,20 @@ ThreadedDriver::Start()
}
void
ThreadedDriver::Dispatch(nsIRunnable* aEvent)
ThreadedDriver::Revive()
{
mThread->Dispatch(aEvent, NS_DISPATCH_NORMAL);
// If we were switching, switch now. Otherwise, tell thread to run the main
// loop again.
if (mNextDriver) {
mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
mStateComputedTime, mNextStateComputedTime);
mGraphImpl->SetCurrentDriver(mNextDriver);
mNextDriver->Init();
mNextDriver->Start();
} else {
nsCOMPtr<nsIRunnable> event = new MediaStreamGraphInitThreadRunnable(this);
mThread->Dispatch(event, NS_DISPATCH_NORMAL);
}
}
void
@ -108,15 +212,13 @@ ThreadedDriver::Stop()
if (mThread) {
mThread->Shutdown();
mThread = nullptr;
}
}
SystemClockDriver::SystemClockDriver(MediaStreamGraphImpl* aGraphImpl)
: ThreadedDriver(aGraphImpl),
mInitialTimeStamp(TimeStamp::Now()),
mLastTimeStamp(TimeStamp::Now()),
mCurrentTimeStamp(TimeStamp::Now())
mLastTimeStamp(TimeStamp::Now())
{}
SystemClockDriver::~SystemClockDriver()
@ -132,14 +234,29 @@ ThreadedDriver::RunThread()
GraphTime prevCurrentTime, nextCurrentTime;
GetIntervalForIteration(prevCurrentTime, nextCurrentTime);
GraphTime nextStateComputedTime =
mStateComputedTime = mNextStateComputedTime;
mNextStateComputedTime =
mGraphImpl->RoundUpToNextAudioBlock(
IterationEnd() + mGraphImpl->MillisecondsToMediaTime(AUDIO_TARGET_MS));
nextCurrentTime + mGraphImpl->MillisecondsToMediaTime(AUDIO_TARGET_MS));
STREAM_LOG(PR_LOG_DEBUG,
("interval[%ld; %ld] state[%ld; %ld]",
(long)mIterationStart, (long)mIterationEnd,
(long)mStateComputedTime, (long)mNextStateComputedTime));
stillProcessing = mGraphImpl->OneIteration(prevCurrentTime,
nextCurrentTime,
StateComputedTime(),
nextStateComputedTime);
mNextStateComputedTime);
if (mNextDriver && stillProcessing) {
STREAM_LOG(PR_LOG_DEBUG, ("Switching to AudioCallbackDriver"));
mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
mStateComputedTime, mNextStateComputedTime);
mGraphImpl->SetCurrentDriver(mNextDriver);
mNextDriver->Init();
mNextDriver->Start();
return;
}
}
}
@ -153,9 +270,9 @@ SystemClockDriver::GetIntervalForIteration(GraphTime& aFrom, GraphTime& aTo)
mCurrentTimeStamp = now;
PR_LOG(gMediaStreamGraphLog, PR_LOG_DEBUG+1, ("Updating current time to %f (real %f, mStateComputedTime %f)",
mGraphImpl->MediaTimeToSeconds(aTo),
(now - mInitialTimeStamp).ToSeconds(),
mGraphImpl->MediaTimeToSeconds(StateComputedTime())));
mGraphImpl->MediaTimeToSeconds(aTo),
(now - mInitialTimeStamp).ToSeconds(),
mGraphImpl->MediaTimeToSeconds(StateComputedTime())));
if (mStateComputedTime < aTo) {
STREAM_LOG(PR_LOG_WARNING, ("Media graph global underrun detected"));
@ -176,9 +293,10 @@ SystemClockDriver::GetCurrentTime()
}
TimeStamp
SystemClockDriver::GetCurrentTimeStamp()
OfflineClockDriver::GetCurrentTimeStamp()
{
return mCurrentTimeStamp;
MOZ_CRASH("This driver does not support getting the current timestamp.");
return TimeStamp();
}
void
@ -200,7 +318,7 @@ SystemClockDriver::WaitForNextIteration()
mGraphImpl->PausedIndefinitly();
}
if (timeout > 0) {
mMonitor.Wait(timeout);
mGraphImpl->GetMonitor().Wait(timeout);
STREAM_LOG(PR_LOG_DEBUG+1, ("Resuming after timeout; at %f, elapsed=%f",
(TimeStamp::Now() - mInitialTimeStamp).ToSeconds(),
(TimeStamp::Now() - now).ToSeconds()));
@ -215,7 +333,7 @@ void
SystemClockDriver::WakeUp()
{
mWaitState = WAITSTATE_WAKING_UP;
mMonitor.Notify();
mGraphImpl->GetMonitor().Notify();
}
OfflineClockDriver::OfflineClockDriver(MediaStreamGraphImpl* aGraphImpl, GraphTime aSlice)
@ -225,17 +343,35 @@ OfflineClockDriver::OfflineClockDriver(MediaStreamGraphImpl* aGraphImpl, GraphTi
}
class MediaStreamGraphShutdownThreadRunnable2 : public nsRunnable {
public:
explicit MediaStreamGraphShutdownThreadRunnable2(nsIThread* aThread)
: mThread(aThread)
{
}
NS_IMETHOD Run()
{
MOZ_ASSERT(NS_IsMainThread());
mThread->Shutdown();
return NS_OK;
}
private:
nsRefPtr<nsIThread> mThread;
};
OfflineClockDriver::~OfflineClockDriver()
{ }
{
// transfer the ownership of mThread to the event
nsCOMPtr<nsIRunnable> event = new MediaStreamGraphShutdownThreadRunnable2(mThread);
mThread = nullptr;
NS_DispatchToMainThread(event);
}
void
OfflineClockDriver::GetIntervalForIteration(GraphTime& aFrom, GraphTime& aTo)
{
aFrom = mIterationStart = IterationEnd();
aTo = mIterationEnd = IterationEnd() + mGraphImpl->MillisecondsToMediaTime(mSlice);
PR_LOG(gMediaStreamGraphLog, PR_LOG_DEBUG+1, ("Updating offline current time to %f (%f)",
mGraphImpl->MediaTimeToSeconds(aTo),
mGraphImpl->MediaTimeToSeconds(StateComputedTime())));
if (mStateComputedTime < aTo) {
STREAM_LOG(PR_LOG_WARNING, ("Media graph global underrun detected"));
@ -255,6 +391,7 @@ OfflineClockDriver::GetCurrentTime()
return mIterationEnd;
}
void
OfflineClockDriver::WaitForNextIteration()
{
@ -268,5 +405,298 @@ OfflineClockDriver::WakeUp()
}
AudioCallbackDriver::AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl)
: GraphDriver(aGraphImpl),
mStarted(false)
{
}
AudioCallbackDriver::~AudioCallbackDriver()
{ }
bool
AudioCallbackDriver::Init(dom::AudioChannel aChannel = dom::AudioChannel::Normal)
{
cubeb_stream_params params;
uint32_t latency;
mSampleRate = params.rate = CubebUtils::PreferredSampleRate();
#if defined(__ANDROID__)
#if defined(MOZ_B2G)
params.stream_type = CubebUtils::ConvertChannelToCubebType(aChannels);
#else
params.stream_type = CUBEB_STREAM_TYPE_MUSIC;
#endif
if (params.stream_type == CUBEB_STREAM_TYPE_MAX) {
return false;
}
#endif
params.channels = mGraphImpl->AudioChannelCount();
if (AUDIO_OUTPUT_FORMAT == AUDIO_FORMAT_S16) {
params.format = CUBEB_SAMPLE_S16NE;
} else {
params.format = CUBEB_SAMPLE_FLOAT32NE;
}
if (cubeb_get_min_latency(CubebUtils::GetCubebContext(), params, &latency) != CUBEB_OK) {
NS_WARNING("Could not get minimal latency from cubeb.");
return false;
}
cubeb_stream* stream;
if (cubeb_stream_init(CubebUtils::GetCubebContext(), &stream,
"AudioCallbackDriver", params, latency,
DataCallback_s, StateCallback_s, this) == CUBEB_OK) {
mAudioStream.own(stream);
} else {
NS_WARNING("Could not create a cubeb stream for MediaStreamGraph.");
return false;
}
return true;
}
void
AudioCallbackDriver::Destroy()
{
mAudioStream.reset();
}
void
AudioCallbackDriver::Start()
{
STREAM_LOG(PR_LOG_DEBUG, ("Starting audio threads for MediaStreamGraph %p", mGraphImpl));
if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
NS_WARNING("Could not start cubeb stream for MSG.");
}
{
MonitorAutoLock mon(mGraphImpl->GetMonitor());
mStarted = true;
}
if (mPreviousDriver) {
nsCOMPtr<nsIRunnable> event = new MediaStreamGraphShutdownThreadRunnable(mPreviousDriver);
mPreviousDriver = nullptr;
NS_DispatchToMainThread(event);
}
}
void
AudioCallbackDriver::Stop()
{
STREAM_LOG(PR_LOG_DEBUG, ("Stopping audio threads for MediaStreamGraph %p", mGraphImpl));
if (cubeb_stream_stop(mAudioStream) != CUBEB_OK) {
NS_WARNING("Could not stop cubeb stream for MSG.");
}
}
void
AudioCallbackDriver::Revive()
{
// If we were switching, switch now. Otherwise, start the audio thread again.
if (mNextDriver) {
mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
mStateComputedTime, mNextStateComputedTime);
mGraphImpl->SetCurrentDriver(mNextDriver);
mNextDriver->Init();
mNextDriver->Start();
} else {
Init();
Start();
}
}
void
AudioCallbackDriver::GetIntervalForIteration(GraphTime& aFrom,
GraphTime& aTo)
{
}
GraphTime
AudioCallbackDriver::GetCurrentTime()
{
uint64_t position = 0;
if (cubeb_stream_get_position(mAudioStream, &position) != CUBEB_OK) {
NS_WARNING("Could not get current time from cubeb.");
}
return mSampleRate * position;
}
void
AudioCallbackDriver::WakeUp()
{
MOZ_CRASH("AudioCallbackDriver should never have to wakeup.");
}
/* static */ long
AudioCallbackDriver::DataCallback_s(cubeb_stream* aStream,
void* aUser, void* aBuffer,
long aFrames)
{
AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
return driver->DataCallback(static_cast<AudioDataValue*>(aBuffer), aFrames);
}
/* static */ void
AudioCallbackDriver::StateCallback_s(cubeb_stream* aStream, void * aUser,
cubeb_state aState)
{
AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
driver->StateCallback(aState);
}
long
AudioCallbackDriver::DataCallback(AudioDataValue* aBuffer, long aFrames)
{
bool stillProcessing;
if (mStateComputedTime == 0) {
MonitorAutoLock mon(mGraphImpl->GetMonitor());
// Because this function is called during cubeb_stream_init (to prefill the
// audio buffers), it can be that we don't have a message here (because this
// driver is the first one for this graph), and the graph would exit. Simply
// return here until we have messages.
if (!mGraphImpl->MessagesQueued()) {
PodZero(aBuffer, aFrames * mGraphImpl->AudioChannelCount());
return aFrames;
}
mGraphImpl->SwapMessageQueues();
}
uint32_t durationMS = aFrames * 1000 / mSampleRate;
// For now, simply average the duration with the previous
// duration so there is some damping against sudden changes.
if (!mIterationDurationMS) {
mIterationDurationMS = durationMS;
} else {
mIterationDurationMS += durationMS;
mIterationDurationMS /= 2;
}
mBuffer.SetBuffer(aBuffer, aFrames);
mScratchBuffer.Empty(mBuffer);
mStateComputedTime = mNextStateComputedTime;
// State computed time is decided by the audio callback's buffer length. We
// compute the iteration start and end from there, trying to keep the amount
// of buffering in the graph constant.
mNextStateComputedTime =
mGraphImpl->RoundUpToNextAudioBlock(mStateComputedTime + mBuffer.Available());
mIterationStart = mIterationEnd;
// inGraph is the number of audio frames there is between the state time and
// the current time, i.e. the maximum theoretical length of the interval we
// could use as [mIterationStart; mIterationEnd].
GraphTime inGraph = mStateComputedTime - mIterationStart;
// We want the interval [mIterationStart; mIterationEnd] to be before the
// interval [mStateComputedTime; mNextStateComputedTime]. We also want
// the distance between these intervals to be roughly equivalent each time, to
// ensure there is no clock drift between current time and state time. Since
// we can't act on the state time because we have to fill the audio buffer, we
// reclock the current time against the state time, here.
mIterationEnd = mIterationStart + 0.8 * inGraph;
STREAM_LOG(PR_LOG_DEBUG, ("interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) (duration ticks: %ld)\n",
(long)mIterationStart, (long)mIterationEnd,
(long)mStateComputedTime, (long)mNextStateComputedTime,
(long)aFrames, (uint32_t)durationMS,
(long)(mNextStateComputedTime - mStateComputedTime)));
mCurrentTimeStamp = TimeStamp::Now();
if (mStateComputedTime < mIterationEnd) {
STREAM_LOG(PR_LOG_WARNING, ("Media graph global underrun detected"));
mIterationEnd = mStateComputedTime;
}
stillProcessing = mGraphImpl->OneIteration(mIterationStart,
mIterationEnd,
mStateComputedTime,
mNextStateComputedTime);
if (stillProcessing) {
mBuffer.BufferFilled();
}
if (mNextDriver && stillProcessing) {
{
// If the audio stream has not been started by the previous driver or
// the graph itself, keep it alive.
MonitorAutoLock mon(mGraphImpl->GetMonitor());
if (!IsStarted()) {
return aFrames;
}
}
STREAM_LOG(PR_LOG_DEBUG, ("Switching to system driver."));
mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
mStateComputedTime, mNextStateComputedTime);
mGraphImpl->SetCurrentDriver(mNextDriver);
mNextDriver->Init();
mNextDriver->Start();
// Returning less than aFrames starts the draining and eventually stops the
// audio thread. This function will never get called again.
return aFrames - 1;
}
if (!stillProcessing) {
STREAM_LOG(PR_LOG_DEBUG, ("Stopping audio thread for MediaStreamGraph %p", this));
return aFrames - 1;
}
return aFrames;
}
void
AudioCallbackDriver::StateCallback(cubeb_state aState)
{
STREAM_LOG(PR_LOG_DEBUG, ("AudioCallbackDriver State: %d", aState));
}
void
AudioCallbackDriver::MixerCallback(AudioDataValue* aMixedBuffer,
AudioSampleFormat aFormat,
uint32_t aChannels,
uint32_t aFrames,
uint32_t aSampleRate)
{
uint32_t toWrite = mBuffer.Available();
if (!mBuffer.Available()) {
NS_WARNING("MediaStreamGraph SpillBuffer full, expect frame drop.");
}
MOZ_ASSERT(mBuffer.Available() <= aFrames);
mBuffer.WriteFrames(aMixedBuffer, mBuffer.Available());
MOZ_ASSERT(mBuffer.Available() == 0, "Missing frames to fill audio callback's buffer.");
DebugOnly<uint32_t> written = mScratchBuffer.Fill(aMixedBuffer + toWrite * aChannels, aFrames - toWrite);
NS_WARN_IF_FALSE(written == aFrames - toWrite, "Dropping frames.");
};
uint32_t
AudioCallbackDriver::IterationDuration()
{
// The real fix would be to have an API in cubeb to give us the number. Short
// of that, we approximate it here. bug 1019507
return mIterationDurationMS;
}
bool
AudioCallbackDriver::IsStarted() {
mGraphImpl->GetMonitor().AssertCurrentThreadOwns();
return mStarted;
}
} // namepace mozilla

289
content/media/GraphDriver.h
View File

@ -6,13 +6,49 @@
#ifndef GRAPHDRIVER_H_
#define GRAPHDRIVER_H_
#include "nsAutoPtr.h"
#include "nsAutoRef.h"
#include "AudioBufferUtils.h"
#include "AudioMixer.h"
#include "AudioSegment.h"
struct cubeb_stream;
namespace mozilla {
/**
* We make the initial graph time nonzero so that zero times can have
* special meaning if necessary.
* Assume we can run an iteration of the MediaStreamGraph loop in this much time
* or less.
* We try to run the control loop at this rate.
*/
static const int32_t INITIAL_CURRENT_TIME = 1;
static const int MEDIA_GRAPH_TARGET_PERIOD_MS = 10;
/**
* Assume that we might miss our scheduled wakeup of the MediaStreamGraph by
* this much.
*/
static const int SCHEDULE_SAFETY_MARGIN_MS = 10;
/**
* Try have this much audio buffered in streams and queued to the hardware.
* The maximum delay to the end of the next control loop
* is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS.
* There is no point in buffering more audio than this in a stream at any
* given time (until we add processing).
* This is not optimal yet.
*/
static const int AUDIO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
SCHEDULE_SAFETY_MARGIN_MS;
/**
* Try have this much video buffered. Video frames are set
* near the end of the iteration of the control loop. The maximum delay
* to the setting of the next video frame is 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
* SCHEDULE_SAFETY_MARGIN_MS. This is not optimal yet.
*/
static const int VIDEO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
SCHEDULE_SAFETY_MARGIN_MS;
class MediaStreamGraphImpl;
class MessageBlock;
@ -23,6 +59,8 @@ class MessageBlock;
typedef int64_t GraphTime;
const GraphTime GRAPH_TIME_MAX = MEDIA_TIME_MAX;
class AudioCallbackDriver;
/**
* A driver is responsible for the scheduling of the processing, the thread
* management, and give the different clocks to a MediaStreamGraph. This is an
@ -35,14 +73,8 @@ class GraphDriver
{
public:
GraphDriver(MediaStreamGraphImpl* aGraphImpl);
virtual ~GraphDriver()
{ }
/**
* Runs main control loop on the graph thread. Normally a single invocation
* of this runs for the entire lifetime of the graph thread.
*/
virtual void RunThread() = 0;
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GraphDriver);
/* When the graph wakes up to do an iteration, this returns the range of time
* that will be processed. */
virtual void GetIntervalForIteration(GraphTime& aFrom,
@ -55,20 +87,35 @@ public:
virtual void WaitForNextIteration() = 0;
/* Wakes up the graph if it is waiting. */
virtual void WakeUp() = 0;
/* Return true on success, false on error. */
// XXX put this in the AudioCallbackDriver ctor
virtual bool Init(dom::AudioChannel aChannel = dom::AudioChannel::Normal) { return true; }
virtual void Destroy() {}
/* Start the graph, creating the thread. */
virtual void Start() = 0;
/* Stop the graph, shutting down the thread. */
virtual void Stop() = 0;
/* Dispatch an event to the graph's thread. */
virtual void Dispatch(nsIRunnable* aEvent) = 0;
/* Revive this driver, as more messages just arrived. */
virtual void Revive() = 0;
/* Rate at which the GraphDriver runs, in ms. This can either be user
* controlled (because we are using a {System,Offline}ClockDriver, and decide
* how often we want to wakeup/how much we want to process per iteration), or
* it can be indirectly set by the latency of the audio backend, and the
* number of buffers of this audio backend: say we have four buffers, and 40ms
* latency, we will get a callback approximately every 10ms. */
virtual uint32_t IterationDuration() = 0;
/* Return whether we are switching or not. */
bool Switching() {
return mNextDriver || mPreviousDriver;
}
/**
* If we are running a real time graph, get the current time stamp to schedule
* video frames. This has to be reimplemented by real time drivers.
*/
virtual TimeStamp GetCurrentTimeStamp() {
MOZ_ASSERT(false, "This clock does not support getting the current time stamp.");
return TimeStamp();
return mCurrentTimeStamp;
}
bool IsWaiting() {
@ -92,55 +139,54 @@ public:
return mStateComputedTime;
}
virtual void GetAudioBuffer(float** aBuffer, long& aFrames) {
MOZ_CRASH("This is not an Audio GraphDriver!");
}
virtual AudioCallbackDriver* AsAudioCallbackDriver() {
return nullptr;
}
/**
* Tell the driver it has to stop and return the current time of the graph, so
* another driver can start from the right point in time.
*/
virtual void SwitchAtNextIteration(GraphDriver* aDriver);
/**
* Set the time for a graph, on a driver. This is used so a new driver just
* created can start at the right point in time.
*/
void SetGraphTime(GraphDriver* aPreviousDriver,
GraphTime aLastSwitchNextIterationStart,
GraphTime aLastSwitchNextIterationEnd,
GraphTime aLastSwitchNextStateComputedTime,
GraphTime aLastSwitchStateComputedTime);
/**
* Whenever the graph has computed the time until it has all state
* (mStateComputedState), it calls this to indicate the new time until which
* we have computed state.
*/
void UpdateStateComputedTime(GraphTime aStateComputedTime) {
MOZ_ASSERT(aStateComputedTime > mIterationEnd);
mStateComputedTime = aStateComputedTime;
}
Monitor& GetThreadMonitor() {
return mMonitor;
}
void UpdateStateComputedTime(GraphTime aStateComputedTime);
/**
* Call this to indicate that another iteration of the control loop is
* required immediately. The monitor must already be held.
*/
void EnsureImmediateWakeUpLocked() {
mMonitor.AssertCurrentThreadOwns();
mWaitState = WAITSTATE_WAKING_UP;
mMonitor.Notify();
}
void EnsureImmediateWakeUpLocked();
/**
* Call this to indicate that another iteration of the control loop is
* required on its regular schedule. The monitor must not be held.
* This function has to be idempotent.
*/
void EnsureNextIteration() {
MonitorAutoLock lock(mMonitor);
EnsureNextIterationLocked();
}
void EnsureNextIteration();
/**
* Same thing, but not locked.
*/
void EnsureNextIterationLocked() {
mMonitor.AssertCurrentThreadOwns();
if (mNeedAnotherIteration) {
return;
}
mNeedAnotherIteration = true;
if (IsWaitingIndefinitly()) {
WakeUp();
}
}
void EnsureNextIterationLocked();
protected:
// Time of the start of this graph iteration.
@ -149,13 +195,12 @@ protected:
GraphTime mIterationEnd;
// Time, in the future, for which blocking has been computed.
GraphTime mStateComputedTime;
GraphTime mNextStateComputedTime;
// The MediaStreamGraphImpl that owns this driver. This has a lifetime longer
// than the driver, and will never be null.
MediaStreamGraphImpl* mGraphImpl;
/**
* This enum specifies the wait state of the graph thread.
*/
// This enum specifies the wait state of the driver.
enum WaitState {
// RunThread() is running normally
WAITSTATE_RUNNING,
@ -170,9 +215,18 @@ protected:
};
WaitState mWaitState;
// True if the graph needs another iteration after the current iteration.
bool mNeedAnotherIteration;
// Monitor to synchronize the graph thread and the main thread.
Monitor mMonitor;
TimeStamp mCurrentTimeStamp;
// This is non-null only when this driver has recently switched from an other
// driver, and has not cleaned it up yet (for example because the audio stream
// is currently calling the callback during initialization).
nsRefPtr<GraphDriver> mPreviousDriver;
// This is non-null only when this driver is going to switch to an other
// driver at the end of this iteration.
nsRefPtr<GraphDriver> mNextDriver;
virtual ~GraphDriver()
{ }
};
/**
@ -186,22 +240,31 @@ public:
DriverHolder(MediaStreamGraphImpl* aGraphImpl);
GraphTime GetCurrentTime();
// Immediately switch to another driver.
void Switch(GraphDriver* aDriver);
// Create the new driver, but switch to a new one when the new driver is
// ready. System drivers don't have much problems here, but audio drivers can
// take a little while to start to fire callbacks.
void SwitchAtNextIteration(GraphDriver* aDriver);
GraphDriver* GetDriver() {
MOZ_ASSERT(mDriver);
return mDriver.get();
}
void SetCurrentDriver(GraphDriver* aDriver) {
mDriver = aDriver;
}
protected:
// The current driver
nsAutoPtr<GraphDriver> mDriver;
nsRefPtr<GraphDriver> mDriver;
// The lifetime of this pointer is equal to the lifetime of the graph, so it
// will never be null.
MediaStreamGraphImpl* mGraphImpl;
// Time we last switched driver, to properly offset from the new clock, that
// starts at zero.
GraphTime mLastSwitchOffset;
// XXX
GraphTime mNextIterationStart;
GraphTime mNextStateComputedTime;
};
class MediaStreamGraphInitThreadRunnable;
@ -216,10 +279,17 @@ public:
virtual ~ThreadedDriver();
virtual void Start() MOZ_OVERRIDE;
virtual void Stop() MOZ_OVERRIDE;
virtual void Dispatch(nsIRunnable* aEvent) MOZ_OVERRIDE;
virtual void Revive() MOZ_OVERRIDE;
/**
* Runs main control loop on the graph thread. Normally a single invocation
* of this runs for the entire lifetime of the graph thread.
*/
void RunThread();
friend MediaStreamGraphInitThreadRunnable;
private:
friend class MediaStreamGraphInitThreadRunnable;
uint32_t IterationDuration() {
return MEDIA_GRAPH_TARGET_PERIOD_MS;
}
protected:
nsCOMPtr<nsIThread> mThread;
};
@ -238,12 +308,10 @@ public:
virtual void WaitForNextIteration() MOZ_OVERRIDE;
virtual void WakeUp() MOZ_OVERRIDE;
virtual TimeStamp GetCurrentTimeStamp() MOZ_OVERRIDE;
private:
TimeStamp mInitialTimeStamp;
TimeStamp mLastTimeStamp;
TimeStamp mCurrentTimeStamp;
};
/**
@ -260,12 +328,119 @@ public:
virtual GraphTime GetCurrentTime() MOZ_OVERRIDE;
virtual void WaitForNextIteration() MOZ_OVERRIDE;
virtual void WakeUp() MOZ_OVERRIDE;
virtual TimeStamp GetCurrentTimeStamp() MOZ_OVERRIDE;
private:
// Time, in GraphTime, for each iteration
GraphTime mSlice;
};
/**
* This is a graph driver that is based on callback functions called by the
* audio api. This ensures minimal audio latency, because it means there is no
* buffering happening: the audio is generated inside the callback.
*
* This design is less flexible than running our own thread:
* - We have no control over the thread:
* - It cannot block, and it has to run for a shorter amount of time than the
* buffer it is going to fill, or an under-run is going to occur (short burst
* of silence in the final audio output).
* - We can't know for sure when the callback function is going to be called
* (although we compute an estimation so we can schedule video frames)
* - Creating and shutting the thread down is a blocking operation, that can
* take _seconds_ in some cases (because IPC has to be set up, and
* sometimes hardware components are involved and need to be warmed up)
* - We have no control on how much audio we generate, we have to return exactly
* the number of frames asked for by the callback. Since for the Web Audio
* API, we have to do block processing at 128 frames per block, we need to
* keep a little spill buffer to store the extra frames.
*/
class AudioCallbackDriver : public GraphDriver,
public MixerCallbackReceiver
{
public:
AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl);
virtual ~AudioCallbackDriver();
virtual bool Init(dom::AudioChannel aChannel) MOZ_OVERRIDE;
virtual void Destroy() MOZ_OVERRIDE;
virtual void Start() MOZ_OVERRIDE;
virtual void Stop() MOZ_OVERRIDE;
virtual void Revive() MOZ_OVERRIDE;
virtual void GetIntervalForIteration(GraphTime& aFrom,
GraphTime& aTo) MOZ_OVERRIDE;
virtual GraphTime GetCurrentTime() MOZ_OVERRIDE;
virtual void WaitForNextIteration() MOZ_OVERRIDE { }
virtual void WakeUp() MOZ_OVERRIDE;
/* Static wrapper function cubeb calls back. */
static long DataCallback_s(cubeb_stream * aStream,
void * aUser, void * aBuffer,
long aFrames);
static void StateCallback_s(cubeb_stream* aStream, void * aUser,
cubeb_state aState);
/* This function is called by the underlying audio backend when a refill is
* needed. This is what drives the whole graph when it is used to output
* audio. If the return value is exactly aFrames, this function will get
* called again. If it is less than aFrames, the stream will go in draining
* mode, and this function will not be called again. */
long DataCallback(AudioDataValue* aBuffer, long aFrames);
/* This function is called by the underlying audio backend, but is only used
* for informational purposes at the moment. */
void StateCallback(cubeb_state aState);
/* This is an approximation of the number of millisecond there are between two
* iterations of the graph. */
uint32_t IterationDuration();
/* This function gets called when the graph has produced the audio frames for
* this iteration. */
virtual void MixerCallback(AudioDataValue* aMixedBuffer,
AudioSampleFormat aFormat,
uint32_t aChannels,
uint32_t aFrames,
uint32_t aSampleRate) MOZ_OVERRIDE;
virtual AudioCallbackDriver* AsAudioCallbackDriver() {
return this;
}
bool IsStarted();
private:
/* MediaStreamGraph are always down/up mixed to stereo for now. */
static const uint32_t ChannelCount = 2;
/* The size of this buffer comes from the fact that some audio backends can
* call back with a number of frames lower than one block (128 frames), so we
* need to keep at most two block in the SpillBuffer, because we always round
* up to block boundaries during an iteration. */
SpillBuffer<AudioDataValue, WEBAUDIO_BLOCK_SIZE * 2, ChannelCount> mScratchBuffer;
/* Wrapper to ensure we write exactly the number of frames we need in the
* audio buffer cubeb passes us. */
AudioCallbackBufferWrapper<AudioDataValue, ChannelCount> mBuffer;
/* cubeb stream for this graph. This is guaranteed to be non-null after Init()
* has been called. */
nsAutoRef<cubeb_stream> mAudioStream;
/* The sample rate for the aforementionned cubeb stream. */
uint32_t mSampleRate;
/* Approximation of the time between two callbacks. This is used to schedule
* video frames. This is in milliseconds. */
uint32_t mIterationDurationMS;
/* cubeb_stream_init calls the audio callback to prefill the buffers. The
* previous driver has to be kept alive until the audio stream has been
* started, because it is responsible to call cubeb_stream_start, so we delay
* the cleanup of the previous driver until it has started the audio stream.
* Otherwise, there is a race where we kill the previous driver thread
* between cubeb_stream_init and cubeb_stream_start,
* and callbacks after the prefill never get called.
* This is written on the previous driver's thread (if switching) or main
* thread (if this driver is the first one).
* This is read on previous driver's thread (during callbacks from
* cubeb_stream_init) and the audio thread (when switching away from this
* driver back to a SystemClockDriver).
* This is synchronized by the Graph's monitor.
* */
bool mStarted;
};
}
#endif // GRAPHDRIVER_H_

184
content/media/MediaStreamGraph.cpp
View File

@ -101,7 +101,7 @@ MediaStreamGraphImpl::RemoveStream(MediaStream* aStream)
// Pending updates are not needed (since the main thread has already given
// up the stream) so we will just drop them.
{
MonitorAutoLock lock(CurrentDriver()->GetThreadMonitor());
MonitorAutoLock lock(mMonitor);
for (uint32_t i = 0; i < mStreamUpdates.Length(); ++i) {
if (mStreamUpdates[i].mStream == aStream) {
mStreamUpdates[i].mStream = nullptr;
@ -260,7 +260,7 @@ StreamTime
MediaStreamGraphImpl::GraphTimeToStreamTime(MediaStream* aStream,
GraphTime aTime)
{
NS_ASSERTION(aTime <= CurrentDriver()->StateComputedTime(),
MOZ_ASSERT(aTime <= CurrentDriver()->StateComputedTime(),
"Don't ask about times where we haven't made blocking decisions yet");
if (aTime <= IterationEnd()) {
return std::max<StreamTime>(0, aTime - aStream->mBufferStartTime);
@ -457,9 +457,9 @@ MediaStreamGraphImpl::WillUnderrun(MediaStream* aStream, GraphTime aTime,
#endif
// We should block after bufferEnd.
if (bufferEnd <= aTime) {
STREAM_LOG(PR_LOG_DEBUG+1, ("MediaStream %p will block due to data underrun, "
"bufferEnd %f",
aStream, MediaTimeToSeconds(bufferEnd)));
STREAM_LOG(PR_LOG_DEBUG+1, ("MediaStream %p will block due to data underrun at %ld, "
"bufferEnd %ld",
aStream, aTime, bufferEnd));
return true;
}
// We should keep blocking if we're currently blocked and we don't have
@ -469,8 +469,8 @@ MediaStreamGraphImpl::WillUnderrun(MediaStream* aStream, GraphTime aTime,
// we can.
if (bufferEnd <= aEndBlockingDecisions && aStream->mBlocked.GetBefore(aTime)) {
STREAM_LOG(PR_LOG_DEBUG+1, ("MediaStream %p will block due to speculative data underrun, "
"bufferEnd %f",
aStream, MediaTimeToSeconds(bufferEnd)));
"bufferEnd %f (end at %ld)",
aStream, MediaTimeToSeconds(bufferEnd), bufferEnd));
return true;
}
// Reconsider decisions at bufferEnd
@ -520,8 +520,14 @@ MediaStreamGraphImpl::UpdateStreamOrder()
}
}
if (!audioTrackPresent && CurrentDriver()->AsAudioCallbackDriver()) {
SystemClockDriver* driver = new SystemClockDriver(this);
MOZ_ASSERT(CurrentDriver()->AsAudioCallbackDriver());
mMixer.RemoveCallback(CurrentDriver()->AsAudioCallbackDriver());
CurrentDriver()->SwitchAtNextIteration(driver);
}
if (!mMixer && shouldMix) {
mMixedAudioStream->SetMicrophoneActive(true);
if (shouldMix) {
if (gFarendObserver && !mMixer.FindCallback(gFarendObserver)) {
mMixer.AddCallback(gFarendObserver);
@ -897,17 +903,11 @@ MediaStreamGraphImpl::CreateOrDestroyAudioStreams(GraphTime aAudioOutputStartTim
audioOutputStream->mLastTickWritten = 0;
audioOutputStream->mTrackID = tracks->GetID();
// If there is a mixer, there is a micrphone active.
audioOutputStream->mStream->SetMicrophoneActive(mMixer);
if (!mMixedAudioStream) {
mMixedAudioStream = new AudioStream();
// XXX for now, allocate stereo output. But we need to fix this to
// match the system's ideal channel configuration.
// NOTE: we presume this is either fast or async-under-the-covers
mMixedAudioStream->Init(AudioChannelCount(), mSampleRate,
AudioChannel::Normal,
AudioStream::LowLatency);
if (!CurrentDriver()->AsAudioCallbackDriver() &&
!CurrentDriver()->Switching()) {
AudioCallbackDriver* driver = new AudioCallbackDriver(this);
mMixer.AddCallback(driver);
CurrentDriver()->SwitchAtNextIteration(driver);
}
}
}
@ -920,18 +920,6 @@ MediaStreamGraphImpl::CreateOrDestroyAudioStreams(GraphTime aAudioOutputStartTim
}
}
void
MediaStreamGraphImpl::MixerCallback(AudioDataValue* aMixedBuffer,
AudioSampleFormat aFormat,
uint32_t aChannels,
uint32_t aFrames,
uint32_t aSampleRate)
{
MOZ_ASSERT(mMixedAudioStream);
mMixedAudioStream->Write(aMixedBuffer, aFrames, nullptr);
}
TrackTicks
MediaStreamGraphImpl::PlayAudio(MediaStream* aStream,
GraphTime aFrom, GraphTime aTo)
@ -999,6 +987,7 @@ MediaStreamGraphImpl::PlayAudio(MediaStream* aStream,
if (blocked) {
output.InsertNullDataAtStart(toWrite);
ticksWritten += toWrite;
STREAM_LOG(PR_LOG_DEBUG+1, ("MediaStream %p writing %ld blocking-silence samples for %f to %f (%ld to %ld)\n",
aStream, toWrite, MediaTimeToSeconds(t), MediaTimeToSeconds(end),
offset, offset + toWrite));
@ -1011,14 +1000,16 @@ MediaStreamGraphImpl::PlayAudio(MediaStream* aStream,
if (endTicksNeeded <= endTicksAvailable) {
output.AppendSlice(*audio, offset, endTicksNeeded);
ticksWritten += toWrite;
offset = endTicksNeeded;
} else {
MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not ended.");
// MOZ_ASSERT(track->IsEnded(), "Not enough data, and track not ended.");
// If we are at the end of the track, maybe write the remaining
// samples, and pad with/output silence.
if (endTicksNeeded > endTicksAvailable &&
offset < endTicksAvailable) {
output.AppendSlice(*audio, offset, endTicksAvailable);
ticksWritten += toWrite;
toWrite -= endTicksAvailable - offset;
offset = endTicksAvailable;
}
@ -1061,9 +1052,16 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream)
return;
// Display the next frame a bit early. This is better than letting the current
// frame be displayed for too long.
GraphTime framePosition = IterationEnd() + MEDIA_GRAPH_TARGET_PERIOD_MS;
NS_ASSERTION(framePosition >= aStream->mBufferStartTime, "frame position before buffer?");
// frame be displayed for too long. Because depending on the GraphDriver in
// use, we can't really estimate the graph interval duration, we clamp it to
// the current state computed time.
GraphTime framePosition = IterationEnd() + MillisecondsToMediaTime(CurrentDriver()->IterationDuration());
if (framePosition > CurrentDriver()->StateComputedTime()) {
NS_WARN_IF_FALSE(std::abs(framePosition - CurrentDriver()->StateComputedTime()) <
MillisecondsToMediaTime(5), "Graph thread slowdown?");
framePosition = CurrentDriver()->StateComputedTime();
}
MOZ_ASSERT(framePosition >= aStream->mBufferStartTime, "frame position before buffer?");
StreamTime frameBufferTime = GraphTimeToStreamTime(aStream, framePosition);
TrackTicks start;
@ -1126,7 +1124,7 @@ MediaStreamGraphImpl::ShouldUpdateMainThread()
}
TimeStamp now = TimeStamp::Now();
if ((now - mLastMainThreadUpdate).ToMilliseconds() > MEDIA_GRAPH_TARGET_PERIOD_MS) {
if ((now - mLastMainThreadUpdate).ToMilliseconds() > CurrentDriver()->IterationDuration()) {
mLastMainThreadUpdate = now;
return true;
}
@ -1136,7 +1134,7 @@ MediaStreamGraphImpl::ShouldUpdateMainThread()
void
MediaStreamGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate)
{
CurrentDriver()->GetThreadMonitor().AssertCurrentThreadOwns();
mMonitor.AssertCurrentThreadOwns();
// We don't want to frequently update the main thread about timing update
// when we are not running in realtime.
@ -1371,10 +1369,24 @@ MediaStreamGraphImpl::Process(GraphTime aFrom, GraphTime aTo)
}
}
if (ticksPlayed) {
if (CurrentDriver()->AsAudioCallbackDriver() && ticksPlayed) {
mMixer.FinishMixing();
}
// If we are switching away from an AudioCallbackDriver, we don't need the
// mixer anymore.
if (CurrentDriver()->AsAudioCallbackDriver() &&
CurrentDriver()->Switching()) {
bool isStarted;
{
MonitorAutoLock mon(mMonitor);
isStarted = CurrentDriver()->AsAudioCallbackDriver()->IsStarted();
}
if (isStarted) {
mMixer.RemoveCallback(CurrentDriver()->AsAudioCallbackDriver());
}
}
if (!allBlockedForever) {
CurrentDriver()->EnsureNextIteration();
}
@ -1384,6 +1396,24 @@ bool
MediaStreamGraphImpl::OneIteration(GraphTime aFrom, GraphTime aTo,
GraphTime aStateFrom, GraphTime aStateEnd)
{
{
MonitorAutoLock lock(mMemoryReportMonitor);
if (mNeedsMemoryReport) {
mNeedsMemoryReport = false;
for (uint32_t i = 0; i < mStreams.Length(); ++i) {
AudioNodeStream* stream = mStreams[i]->AsAudioNodeStream();
if (stream) {
AudioNodeSizes usage;
stream->SizeOfAudioNodesIncludingThis(MallocSizeOf, usage);
mAudioStreamSizes.AppendElement(usage);
}
}
lock.Notify();
}
}
UpdateCurrentTimeForStreams(aFrom, aTo);
UpdateGraph(aStateEnd);
@ -1393,7 +1423,7 @@ MediaStreamGraphImpl::OneIteration(GraphTime aFrom, GraphTime aTo,
// Send updates to the main thread and wait for the next control loop
// iteration.
{
MonitorAutoLock lock(CurrentDriver()->GetThreadMonitor());
MonitorAutoLock lock(mMonitor);
bool finalUpdate = mForceShutDown ||
(IterationEnd() >= mEndTime && AllFinishedStreamsNotified()) ||
(IsEmpty() && mBackMessageQueue.IsEmpty());
@ -1420,7 +1450,7 @@ MediaStreamGraphImpl::OneIteration(GraphTime aFrom, GraphTime aTo,
void
MediaStreamGraphImpl::ApplyStreamUpdate(StreamUpdate* aUpdate)
{
CurrentDriver()->GetThreadMonitor().AssertCurrentThreadOwns();
mMonitor.AssertCurrentThreadOwns();
MediaStream* stream = aUpdate->mStream;
if (!stream)
@ -1442,7 +1472,7 @@ MediaStreamGraphImpl::ForceShutDown()
NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread");
STREAM_LOG(PR_LOG_DEBUG, ("MediaStreamGraph %p ForceShutdown", this));
{
MonitorAutoLock lock(CurrentDriver()->GetThreadMonitor());
MonitorAutoLock lock(mMonitor);
mForceShutDown = true;
CurrentDriver()->EnsureNextIterationLocked();
}
@ -1450,34 +1480,17 @@ MediaStreamGraphImpl::ForceShutDown()
namespace {
class MediaStreamGraphThreadRunnable : public nsRunnable {
public:
explicit MediaStreamGraphThreadRunnable(GraphDriver* aDriver)
: mDriver(aDriver)
{
}
NS_IMETHOD Run()
{
mDriver->RunThread();
return NS_OK;
}
private:
GraphDriver* mDriver;
};
class MediaStreamGraphShutDownRunnable : public nsRunnable {
public:
MediaStreamGraphShutDownRunnable(MediaStreamGraphImpl* aGraph,
GraphDriver* aDriver)
: mGraph(aGraph),
mDriver(aDriver)
MediaStreamGraphShutDownRunnable(MediaStreamGraphImpl* aGraph)
: mGraph(aGraph)
{}
NS_IMETHOD Run()
{
NS_ASSERTION(mGraph->mDetectedNotRunning,
"We should know the graph thread control loop isn't running!");
mDriver->Stop();
STREAM_LOG(PR_LOG_DEBUG, ("Shutting drown graph %p", mGraph));
// mGraph's thread is not running so it's OK to do whatever here
if (mGraph->IsEmpty()) {
@ -1504,7 +1517,6 @@ public:
}
private:
MediaStreamGraphImpl* mGraph;
GraphDriver* mDriver;
};
class MediaStreamGraphStableStateRunnable : public nsRunnable {
@ -1566,7 +1578,7 @@ MediaStreamGraphImpl::RunInStableState()
nsTArray<nsAutoPtr<ControlMessage> > controlMessagesToRunDuringShutdown;
{
MonitorAutoLock lock(CurrentDriver()->GetThreadMonitor());
MonitorAutoLock lock(mMonitor);
mPostedRunInStableStateEvent = false;
runnables.SwapElements(mUpdateRunnables);
@ -1586,7 +1598,16 @@ MediaStreamGraphImpl::RunInStableState()
// Start the thread now. We couldn't start it earlier because
// the graph might exit immediately on finding it has no streams. The
// first message for a new graph must create a stream.
CurrentDriver()->Start();
{
// We should exit the monitor for now, because starting a stream might
// take locks, and we don't want to deadlock. We probably want this to be
// async on another thread.
MonitorAutoUnlock unlock(mMonitor);
STREAM_LOG(PR_LOG_DEBUG, ("Starting a graph with a %s\n",
CurrentDriver()->AsAudioCallbackDriver() ? "AudioDriver" :
"SystemDriver"));
CurrentDriver()->Start();
}
}
if (mCurrentTaskMessageQueue.IsEmpty()) {
@ -1602,7 +1623,7 @@ MediaStreamGraphImpl::RunInStableState()
// synchronously because it spins the event loop waiting for threads
// to shut down, and we don't want to do that in a stable state handler.
mLifecycleState = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
nsCOMPtr<nsIRunnable> event = new MediaStreamGraphShutDownRunnable(this, CurrentDriver());
nsCOMPtr<nsIRunnable> event = new MediaStreamGraphShutDownRunnable(this );
NS_DispatchToMainThread(event);
}
} else {
@ -1624,8 +1645,10 @@ MediaStreamGraphImpl::RunInStableState()
// Revive the MediaStreamGraph since we have more messages going to it.
// Note that we need to put messages into its queue before reviving it,
// or it might exit immediately.
nsCOMPtr<nsIRunnable> event = new MediaStreamGraphThreadRunnable(CurrentDriver());
CurrentDriver()->Dispatch(event);
{
MonitorAutoUnlock unlock(mMonitor);
CurrentDriver()->Revive();
}
}
}
@ -1641,7 +1664,7 @@ MediaStreamGraphImpl::RunInStableState()
// Stop MediaStreamGraph threads. Do not clear gGraph since
// we have outstanding DOM objects that may need it.
mLifecycleState = LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN;
nsCOMPtr<nsIRunnable> event = new MediaStreamGraphShutDownRunnable(this, CurrentDriver());
nsCOMPtr<nsIRunnable> event = new MediaStreamGraphShutDownRunnable(this);
NS_DispatchToMainThread(event);
}
@ -1686,7 +1709,7 @@ MediaStreamGraphImpl::EnsureRunInStableState()
void
MediaStreamGraphImpl::EnsureStableStateEventPosted()
{
CurrentDriver()->GetThreadMonitor().AssertCurrentThreadOwns();
mMonitor.AssertCurrentThreadOwns();
if (mPostedRunInStableStateEvent)
return;
@ -2510,10 +2533,12 @@ MediaInputPort::GetNextInputInterval(GraphTime aTime)
GraphTime t = aTime;
GraphTime end;
for (;;) {
if (!mDest->mBlocked.GetAt(t, &end))
if (!mDest->mBlocked.GetAt(t, &end)) {
break;
if (end >= GRAPH_TIME_MAX)
}
if (end >= GRAPH_TIME_MAX) {
return result;
}
t = end;
}
result.mStart = t;
@ -2646,6 +2671,7 @@ MediaStreamGraphImpl::MediaStreamGraphImpl(bool aRealtime,
: mDriverHolder(MOZ_THIS_IN_INITIALIZER_LIST())
, mProcessingGraphUpdateIndex(0)
, mPortCount(0)
, mMonitor("MediaStreamGraphImpl")
, mLifecycleState(LIFECYCLE_THREAD_NOT_STARTED)
, mEndTime(GRAPH_TIME_MAX)
, mSampleRate(aSampleRate)
@ -2674,15 +2700,17 @@ MediaStreamGraphImpl::MediaStreamGraphImpl(bool aRealtime,
#endif
if (mRealtime) {
printf("New Graph, using a SystemClockDriver %p\n", this);
mDriverHolder.Switch(new SystemClockDriver(this));
if (aHint & DOMMediaStream::HINT_CONTENTS_AUDIO) {
AudioCallbackDriver* driver = new AudioCallbackDriver(this);
mDriverHolder.Switch(driver);
mMixer.AddCallback(driver);
} else {
mDriverHolder.Switch(new SystemClockDriver(this));
}
} else {
printf("New Graph, using a OfflineClockDriver %p\n", this);
mDriverHolder.Switch(new OfflineClockDriver(this, MEDIA_GRAPH_TARGET_PERIOD_MS));
}
mMixer.AddCallback(this);
mLastMainThreadUpdate = TimeStamp::Now();
RegisterWeakMemoryReporter(this);
@ -2730,7 +2758,7 @@ MediaStreamGraph::GetInstance(DOMMediaStream::TrackTypeHints aHint)
CubebUtils::InitPreferredSampleRate();
gGraph = new MediaStreamGraphImpl(true, CubebUtils::PreferredSampleRate());
gGraph = new MediaStreamGraphImpl(true, CubebUtils::PreferredSampleRate(), aHint);
STREAM_LOG(PR_LOG_DEBUG, ("Starting up MediaStreamGraph %p", gGraph));
}
@ -2745,6 +2773,8 @@ MediaStreamGraph::CreateNonRealtimeInstance(TrackRate aSampleRate)
MediaStreamGraphImpl* graph = new MediaStreamGraphImpl(false, aSampleRate);
STREAM_LOG(PR_LOG_DEBUG, ("Starting up Offline MediaStreamGraph %p", graph));
return graph;
}

82
content/media/MediaStreamGraphImpl.h
View File

@ -23,39 +23,6 @@ namespace mozilla {
template <typename T>
class LinkedList;
/**
* Assume we can run an iteration of the MediaStreamGraph loop in this much time
* or less.
* We try to run the control loop at this rate.
*/
static const int MEDIA_GRAPH_TARGET_PERIOD_MS = 10;
/**
* Assume that we might miss our scheduled wakeup of the MediaStreamGraph by
* this much.
*/
static const int SCHEDULE_SAFETY_MARGIN_MS = 10;
/**
* Try have this much audio buffered in streams and queued to the hardware.
* The maximum delay to the end of the next control loop
* is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS.
* There is no point in buffering more audio than this in a stream at any
* given time (until we add processing).
* This is not optimal yet.
*/
static const int AUDIO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
SCHEDULE_SAFETY_MARGIN_MS;
/**
* Try have this much video buffered. Video frames are set
* near the end of the iteration of the control loop. The maximum delay
* to the setting of the next video frame is 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
* SCHEDULE_SAFETY_MARGIN_MS. This is not optimal yet.
*/
static const int VIDEO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
SCHEDULE_SAFETY_MARGIN_MS;
/**
* A per-stream update message passed from the media graph thread to the
* main thread.
@ -100,12 +67,12 @@ protected:
MediaStream* mStream;
};
struct MessageBlock {
class MessageBlock {
public:
int64_t mGraphUpdateIndex;
nsTArray<nsAutoPtr<ControlMessage> > mMessages;
};
/**
* The implementation of a media stream graph. This class is private to this
* file. It's not in the anonymous namespace because MediaStream needs to
@ -115,8 +82,7 @@ struct MessageBlock {
* OfflineAudioContext object.
*/
class MediaStreamGraphImpl : public MediaStreamGraph,
public nsIMemoryReporter,
public MixerCallbackReceiver {
public nsIMemoryReporter {
public:
NS_DECL_ISUPPORTS
NS_DECL_NSIMEMORYREPORTER
@ -186,9 +152,9 @@ public:
bool OneIteration(GraphTime aFrom, GraphTime aTo,
GraphTime aStateFrom, GraphTime aStateEnd);
// Get
// Get the message queue, from the current GraphDriver thread.
nsTArray<MessageBlock>& MessageQueue() {
CurrentDriver()->GetThreadMonitor().AssertCurrentThreadOwns();
mMonitor.AssertCurrentThreadOwns();
return mFrontMessageQueue;
}
@ -230,7 +196,7 @@ public:
void UpdateGraph(GraphTime aEndBlockingDecisions);
void SwapMessageQueues() {
CurrentDriver()->GetThreadMonitor().AssertCurrentThreadOwns();
mMonitor.AssertCurrentThreadOwns();
mFrontMessageQueue.SwapElements(mBackMessageQueue);
}
/**
@ -361,15 +327,6 @@ public:
* to the audio output stream. Returns the number of frames played.
*/
TrackTicks PlayAudio(MediaStream* aStream, GraphTime aFrom, GraphTime aTo);
/* The mixer call the Graph back when all the media streams that have audio
* outputs have been mixed down, so it can write to its AudioStream to output
* sound. */
virtual void MixerCallback(AudioDataValue* aMixedBuffer,
AudioSampleFormat aFormat,
uint32_t aChannels,
uint32_t aFrames,
uint32_t aSampleRate) MOZ_OVERRIDE;
/**
* Set the correct current video frame for stream aStream.
*/
@ -446,6 +403,20 @@ public:
return RateConvertTicksRoundDown(aRate, GraphRate(), aTime);
}
/**
* Effectively set the new driver, while we are switching.
* It is only safe to call this at the very end of an iteration, when there
* has been a SwitchAtNextIteration call during the iteration. The driver
* should return and pass the control to the new driver shortly after.
*/
void SetCurrentDriver(GraphDriver* aDriver) {
mDriverHolder.SetCurrentDriver(aDriver);
}
Monitor& GetMonitor() {
return mMonitor;
}
// Data members
// The following state is managed on the graph thread only, unless
@ -477,6 +448,13 @@ public:
*/
int32_t mPortCount;
// mMonitor guards the data below.
// MediaStreamGraph normally does its work without holding mMonitor, so it is
// not safe to just grab mMonitor from some thread and start monkeying with
// the graph. Instead, communicate with the graph thread using provided
// mechanisms such as the ControlMessage queue.
Monitor mMonitor;
// Data guarded by mMonitor (must always be accessed with mMonitor held,
// regardless of the value of mLifecycleState.
@ -496,6 +474,12 @@ public:
nsTArray<MessageBlock> mFrontMessageQueue;
/* Message queue in which the main thread appends messages. */
nsTArray<MessageBlock> mBackMessageQueue;
/* True if there will messages to process if we swap the message queues. */
bool MessagesQueued() {
mMonitor.AssertCurrentThreadOwns();
return !mBackMessageQueue.IsEmpty();
}
/**
* This enum specifies where this graph is in its lifecycle. This is used
* to control shutdown.

3
content/media/TrackUnionStream.h
View File

@ -250,8 +250,7 @@ protected:
StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd);
TrackTicks startTicks = outputTrack->GetEnd();
StreamTime outputStart = GraphTimeToStreamTime(interval.mStart);
NS_WARN_IF_FALSE(startTicks == TimeToTicksRoundUp(rate, outputStart),
"Samples missing");
MOZ_ASSERT(startTicks == TimeToTicksRoundUp(rate, outputStart), "Samples missing");
TrackTicks endTicks = TimeToTicksRoundUp(rate, outputEnd);
TrackTicks ticks = endTicks - startTicks;
StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart);

1
content/media/compiledtest/TestAudioBuffers.cpp
View File

@ -5,6 +5,7 @@
#include <stdint.h>
#include <assert.h>
#include <mozilla/NullPtr.h>
#include "AudioBufferUtils.h"
const uint32_t FRAMES = 256;