diff --git a/dom/media/AudioCaptureStream.cpp b/dom/media/AudioCaptureStream.cpp new file mode 100644 index 00000000000..f2205032895 --- /dev/null +++ b/dom/media/AudioCaptureStream.cpp @@ -0,0 +1,133 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaStreamGraphImpl.h" +#include "mozilla/MathAlgorithms.h" +#include "mozilla/unused.h" + +#include "AudioSegment.h" +#include "mozilla/Logging.h" +#include "mozilla/Attributes.h" +#include "AudioCaptureStream.h" +#include "ImageContainer.h" +#include "AudioNodeEngine.h" +#include "AudioNodeStream.h" +#include "AudioNodeExternalInputStream.h" +#include "webaudio/MediaStreamAudioDestinationNode.h" +#include +#include "DOMMediaStream.h" + +using namespace mozilla::layers; +using namespace mozilla::dom; +using namespace mozilla::gfx; + +namespace mozilla +{ + +// We are mixing to mono until PeerConnection can accept stereo +static const uint32_t MONO = 1; + +AudioCaptureStream::AudioCaptureStream(DOMMediaStream* aWrapper) + : ProcessedMediaStream(aWrapper), mTrackCreated(false) +{ + MOZ_ASSERT(NS_IsMainThread()); + MOZ_COUNT_CTOR(AudioCaptureStream); + mMixer.AddCallback(this); +} + +AudioCaptureStream::~AudioCaptureStream() +{ + MOZ_COUNT_DTOR(AudioCaptureStream); + mMixer.RemoveCallback(this); +} + +void +AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo, + uint32_t aFlags) +{ + uint32_t inputCount = mInputs.Length(); + StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK); + // Notify the DOM everything is in order. + if (!mTrackCreated) { + for (uint32_t i = 0; i < mListeners.Length(); i++) { + MediaStreamListener* l = mListeners[i]; + AudioSegment tmp; + l->NotifyQueuedTrackChanges( + Graph(), AUDIO_TRACK, 0, MediaStreamListener::TRACK_EVENT_CREATED, tmp); + l->NotifyFinishedTrackCreation(Graph()); + } + mTrackCreated = true; + } + + // If the captured stream is connected back to a object on the page (be it an + // HTMLMediaElement with a stream as source, or an AudioContext), a cycle + // situation occur. This can work if it's an AudioContext with at least one + // DelayNode, but the MSG will mute the whole cycle otherwise. + bool blocked = mFinished || mBlocked.GetAt(aFrom); + if (blocked || InMutedCycle() || inputCount == 0) { + track->Get()->AppendNullData(aTo - aFrom); + } else { + // We mix down all the tracks of all inputs, to a stereo track. Everything + // is {up,down}-mixed to stereo. + mMixer.StartMixing(); + AudioSegment output; + for (uint32_t i = 0; i < inputCount; i++) { + MediaStream* s = mInputs[i]->GetSource(); + StreamBuffer::TrackIter tracks(s->GetStreamBuffer(), MediaSegment::AUDIO); + while (!tracks.IsEnded()) { + AudioSegment* inputSegment = tracks->Get(); + StreamTime inputStart = s->GraphTimeToStreamTime(aFrom); + StreamTime inputEnd = s->GraphTimeToStreamTime(aTo); + AudioSegment toMix; + toMix.AppendSlice(*inputSegment, inputStart, inputEnd); + // Care for streams blocked in the [aTo, aFrom] range. + if (inputEnd - inputStart < aTo - aFrom) { + toMix.AppendNullData((aTo - aFrom) - (inputEnd - inputStart)); + } + toMix.Mix(mMixer, MONO, Graph()->GraphRate()); + tracks.Next(); + } + } + // This calls MixerCallback below + mMixer.FinishMixing(); + } + + // Regardless of the status of the input tracks, we go foward. + mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTime((aTo))); +} + +void +AudioCaptureStream::MixerCallback(AudioDataValue* aMixedBuffer, + AudioSampleFormat aFormat, uint32_t aChannels, + uint32_t aFrames, uint32_t aSampleRate) +{ + nsAutoTArray, MONO> output; + nsAutoTArray bufferPtrs; + output.SetLength(MONO); + bufferPtrs.SetLength(MONO); + + uint32_t written = 0; + // We need to copy here, because the mixer will reuse the storage, we should + // not hold onto it. Buffers are in planar format. + for (uint32_t channel = 0; channel < aChannels; channel++) { + AudioDataValue* out = output[channel].AppendElements(aFrames); + PodCopy(out, aMixedBuffer + written, aFrames); + bufferPtrs[channel] = out; + written += aFrames; + } + AudioChunk chunk; + chunk.mBuffer = new mozilla::SharedChannelArrayBuffer(&output); + chunk.mDuration = aFrames; + chunk.mBufferFormat = aFormat; + chunk.mVolume = 1.0f; + chunk.mChannelData.SetLength(MONO); + for (uint32_t channel = 0; channel < aChannels; channel++) { + chunk.mChannelData[channel] = bufferPtrs[channel]; + } + + // Now we have mixed data, simply append it to out track. + EnsureTrack(AUDIO_TRACK)->Get()->AppendAndConsumeChunk(&chunk); +} +} diff --git a/dom/media/AudioCaptureStream.h b/dom/media/AudioCaptureStream.h new file mode 100644 index 00000000000..322dcd88045 --- /dev/null +++ b/dom/media/AudioCaptureStream.h @@ -0,0 +1,40 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_AUDIOCAPTURESTREAM_H_ +#define MOZILLA_AUDIOCAPTURESTREAM_H_ + +#include "MediaStreamGraph.h" +#include "AudioMixer.h" +#include + +namespace mozilla +{ + +class DOMMediaStream; + +/** + * See MediaStreamGraph::CreateAudioCaptureStream. + */ +class AudioCaptureStream : public ProcessedMediaStream, + public MixerCallbackReceiver +{ +public: + explicit AudioCaptureStream(DOMMediaStream* aWrapper); + virtual ~AudioCaptureStream(); + + void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override; + +protected: + enum { AUDIO_TRACK = 1 }; + void MixerCallback(AudioDataValue* aMixedBuffer, AudioSampleFormat aFormat, + uint32_t aChannels, uint32_t aFrames, + uint32_t aSampleRate) override; + AudioMixer mMixer; + bool mTrackCreated; +}; +} + +#endif /* MOZILLA_AUDIOCAPTURESTREAM_H_ */ diff --git a/dom/media/AudioMixer.h b/dom/media/AudioMixer.h index c992942e25f..c86aa33455b 100644 --- a/dom/media/AudioMixer.h +++ b/dom/media/AudioMixer.h @@ -26,7 +26,9 @@ struct MixerCallbackReceiver { * stream. * * AudioMixer::Mix is to be called repeatedly with buffers that have the same - * length, sample rate, sample format and channel count. + * length, sample rate, sample format and channel count. This class works with + * interleaved and plannar buffers, but the buffer mixed must be of the same + * type during a mixing cycle. * * When all the tracks have been mixed, calling FinishMixing will call back with * a buffer containing the mixed audio data. @@ -71,7 +73,7 @@ public: mSampleRate = mChannels = mFrames = 0; } - /* Add a buffer to the mix. aSamples is interleaved. */ + /* Add a buffer to the mix. */ void Mix(AudioDataValue* aSamples, uint32_t aChannels, uint32_t aFrames, diff --git a/dom/media/AudioSegment.cpp b/dom/media/AudioSegment.cpp index ddd16bbe16a..15ac13e5537 100644 --- a/dom/media/AudioSegment.cpp +++ b/dom/media/AudioSegment.cpp @@ -146,6 +146,103 @@ void AudioSegment::ResampleChunks(SpeexResamplerState* aResampler, uint32_t aInR } } +// This helps to to safely get a pointer to the position we want to start +// writing a planar audio buffer, depending on the channel and the offset in the +// buffer. +static AudioDataValue* +PointerForOffsetInChannel(AudioDataValue* aData, size_t aLengthSamples, + uint32_t aChannelCount, uint32_t aChannel, + uint32_t aOffsetSamples) +{ + size_t samplesPerChannel = aLengthSamples / aChannelCount; + size_t beginningOfChannel = samplesPerChannel * aChannel; + MOZ_ASSERT(aChannel * samplesPerChannel + aOffsetSamples < aLengthSamples, + "Offset request out of bounds."); + return aData + beginningOfChannel + aOffsetSamples; +} + +void +AudioSegment::Mix(AudioMixer& aMixer, uint32_t aOutputChannels, + uint32_t aSampleRate) +{ + nsAutoTArray + buf; + nsAutoTArray channelData; + uint32_t offsetSamples = 0; + uint32_t duration = GetDuration(); + + if (duration <= 0) { + MOZ_ASSERT(duration == 0); + return; + } + + uint32_t outBufferLength = duration * aOutputChannels; + buf.SetLength(outBufferLength); + + for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) { + AudioChunk& c = *ci; + uint32_t frames = c.mDuration; + + // If the chunk is silent, simply write the right number of silence in the + // buffers. + if (c.mBufferFormat == AUDIO_FORMAT_SILENCE) { + for (uint32_t channel = 0; channel < aOutputChannels; channel++) { + AudioDataValue* ptr = + PointerForOffsetInChannel(buf.Elements(), outBufferLength, + aOutputChannels, channel, offsetSamples); + PodZero(ptr, frames); + } + } else { + // Othewise, we need to upmix or downmix appropriately, depending on the + // desired input and output channels. + channelData.SetLength(c.mChannelData.Length()); + for (uint32_t i = 0; i < channelData.Length(); ++i) { + channelData[i] = c.mChannelData[i]; + } + if (channelData.Length() < aOutputChannels) { + // Up-mix. + AudioChannelsUpMix(&channelData, aOutputChannels, gZeroChannel); + for (uint32_t channel = 0; channel < aOutputChannels; channel++) { + AudioDataValue* ptr = + PointerForOffsetInChannel(buf.Elements(), outBufferLength, + aOutputChannels, channel, offsetSamples); + PodCopy(ptr, reinterpret_cast(channelData[channel]), + frames); + } + MOZ_ASSERT(channelData.Length() == aOutputChannels); + } else if (channelData.Length() > aOutputChannels) { + // Down mix. + nsAutoTArray outChannelPtrs; + outChannelPtrs.SetLength(aOutputChannels); + uint32_t offsetSamples = 0; + for (uint32_t channel = 0; channel < aOutputChannels; channel++) { + outChannelPtrs[channel] = + PointerForOffsetInChannel(buf.Elements(), outBufferLength, + aOutputChannels, channel, offsetSamples); + } + AudioChannelsDownMix(channelData, outChannelPtrs.Elements(), + aOutputChannels, frames); + } else { + // The channel count is already what we want, just copy it over. + for (uint32_t channel = 0; channel < aOutputChannels; channel++) { + AudioDataValue* ptr = + PointerForOffsetInChannel(buf.Elements(), outBufferLength, + aOutputChannels, channel, offsetSamples); + PodCopy(ptr, reinterpret_cast(channelData[channel]), + frames); + } + } + } + offsetSamples += frames; + } + + if (offsetSamples) { + MOZ_ASSERT(offsetSamples == outBufferLength / aOutputChannels, + "We forgot to write some samples?"); + aMixer.Mix(buf.Elements(), aOutputChannels, offsetSamples, aSampleRate); + } +} + void AudioSegment::WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aOutputChannels, uint32_t aSampleRate) { diff --git a/dom/media/AudioSegment.h b/dom/media/AudioSegment.h index 22bad4e7790..25c0057f1b7 100644 --- a/dom/media/AudioSegment.h +++ b/dom/media/AudioSegment.h @@ -299,7 +299,14 @@ public: return chunk; } void ApplyVolume(float aVolume); - void WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aChannelCount, uint32_t aSampleRate); + // Mix the segment into a mixer, interleaved. This is useful to output a + // segment to a system audio callback. It up or down mixes to aChannelCount + // channels. + void WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aChannelCount, + uint32_t aSampleRate); + // Mix the segment into a mixer, keeping it planar, up or down mixing to + // aChannelCount channels. + void Mix(AudioMixer& aMixer, uint32_t aChannelCount, uint32_t aSampleRate); int ChannelCount() { NS_WARN_IF_FALSE(!mChunks.IsEmpty(), diff --git a/dom/media/DOMMediaStream.cpp b/dom/media/DOMMediaStream.cpp index 40bdee8bd36..e75d5fbfad0 100644 --- a/dom/media/DOMMediaStream.cpp +++ b/dom/media/DOMMediaStream.cpp @@ -301,6 +301,18 @@ DOMMediaStream::InitTrackUnionStream(nsIDOMWindow* aWindow, InitStreamCommon(aGraph->CreateTrackUnionStream(this)); } +void +DOMMediaStream::InitAudioCaptureStream(nsIDOMWindow* aWindow, + MediaStreamGraph* aGraph) +{ + mWindow = aWindow; + + if (!aGraph) { + aGraph = MediaStreamGraph::GetInstance(); + } + InitStreamCommon(aGraph->CreateAudioCaptureStream(this)); +} + void DOMMediaStream::InitStreamCommon(MediaStream* aStream) { @@ -329,6 +341,15 @@ DOMMediaStream::CreateTrackUnionStream(nsIDOMWindow* aWindow, return stream.forget(); } +already_AddRefed +DOMMediaStream::CreateAudioCaptureStream(nsIDOMWindow* aWindow, + MediaStreamGraph* aGraph) +{ + nsRefPtr stream = new DOMMediaStream(); + stream->InitAudioCaptureStream(aWindow, aGraph); + return stream.forget(); +} + void DOMMediaStream::SetTrackEnabled(TrackID aTrackID, bool aEnabled) { @@ -653,6 +674,15 @@ DOMLocalMediaStream::CreateTrackUnionStream(nsIDOMWindow* aWindow, return stream.forget(); } +already_AddRefed +DOMLocalMediaStream::CreateAudioCaptureStream(nsIDOMWindow* aWindow, + MediaStreamGraph* aGraph) +{ + nsRefPtr stream = new DOMLocalMediaStream(); + stream->InitAudioCaptureStream(aWindow, aGraph); + return stream.forget(); +} + DOMAudioNodeMediaStream::DOMAudioNodeMediaStream(AudioNode* aNode) : mStreamNode(aNode) { diff --git a/dom/media/DOMMediaStream.h b/dom/media/DOMMediaStream.h index ebad4e70afb..fb7bac8733b 100644 --- a/dom/media/DOMMediaStream.h +++ b/dom/media/DOMMediaStream.h @@ -198,6 +198,13 @@ public: static already_AddRefed CreateTrackUnionStream(nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr); + /** + * Create an nsDOMMediaStream whose underlying stream is an + * AudioCaptureStream + */ + static already_AddRefed CreateAudioCaptureStream( + nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr); + void SetLogicalStreamStartTime(StreamTime aTime) { mLogicalStreamStartTime = aTime; @@ -261,6 +268,8 @@ protected: MediaStreamGraph* aGraph = nullptr); void InitTrackUnionStream(nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr); + void InitAudioCaptureStream(nsIDOMWindow* aWindow, + MediaStreamGraph* aGraph = nullptr); void InitStreamCommon(MediaStream* aStream); already_AddRefed CreateAudioTrack(AudioStreamTrack* aStreamTrack); already_AddRefed CreateVideoTrack(VideoStreamTrack* aStreamTrack); @@ -351,6 +360,12 @@ public: CreateTrackUnionStream(nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr); + /** + * Create an nsDOMLocalMediaStream whose underlying stream is an + * AudioCaptureStream. */ + static already_AddRefed CreateAudioCaptureStream( + nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr); + protected: virtual ~DOMLocalMediaStream(); }; diff --git a/dom/media/moz.build b/dom/media/moz.build index 63c6fa3366b..25d2dbb728d 100644 --- a/dom/media/moz.build +++ b/dom/media/moz.build @@ -196,6 +196,7 @@ EXPORTS.mozilla.dom += [ UNIFIED_SOURCES += [ 'AbstractThread.cpp', + 'AudioCaptureStream.cpp', 'AudioChannelFormat.cpp', 'AudioCompactor.cpp', 'AudioSegment.cpp',