Bug 856361. Part 5: Implement MediaStreamAudioSourceNode. r=ehsan

--HG--
extra : rebase_source : ee1a8c2dc39574aeb6165a063553489f08d19380

content/media/MediaStreamGraph.cpp

@@ -22,6 +22,7 @@
 #include "AudioChannelCommon.h"
 #include "AudioNodeEngine.h"
 #include "AudioNodeStream.h"
+#include "AudioNodeExternalInputStream.h"
 #include <algorithm>
 #include "DOMMediaStream.h"
 #include "GeckoProfiler.h"
@@ -2304,6 +2305,21 @@ MediaStreamGraph::CreateTrackUnionStream(DOMMediaStream* aWrapper)
   return stream;
 }
 
+AudioNodeExternalInputStream*
+MediaStreamGraph::CreateAudioNodeExternalInputStream(AudioNodeEngine* aEngine, TrackRate aSampleRate)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  if (!aSampleRate) {
+    aSampleRate = aEngine->NodeMainThread()->Context()->SampleRate();
+  }
+  AudioNodeExternalInputStream* stream = new AudioNodeExternalInputStream(aEngine, aSampleRate);
+  NS_ADDREF(stream);
+  MediaStreamGraphImpl* graph = static_cast<MediaStreamGraphImpl*>(this);
+  stream->SetGraphImpl(graph);
+  graph->AppendMessage(new CreateMessage(stream));
+  return stream;
+}
+
 AudioNodeStream*
 MediaStreamGraph::CreateAudioNodeStream(AudioNodeEngine* aEngine,
                                         AudioNodeStreamKind aKind,

content/media/MediaStreamGraph.h

@@ -191,8 +191,9 @@ class MediaStreamGraphImpl;
 class SourceMediaStream;
 class ProcessedMediaStream;
 class MediaInputPort;
-class AudioNodeStream;
 class AudioNodeEngine;
+class AudioNodeExternalInputStream;
+class AudioNodeStream;
 struct AudioChunk;
 
 /**
@@ -360,6 +361,7 @@ public:
 
   friend class MediaStreamGraphImpl;
   friend class MediaInputPort;
+  friend class AudioNodeExternalInputStream;
 
   virtual SourceMediaStream* AsSourceStream() { return nullptr; }
   virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
@@ -968,6 +970,11 @@ public:
   AudioNodeStream* CreateAudioNodeStream(AudioNodeEngine* aEngine,
                                          AudioNodeStreamKind aKind,
                                          TrackRate aSampleRate = 0);
+
+  AudioNodeExternalInputStream*
+  CreateAudioNodeExternalInputStream(AudioNodeEngine* aEngine,
+                                     TrackRate aSampleRate = 0);
+
   /**
    * Returns the number of graph updates sent. This can be used to track
    * whether a given update has been processed by the graph thread and reflected

content/media/moz.build

@@ -53,6 +53,7 @@ EXPORTS += [
     'AudioChannelFormat.h',
     'AudioEventTimeline.h',
     'AudioNodeEngine.h',
+    'AudioNodeExternalInputStream.h',
     'AudioNodeStream.h',
     'AudioSampleFormat.h',
     'AudioSegment.h',
@@ -97,6 +98,7 @@ CPP_SOURCES += [
     'AudioAvailableEventManager.cpp',
     'AudioChannelFormat.cpp',
     'AudioNodeEngine.cpp',
+    'AudioNodeExternalInputStream.cpp',
     'AudioNodeStream.cpp',
     'AudioSegment.cpp',
     'AudioStream.cpp',

content/media/webaudio/AudioContext.cpp

@@ -16,6 +16,7 @@
 #include "AudioBufferSourceNode.h"
 #include "AudioBuffer.h"
 #include "GainNode.h"
+#include "MediaStreamAudioSourceNode.h"
 #include "DelayNode.h"
 #include "PannerNode.h"
 #include "AudioListener.h"
@@ -253,6 +254,18 @@ AudioContext::CreateAnalyser()
   return analyserNode.forget();
 }
 
+already_AddRefed<MediaStreamAudioSourceNode>
+AudioContext::CreateMediaStreamSource(const DOMMediaStream& aMediaStream,
+                                      ErrorResult& aRv)
+{
+  if (mIsOffline) {
+    aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+    return nullptr;
+  }
+  nsRefPtr<MediaStreamAudioSourceNode> mediaStreamAudioSourceNode = new MediaStreamAudioSourceNode(this, &aMediaStream);
+  return mediaStreamAudioSourceNode.forget();
+}
+
 already_AddRefed<GainNode>
 AudioContext::CreateGain()
 {

content/media/webaudio/AudioContext.h

@@ -53,6 +53,7 @@ class DynamicsCompressorNode;
 class GainNode;
 class GlobalObject;
 class MediaStreamAudioDestinationNode;
+class MediaStreamAudioSourceNode;
 class OfflineRenderSuccessCallback;
 class PannerNode;
 class ScriptProcessorNode;
@@ -161,6 +162,9 @@ public:
     return CreateGain();
   }
 
+  already_AddRefed<MediaStreamAudioSourceNode>
+  CreateMediaStreamSource(const DOMMediaStream& aMediaStream, ErrorResult& aRv);
+
   already_AddRefed<DelayNode>
   CreateDelay(double aMaxDelayTime, ErrorResult& aRv);
 

content/media/webaudio/MediaStreamAudioSourceNode.cpp

@@ -0,0 +1,48 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaStreamAudioSourceNode.h"
+#include "mozilla/dom/MediaStreamAudioSourceNodeBinding.h"
+#include "AudioNodeEngine.h"
+#include "AudioNodeExternalInputStream.h"
+#include "DOMMediaStream.h"
+
+namespace mozilla {
+namespace dom {
+
+MediaStreamAudioSourceNode::MediaStreamAudioSourceNode(AudioContext* aContext,
+                                                       const DOMMediaStream* aMediaStream)
+  : AudioNode(aContext,
+              2,
+              ChannelCountMode::Max,
+              ChannelInterpretation::Speakers)
+{
+  AudioNodeEngine* engine = new AudioNodeEngine(this);
+  mStream = aContext->Graph()->CreateAudioNodeExternalInputStream(engine);
+  ProcessedMediaStream* outputStream = static_cast<ProcessedMediaStream*>(mStream.get());
+  mInputPort = outputStream->AllocateInputPort(aMediaStream->GetStream(),
+                                               MediaInputPort::FLAG_BLOCK_INPUT);
+}
+
+void
+MediaStreamAudioSourceNode::DestroyMediaStream()
+{
+  if (mInputPort) {
+    mInputPort->Destroy();
+    mInputPort = nullptr;
+  }
+  AudioNode::DestroyMediaStream();
+}
+
+JSObject*
+MediaStreamAudioSourceNode::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aScope)
+{
+  return MediaStreamAudioSourceNodeBinding::Wrap(aCx, aScope, this);
+}
+
+}
+}
+

content/media/webaudio/MediaStreamAudioSourceNode.h

@@ -0,0 +1,34 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MediaStreamAudioSourceNode_h_
+#define MediaStreamAudioSourceNode_h_
+
+#include "AudioNode.h"
+
+namespace mozilla {
+namespace dom {
+
+class MediaStreamAudioSourceNode : public AudioNode
+{
+public:
+  MediaStreamAudioSourceNode(AudioContext* aContext, const DOMMediaStream* aMediaStream);
+
+  virtual JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aScope) MOZ_OVERRIDE;
+
+  virtual void DestroyMediaStream() MOZ_OVERRIDE;
+
+  virtual uint16_t NumberOfInputs() const MOZ_OVERRIDE { return 0; }
+
+private:
+  nsRefPtr<MediaInputPort> mInputPort;
+};
+
+}
+}
+
+#endif
+

content/media/webaudio/moz.build

@@ -40,6 +40,7 @@ EXPORTS.mozilla.dom += [
     'EnableWebAudioCheck.h',
     'GainNode.h',
     'MediaStreamAudioDestinationNode.h',
+    'MediaStreamAudioSourceNode.h',
     'OfflineAudioCompletionEvent.h',
     'PannerNode.h',
     'PeriodicWave.h',
@@ -67,6 +68,7 @@ CPP_SOURCES += [
     'GainNode.cpp',
     'MediaBufferDecoder.cpp',
     'MediaStreamAudioDestinationNode.cpp',
+    'MediaStreamAudioSourceNode.cpp',
     'OfflineAudioCompletionEvent.cpp',
     'PannerNode.cpp',
     'PeriodicWave.cpp',

content/media/webaudio/test/Makefile.in

@@ -68,6 +68,9 @@ MOCHITEST_FILES := \
   test_maxChannelCount.html \
   test_mediaDecoding.html \
   test_mediaStreamAudioDestinationNode.html \
+  test_mediaStreamAudioSourceNode.html \
+  test_mediaStreamAudioSourceNodeCrossOrigin.html \
+  test_mediaStreamAudioSourceNodeResampling.html \
   test_mixingRules.html \
   test_nodeToParamConnection.html \
   test_OfflineAudioContext.html \

content/media/webaudio/test/test_mediaStreamAudioSourceNode.html

@@ -0,0 +1,44 @@
+<!DOCTYPE HTML>
+<html>
+<meta charset="utf-8">
+<head>
+  <title>Test MediaStreamAudioSourceNode processing is correct</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="text/javascript" src="webaudio.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+
+var gTest = {
+  length: 2048,
+  skipOfflineContextTests: true,
+  createGraph: function(context) {
+    var sourceGraph = new AudioContext();
+    var source = sourceGraph.createBufferSource();
+    source.buffer = this.buffer;
+    var dest = sourceGraph.createMediaStreamDestination();
+    source.connect(dest);
+    source.start(0);
+
+    var mediaStreamSource = context.createMediaStreamSource(dest.stream);
+    return mediaStreamSource;
+  },
+  createExpectedBuffers: function(context) {
+    var buffer = context.createBuffer(2, 2048, context.sampleRate);
+    for (var i = 0; i < 2048; ++i) {
+      buffer.getChannelData(0)[i] = Math.sin(440 * 2 * Math.PI * i / context.sampleRate);
+      buffer.getChannelData(1)[i] = buffer.getChannelData(0)[i];
+    }
+    this.buffer = buffer;
+    return buffer;
+  },
+};
+
+runTest();
+
+</script>
+</pre>
+</body>
+</html>

content/media/webaudio/test/test_mediaStreamAudioSourceNodeCrossOrigin.html

@@ -0,0 +1,57 @@
+<!DOCTYPE HTML>
+<html>
+<meta charset="utf-8">
+<head>
+  <title>Test MediaStreamAudioSourceNode doesn't get data from cross-origin media resources</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+SimpleTest.waitForExplicitFinish();
+
+var audio = new Audio("http://example.org:80/tests/content/media/webaudio/test/small-shot.ogg");
+var context = new AudioContext();
+var node = context.createMediaStreamSource(audio.mozCaptureStreamUntilEnded());
+var sp = context.createScriptProcessor(2048, 1);
+node.connect(sp);
+var nonzeroSampleCount = 0;
+var complete = false;
+var iterationCount = 0;
+
+// This test ensures we receive at least expectedSampleCount nonzero samples
+function processSamples(e) {
+  if (complete) {
+    return;
+  }
+
+  if (iterationCount == 0) {
+    // Don't start playing the audio until the AudioContext stuff is connected
+    // and running.
+    audio.play();
+  }
+  ++iterationCount;
+
+  var buf = e.inputBuffer.getChannelData(0);
+  var nonzeroSamplesThisBuffer = 0;
+  for (var i = 0; i < buf.length; ++i) {
+    if (buf[i] != 0) {
+      ++nonzeroSamplesThisBuffer;
+    }
+  }
+  is(nonzeroSamplesThisBuffer, 0,
+     "Checking all samples are zero");
+  if (iterationCount >= 20) {
+    SimpleTest.finish();
+    complete = true;
+  }
+}
+
+audio.oncanplaythrough = function() {
+  sp.onaudioprocess = processSamples;
+};
+</script>
+</pre>
+</body>
+</html>

content/media/webaudio/test/test_mediaStreamAudioSourceNodeResampling.html

@@ -0,0 +1,69 @@
+<!DOCTYPE HTML>
+<html>
+<meta charset="utf-8">
+<head>
+  <title>Test MediaStreamAudioSourceNode processing is correct</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+SimpleTest.waitForExplicitFinish();
+
+var audio = new Audio("small-shot.ogg");
+var context = new AudioContext();
+var node = context.createMediaStreamSource(audio.mozCaptureStreamUntilEnded());
+var sp = context.createScriptProcessor(2048, 1);
+node.connect(sp);
+var expectedMinNonzeroSampleCount;
+var expectedMaxNonzeroSampleCount;
+var nonzeroSampleCount = 0;
+var complete = false;
+var iterationCount = 0;
+
+// This test ensures we receive at least expectedSampleCount nonzero samples
+function processSamples(e) {
+  if (complete) {
+    return;
+  }
+
+  if (iterationCount == 0) {
+    // Don't start playing the audio until the AudioContext stuff is connected
+    // and running.
+    audio.play();
+  }
+  ++iterationCount;
+
+  var buf = e.inputBuffer.getChannelData(0);
+  var nonzeroSamplesThisBuffer = 0;
+  for (var i = 0; i < buf.length; ++i) {
+    if (buf[i] != 0) {
+      ++nonzeroSamplesThisBuffer;
+    }
+  }
+  nonzeroSampleCount += nonzeroSamplesThisBuffer;
+  is(e.inputBuffer.numberOfChannels, 1,
+     "Checking data channel count (nonzeroSamplesThisBuffer=" +
+     nonzeroSamplesThisBuffer + ")");
+  ok(nonzeroSampleCount <= expectedMaxNonzeroSampleCount,
+     "Too many nonzero samples (got " + nonzeroSampleCount + ", expected max " + expectedMaxNonzeroSampleCount + ")");
+  if (nonzeroSampleCount >= expectedMinNonzeroSampleCount &&
+      nonzeroSamplesThisBuffer == 0) {
+    ok(true,
+     "Check received enough nonzero samples (got " + nonzeroSampleCount + ", expected min " + expectedMinNonzeroSampleCount + ")");
+    SimpleTest.finish();
+    complete = true;
+  }
+}
+
+audio.oncanplaythrough = function() {
+  // Use a fuzz factor of 100 to account for samples that just happen to be zero
+  expectedMinNonzeroSampleCount = Math.floor(audio.duration*context.sampleRate) - 100;
+  expectedMaxNonzeroSampleCount = Math.floor(audio.duration*context.sampleRate) + 500;
+  sp.onaudioprocess = processSamples;
+};
+</script>
+</pre>
+</body>
+</html>

content/media/webaudio/test/webaudio.js

@@ -56,7 +56,7 @@ function compareBuffers(buf1, buf2,
 
   is(difference, 0, "Found " + difference + " different samples, maxDifference: " +
      maxDifference + ", first bad index: " + firstBadIndex +
-     " with source offset " + sourceOffset + " and desitnation offset " +
+     " with source offset " + sourceOffset + " and destination offset " +
      destOffset);
 }
 
@@ -87,6 +87,8 @@ function getEmptyBuffer(context, length) {
  *                          to be silence.  The sum of the length of the expected
  *                          buffers should be equal to gTest.length.  This
  *                          function is guaranteed to be called before createGraph.
+ * + skipOfflineContextTests: optional. when true, skips running tests on an offline
+ *                            context by circumventing testOnOfflineContext.
  */
 function runTest()
 {
@@ -95,7 +97,7 @@ function runTest()
   }
 
   SimpleTest.waitForExplicitFinish();
-  addLoadEvent(function() {
+  function runTestFunction () {
     if (!gTest.numberOfChannels) {
       gTest.numberOfChannels = 2; // default
     }
@@ -177,14 +179,25 @@ function runTest()
         };
         context.startRendering();
       }
+
       var context = new OfflineAudioContext(gTest.numberOfChannels, testLength, sampleRate);
       runTestOnContext(context, callback, testOutput);
     }
 
     testOnNormalContext(function() {
-      testOnOfflineContext(function() {
-        testOnOfflineContext(done, 44100);
-      }, 48000);
+      if (!gTest.skipOfflineContextTests) {
+        testOnOfflineContext(function() {
+          testOnOfflineContext(done, 44100);
+        }, 48000);
+      } else {
+        done();
+      }
     });
-  });
+  };
+
+  if (document.readyState !== 'complete') {
+    addLoadEvent(runTestFunction);
+  } else {
+    runTestFunction();
+  }
 }

dom/webidl/AudioContext.webidl

@@ -42,6 +42,8 @@ interface AudioContext : EventTarget {
 
     [Creator]
     AnalyserNode createAnalyser();
+    [Creator, Throws]
+    MediaStreamAudioSourceNode createMediaStreamSource(MediaStream mediaStream);
     [Creator]
     GainNode createGain();
     [Creator, Throws]

dom/webidl/MediaStreamAudioSourceNode.webidl

@@ -0,0 +1,16 @@
+/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * The origin of this IDL file is
+ * https://dvcs.w3.org/hg/audio/raw-file/tip/webaudio/specification.html
+ *
+ * Copyright © 2012 W3C® (MIT, ERCIM, Keio), All Rights Reserved. W3C
+ * liability, trademark and document use rules apply.
+ */
+
+interface MediaStreamAudioSourceNode : AudioNode {
+
+};
+

dom/webidl/WebIDL.mk

@@ -181,6 +181,7 @@ webidl_files = \
   MediaSource.webidl \
   MediaStream.webidl \
   MediaStreamAudioDestinationNode.webidl \
+  MediaStreamAudioSourceNode.webidl \
   MediaStreamEvent.webidl \
   MediaStreamTrack.webidl \
   MessageEvent.webidl \