Bug 922247 - When an AudioContext is not running at 48kHz, resample the input of the MediaRecorder when encoding in Opus. r=roc

--HG--
extra : rebase_source : 8dd9202ed1a3e6b2b38fed78050223d1fccb74e5

content/media/encoder/OpusTrackEncoder.cpp

@@ -119,6 +119,7 @@ OpusTrackEncoder::OpusTrackEncoder()
   , mEncoder(nullptr)
   , mSourceSegment(new AudioSegment())
   , mLookahead(0)
+  , mResampler(nullptr)
 {
 }
 
@@ -147,16 +148,24 @@ OpusTrackEncoder::Init(int aChannels, int aSamplingRate)
   // The granule position is required to be incremented at a rate of 48KHz, and
   // it is simply calculated as |granulepos = samples * (48000/source_rate)|,
   // that is, the source sampling rate must divide 48000 evenly.
+  // If this constraint is not satisfied, we resample the input to 48kHz.
   if (!((aSamplingRate >= 8000) && (kOpusSamplingRate / aSamplingRate) *
          aSamplingRate == kOpusSamplingRate)) {
-    LOG("[Opus] Error! The source sample rate should be greater than 8000 and"
+    int error;
-        " divides 48000 evenly.");
+    mResampler = speex_resampler_init(mChannels,
-    return NS_ERROR_FAILURE;
+                                      aSamplingRate,
+                                      kOpusSamplingRate,
+                                      SPEEX_RESAMPLER_QUALITY_DEFAULT,
+                                      &error);
+
+    if (error != RESAMPLER_ERR_SUCCESS) {
+      return NS_ERROR_FAILURE;
+    }
   }
   mSamplingRate = aSamplingRate;
 
   int error = 0;
-  mEncoder = opus_encoder_create(mSamplingRate, mChannels,
+  mEncoder = opus_encoder_create(GetOutputSampleRate(), mChannels,
                                  OPUS_APPLICATION_AUDIO, &error);
 
   mInitialized = (error == OPUS_OK);
@@ -166,10 +175,16 @@ OpusTrackEncoder::Init(int aChannels, int aSamplingRate)
   return error == OPUS_OK ? NS_OK : NS_ERROR_FAILURE;
 }
 
+int
+OpusTrackEncoder::GetOutputSampleRate()
+{
+  return mResampler ? kOpusSamplingRate : mSamplingRate;
+}
+
 int
 OpusTrackEncoder::GetPacketDuration()
 {
-  return mSamplingRate * kFrameDurationMs / 1000;
+  return GetOutputSampleRate() * kFrameDurationMs / 1000;
 }
 
 nsresult
@@ -282,8 +297,35 @@ OpusTrackEncoder::GetEncodedTrack(nsTArray<uint8_t>* aOutput,
     iter.Next();
   }
 
-  // The ogg time stamping and pre-skip is always timed at 48000.
+  if (mResampler) {
-  aOutputDuration = frameCopied * (kOpusSamplingRate / mSamplingRate);
+    nsAutoTArray<AudioDataValue, 9600> resamplingDest;
+    // We want to consume all the input data, so we slightly oversize the
+    // resampled data buffer so we can fit the output data in. We cannot really
+    // predict the output frame count at each call.
+    uint32_t outframes = frameCopied * kOpusSamplingRate / mSamplingRate + 1;
+    uint32_t inframes = frameCopied;
+
+    resamplingDest.SetLength(outframes * mChannels);
+
+#if MOZ_SAMPLE_TYPE_S16
+    short* in = reinterpret_cast<short*>(pcm.Elements());
+    short* out = reinterpret_cast<short*>(resamplingDest.Elements());
+    speex_resampler_process_interleaved_int(mResampler, in, &inframes,
+                                                        out, &outframes);
+#else
+    float* in = reinterpret_cast<float*>(pcm.Elements());
+    float* out = reinterpret_cast<float*>(resamplingDest.Elements());
+    speex_resampler_process_interleaved_float(mResampler, in, &inframes,
+                                                          out, &outframes);
+#endif
+
+    pcm = resamplingDest;
+    // This is always at 48000Hz.
+    aOutputDuration = outframes;
+  } else {
+    // The ogg time stamping and pre-skip is always timed at 48000.
+    aOutputDuration = frameCopied * (kOpusSamplingRate / mSamplingRate);
+  }
 
   // Remove the raw data which has been pulled to pcm buffer.
   // The value of frameCopied should equal to (or smaller than, if eos)
@@ -294,6 +336,9 @@ OpusTrackEncoder::GetEncodedTrack(nsTArray<uint8_t>* aOutput,
   // encoding.
   if (mSourceSegment->GetDuration() == 0 && mEndOfStream) {
     mDoneEncoding = true;
+    if (mResampler) {
+      speex_resampler_destroy(mResampler);
+    }
     LOG("[Opus] Done encoding.");
   }
 

content/media/encoder/OpusTrackEncoder.h

@@ -6,6 +6,8 @@
 #ifndef OpusTrackEncoder_h_
 #define OpusTrackEncoder_h_
 
+#include <stdint.h>
+#include <speex/speex_resampler.h>
 #include "TrackEncoder.h"
 #include "nsCOMPtr.h"
 
@@ -35,6 +37,12 @@ private:
     DATA
   } mEncoderState;
 
+  /**
+   * Get the samplerate of the data to be fed to the Opus encoder. This might be
+   * different from the intput samplerate if resampling occurs.
+   */
+  int GetOutputSampleRate();
+
   /**
    * The Opus encoder from libopus.
    */
@@ -54,6 +62,12 @@ private:
    * in order to align the time of input and output.
    */
   int mLookahead;
+
+  /**
+   * If the input sample rate does not divide 48kHz evenly, the input data are
+   * resampled.
+   */
+  SpeexResamplerState* mResampler;
 };
 
 }