Bug 853721 - Part 2: Hook up DelayNode to the media streams graph and implement delaying of incoming audio; r=roc
authorEhsan Akhgari <ehsan@mozilla.com>
Mon, 25 Mar 2013 08:34:59 -0400
changeset 139368 5d1887ea9d43347c87f747e479da58dd2a3508cb
parent 139367 bdf9d4faddd9ccc07e36f772475598609d4e276d
child 139369 0c94dce475334e9d13a88dd8f4c45db7b26ef5d3
push id350
push userbbajaj@mozilla.com
push dateMon, 29 Jul 2013 23:00:49 +0000
treeherdermozilla-release@064965b37dbd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersroc
bugs853721
milestone23.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 853721 - Part 2: Hook up DelayNode to the media streams graph and implement delaying of incoming audio; r=roc
content/media/AudioNodeStream.cpp
content/media/MediaStreamGraphImpl.h
content/media/webaudio/DelayNode.cpp
content/media/webaudio/DelayNode.h
content/media/webaudio/WebAudioUtils.h
--- a/content/media/AudioNodeStream.cpp
+++ b/content/media/AudioNodeStream.cpp
@@ -186,19 +186,16 @@ AudioNodeStream::ObtainInputBlock(AudioC
   for (uint32_t i = 0; i < inputCount; ++i) {
     MediaStream* s = mInputs[i]->GetSource();
     AudioNodeStream* a = static_cast<AudioNodeStream*>(s);
     MOZ_ASSERT(a == s->AsAudioNodeStream());
     if (a->IsFinishedOnGraphThread()) {
       continue;
     }
     AudioChunk* chunk = &a->mLastChunk;
-    // XXX when we implement DelayNode, this will no longer be true and we'll
-    // need to treat a null chunk (when the DelayNode hasn't had a chance
-    // to produce data yet) as silence here.
     MOZ_ASSERT(chunk);
     if (chunk->IsNull()) {
       continue;
     }
 
     inputChunks.AppendElement(chunk);
     outputChannelCount =
       GetAudioChannelsSuperset(outputChannelCount, chunk->mChannelData.Length());
--- a/content/media/MediaStreamGraphImpl.h
+++ b/content/media/MediaStreamGraphImpl.h
@@ -248,20 +248,16 @@ public:
    */
   void RecomputeBlockingAt(const nsTArray<MediaStream*>& aStreams,
                            GraphTime aTime, GraphTime aEndBlockingDecisions,
                            GraphTime* aEnd);
   /**
    * Produce data for all streams >= aStreamIndex for the given time interval.
    * Advances block by block, each iteration producing data for all streams
    * for a single block.
-   * This is needed if there are WebAudio delay nodes, whose output for a block
-   * may depend on the output of any other node (including itself) for the
-   * previous block. This is probably also more performant due to better memory
-   * locality.
    * This is called whenever we have an AudioNodeStream in the graph.
    */
   void ProduceDataForStreamsBlockByBlock(uint32_t aStreamIndex,
                                          GraphTime aFrom,
                                          GraphTime aTo);
   /**
    * Returns true if aStream will underrun at aTime for its own playback.
    * aEndBlockingDecisions is when we plan to stop making blocking decisions.
--- a/content/media/webaudio/DelayNode.cpp
+++ b/content/media/webaudio/DelayNode.cpp
@@ -1,36 +1,228 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* vim:set ts=2 sw=2 sts=2 et cindent: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "DelayNode.h"
 #include "mozilla/dom/DelayNodeBinding.h"
+#include "AudioNodeEngine.h"
+#include "AudioNodeStream.h"
+#include "AudioDestinationNode.h"
+#include "WebAudioUtils.h"
 
 namespace mozilla {
 namespace dom {
 
 NS_IMPL_CYCLE_COLLECTION_INHERITED_1(DelayNode, AudioNode,
                                      mDelay)
 
 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(DelayNode)
 NS_INTERFACE_MAP_END_INHERITING(AudioNode)
 
 NS_IMPL_ADDREF_INHERITED(DelayNode, AudioNode)
 NS_IMPL_RELEASE_INHERITED(DelayNode, AudioNode)
 
+class DelayNodeEngine : public AudioNodeEngine
+{
+public:
+  explicit DelayNodeEngine(AudioDestinationNode* aDestination)
+    : mSource(nullptr)
+    , mDestination(static_cast<AudioNodeStream*> (aDestination->Stream()))
+    // Keep the default value in sync with the default value in DelayNode::DelayNode.
+    , mDelay(0.f)
+    , mMaxDelay(0.)
+    , mWriteIndex(0)
+    , mCurrentDelayTime(0.)
+  {
+  }
+
+  void SetSourceStream(AudioNodeStream* aSource)
+  {
+    mSource = aSource;
+  }
+
+  enum Parameters {
+    DELAY,
+    MAX_DELAY
+  };
+  void SetTimelineParameter(uint32_t aIndex, const AudioParamTimeline& aValue) MOZ_OVERRIDE
+  {
+    switch (aIndex) {
+    case DELAY:
+      MOZ_ASSERT(mSource && mDestination);
+      mDelay = aValue;
+      WebAudioUtils::ConvertAudioParamToTicks(mDelay, mSource, mDestination);
+      break;
+    default:
+      NS_ERROR("Bad DelayNodeEngine TimelineParameter");
+    }
+  }
+  void SetDoubleParameter(uint32_t aIndex, double aValue) MOZ_OVERRIDE
+  {
+    switch (aIndex) {
+    case MAX_DELAY: mMaxDelay = aValue; break;
+    default:
+      NS_ERROR("Bad DelayNodeEngine DoubleParameter");
+    }
+  }
+
+  bool EnsureBuffer(uint32_t aNumberOfChannels)
+  {
+    if (aNumberOfChannels == 0) {
+      return false;
+    }
+    if (mBuffer.Length() == 0) {
+      if (!mBuffer.SetLength(aNumberOfChannels)) {
+        return false;
+      }
+      const int32_t numFrames = NS_lround(mMaxDelay) * IdealAudioRate();
+      for (uint32_t channel = 0; channel < aNumberOfChannels; ++channel) {
+        if (!mBuffer[channel].SetLength(numFrames)) {
+          return false;
+        }
+        memset(mBuffer[channel].Elements(), 0, numFrames * sizeof(float));
+      }
+    } else if (mBuffer.Length() != aNumberOfChannels) {
+      // TODO: Handle changes in the channel count
+      return false;
+    }
+    return true;
+  }
+
+  virtual void ProduceAudioBlock(AudioNodeStream* aStream,
+                                 const AudioChunk& aInput,
+                                 AudioChunk* aOutput,
+                                 bool* aFinished)
+  {
+    MOZ_ASSERT(mSource == aStream, "Invalid source stream");
+
+    const bool firstTime = !!!mBuffer.Length();
+    const uint32_t numChannels = aInput.mChannelData.Length();
+
+    if (!EnsureBuffer(numChannels)) {
+      aOutput->SetNull(0);
+      return;
+    }
+
+    AllocateAudioBlock(numChannels, aOutput);
+
+    double delayTime = 0;
+    float computedDelay[WEBAUDIO_BLOCK_SIZE];
+    // Use a smoothing range of 20ms
+    const double smoothingRate = WebAudioUtils::ComputeSmoothingRate(0.02, IdealAudioRate());
+
+    if (mDelay.HasSimpleValue()) {
+      delayTime = std::max(0.0, std::min(mMaxDelay, double(mDelay.GetValue())));
+      if (firstTime) {
+        // Initialize this only the first time to make sure that mCurrentDelayTime
+        // has a valid value when we try to change the delay time further below.
+        mCurrentDelayTime = delayTime;
+      }
+    } else {
+      // Compute the delay values for the duration of the input AudioChunk
+      TrackTicks tick = aStream->GetCurrentPosition();
+      for (size_t counter = 0; counter < WEBAUDIO_BLOCK_SIZE; ++counter) {
+        computedDelay[counter] = std::max(0.0, std::min(mMaxDelay,
+                                   double(mDelay.GetValueAtTime<TrackTicks>(tick + counter))));
+      }
+    }
+
+    for (uint32_t channel = 0; channel < numChannels; ++channel) {
+      double currentDelayTime = mCurrentDelayTime;
+      uint32_t writeIndex = mWriteIndex;
+
+      float* buffer = mBuffer[channel].Elements();
+      const uint32_t bufferLength = mBuffer[channel].Length();
+      const float* input = static_cast<const float*>(aInput.mChannelData[channel]);
+      float* output = static_cast<float*>(const_cast<void*>(aOutput->mChannelData[channel]));
+
+      for (uint32_t i = 0; i < WEBAUDIO_BLOCK_SIZE; ++i) {
+        if (mDelay.HasSimpleValue()) {
+          // If the simple value has changed, smoothly approach it
+          currentDelayTime += (delayTime - currentDelayTime) * smoothingRate;
+        } else {
+          currentDelayTime = computedDelay[i];
+        }
+
+        // Write the input sample to the correct location in our buffer
+        buffer[writeIndex] = input[i];
+
+        // Now, determine the correct read position.  We adjust the read position to be
+        // from currentDelayTime seconds in the past.  We also interpolate the two input
+        // frames in case the read position does not match an integer index.
+        double readPosition = writeIndex + bufferLength -
+                              (currentDelayTime * IdealAudioRate());
+        if (readPosition >= bufferLength) {
+          readPosition -= bufferLength;
+        }
+        MOZ_ASSERT(readPosition >= 0.0, "Why are we reading before the beginning of the buffer?");
+
+        // Here is a the reason why readIndex1 and readIndex will never be out
+        // of bounds.  The maximum value for bufferLength is 180 * 48000 (see
+        // AudioContext::CreateDelay).  The maximum value for mCurrentDelay is
+        // 180.0, so initially readPosition cannot be more than bufferLength +
+        // a fraction less than 1.  Then we take care of that case by
+        // subtracting bufferLength from it if needed.  So, if
+        // |bufferLength-readPosition<1.0|, readIndex1 will end up being zero.
+        // If |1.0<=bufferLength-readPosition<2.0|, readIndex1 will be
+        // bufferLength-1 and readIndex2 will be 0.
+        int readIndex1 = int(readPosition);
+        int readIndex2 = (readIndex1 + 1) % bufferLength;
+        double interpolationFactor = readPosition - readIndex1;
+
+        output[i] = (1.0 - interpolationFactor) * buffer[readIndex1] +
+                           interpolationFactor  * buffer[readIndex2];
+        writeIndex = (writeIndex + 1) % bufferLength;
+      }
+
+      // Remember currentDelayTime and writeIndex for the next ProduceAudioBlock
+      // call when processing the last channel.
+      if (channel == numChannels - 1) {
+        mCurrentDelayTime = currentDelayTime;
+        mWriteIndex = writeIndex;
+      }
+    }
+  }
+
+  AudioNodeStream* mSource;
+  AudioNodeStream* mDestination;
+  AudioParamTimeline mDelay;
+  // Maximum delay time in seconds
+  double mMaxDelay;
+  // Circular buffer for capturing delayed samples.
+  AutoFallibleTArray<FallibleTArray<float>, 2> mBuffer;
+  // Write index for the buffer, to write the frames to the correct index of the buffer
+  // given the current delay.
+  uint32_t mWriteIndex;
+  // Current delay time, in seconds
+  double mCurrentDelayTime;
+};
+
 DelayNode::DelayNode(AudioContext* aContext, double aMaxDelay)
   : AudioNode(aContext)
-  , mDelay(new AudioParam(this, Callback, 0.0f, 0.0f, float(aMaxDelay)))
+  , mDelay(new AudioParam(this, SendDelayToStream, 0.0f, 0.0f, float(aMaxDelay)))
 {
+  DelayNodeEngine* engine = new DelayNodeEngine(aContext->Destination());
+  mStream = aContext->Graph()->CreateAudioNodeStream(engine, MediaStreamGraph::INTERNAL_STREAM);
+  engine->SetSourceStream(static_cast<AudioNodeStream*> (mStream.get()));
+  AudioNodeStream* ns = static_cast<AudioNodeStream*>(mStream.get());
+  ns->SetDoubleParameter(DelayNodeEngine::MAX_DELAY, aMaxDelay);
 }
 
 JSObject*
 DelayNode::WrapObject(JSContext* aCx, JSObject* aScope)
 {
   return DelayNodeBinding::Wrap(aCx, aScope, this);
 }
 
+void
+DelayNode::SendDelayToStream(AudioNode* aNode)
+{
+  DelayNode* This = static_cast<DelayNode*>(aNode);
+  SendTimelineParameterToStream(This, DelayNodeEngine::DELAY, *This->mDelay);
+}
+
 }
 }
 
--- a/content/media/webaudio/DelayNode.h
+++ b/content/media/webaudio/DelayNode.h
@@ -25,16 +25,24 @@ public:
 
   virtual JSObject* WrapObject(JSContext* aCx, JSObject* aScope);
 
   AudioParam* DelayTime() const
   {
     return mDelay;
   }
 
+  virtual bool SupportsMediaStreams() const MOZ_OVERRIDE
+  {
+    return true;
+  }
+
+private:
+  static void SendDelayToStream(AudioNode* aNode);
+
 private:
   nsRefPtr<AudioParam> mDelay;
 };
 
 }
 }
 
 #endif
--- a/content/media/webaudio/WebAudioUtils.h
+++ b/content/media/webaudio/WebAudioUtils.h
@@ -24,16 +24,25 @@ struct WebAudioUtils {
   }
   static bool FuzzyEqual(double v1, double v2)
   {
     using namespace std;
     return fabs(v1 - v2) < 1e-7;
   }
 
   /**
+   * Computes an exponential smoothing rate for a time based variable
+   * over aDuration seconds.
+   */
+  static double ComputeSmoothingRate(double aDuration, double aSampleRate)
+  {
+    return 1.0 - std::exp(-1.0 / (aDuration * aSampleRate));
+  }
+
+  /**
    * Converts AudioParamTimeline floating point time values to tick values
    * with respect to a source and a destination AudioNodeStream.
    *
    * This needs to be called for each AudioParamTimeline that gets sent to an
    * AudioNodeEngine on the engine side where the AudioParamTimeline is
    * received.  This means that such engines need to be aware of their source
    * and destination streams as well.
    */