Bug 853721 - Part 2: Hook up DelayNode to the media streams graph and implement delaying of incoming audio; r=roc
authorEhsan Akhgari <ehsan@mozilla.com>
Mon, 25 Mar 2013 08:34:59 -0400
changeset 127569 5d1887ea9d43347c87f747e479da58dd2a3508cb
parent 127568 bdf9d4faddd9ccc07e36f772475598609d4e276d
child 127570 0c94dce475334e9d13a88dd8f4c45db7b26ef5d3
push id1655
push userbhackett@mozilla.com
push dateThu, 11 Apr 2013 23:17:41 +0000
reviewersroc
bugs853721
milestone23.0a1
Bug 853721 - Part 2: Hook up DelayNode to the media streams graph and implement delaying of incoming audio; r=roc
content/media/AudioNodeStream.cpp
content/media/MediaStreamGraphImpl.h
content/media/webaudio/DelayNode.cpp
content/media/webaudio/DelayNode.h
content/media/webaudio/WebAudioUtils.h
--- a/content/media/AudioNodeStream.cpp
+++ b/content/media/AudioNodeStream.cpp
@@ -186,19 +186,16 @@ AudioNodeStream::ObtainInputBlock(AudioC
   for (uint32_t i = 0; i < inputCount; ++i) {
     MediaStream* s = mInputs[i]->GetSource();
     AudioNodeStream* a = static_cast<AudioNodeStream*>(s);
     MOZ_ASSERT(a == s->AsAudioNodeStream());
     if (a->IsFinishedOnGraphThread()) {
       continue;
     }
     AudioChunk* chunk = &a->mLastChunk;
-    // XXX when we implement DelayNode, this will no longer be true and we'll
-    // need to treat a null chunk (when the DelayNode hasn't had a chance
-    // to produce data yet) as silence here.
     MOZ_ASSERT(chunk);
     if (chunk->IsNull()) {
       continue;
     }
 
     inputChunks.AppendElement(chunk);
     outputChannelCount =
       GetAudioChannelsSuperset(outputChannelCount, chunk->mChannelData.Length());
--- a/content/media/MediaStreamGraphImpl.h
+++ b/content/media/MediaStreamGraphImpl.h
@@ -248,20 +248,16 @@ public:
    */
   void RecomputeBlockingAt(const nsTArray<MediaStream*>& aStreams,
                            GraphTime aTime, GraphTime aEndBlockingDecisions,
                            GraphTime* aEnd);
   /**
    * Produce data for all streams >= aStreamIndex for the given time interval.
    * Advances block by block, each iteration producing data for all streams
    * for a single block.
-   * This is needed if there are WebAudio delay nodes, whose output for a block
-   * may depend on the output of any other node (including itself) for the
-   * previous block. This is probably also more performant due to better memory
-   * locality.
    * This is called whenever we have an AudioNodeStream in the graph.
    */
   void ProduceDataForStreamsBlockByBlock(uint32_t aStreamIndex,
                                          GraphTime aFrom,
                                          GraphTime aTo);
   /**
    * Returns true if aStream will underrun at aTime for its own playback.
    * aEndBlockingDecisions is when we plan to stop making blocking decisions.
--- a/content/media/webaudio/DelayNode.cpp
+++ b/content/media/webaudio/DelayNode.cpp
@@ -1,36 +1,228 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* vim:set ts=2 sw=2 sts=2 et cindent: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "DelayNode.h"
 #include "mozilla/dom/DelayNodeBinding.h"
+#include "AudioNodeEngine.h"
+#include "AudioNodeStream.h"
+#include "AudioDestinationNode.h"
+#include "WebAudioUtils.h"
 
 namespace mozilla {
 namespace dom {
 
 NS_IMPL_CYCLE_COLLECTION_INHERITED_1(DelayNode, AudioNode,
                                      mDelay)
 
 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(DelayNode)
 NS_INTERFACE_MAP_END_INHERITING(AudioNode)
 
 NS_IMPL_ADDREF_INHERITED(DelayNode, AudioNode)
 NS_IMPL_RELEASE_INHERITED(DelayNode, AudioNode)
 
+class DelayNodeEngine : public AudioNodeEngine
+{
+public:
+  explicit DelayNodeEngine(AudioDestinationNode* aDestination)
+    : mSource(nullptr)
+    , mDestination(static_cast<AudioNodeStream*> (aDestination->Stream()))
+    // Keep the default value in sync with the default value in DelayNode::DelayNode.
+    , mDelay(0.f)
+    , mMaxDelay(0.)
+    , mWriteIndex(0)
+    , mCurrentDelayTime(0.)
+  {
+  }
+
+  void SetSourceStream(AudioNodeStream* aSource)
+  {
+    mSource = aSource;
+  }
+
+  enum Parameters {
+    DELAY,
+    MAX_DELAY
+  };
+  void SetTimelineParameter(uint32_t aIndex, const AudioParamTimeline& aValue) MOZ_OVERRIDE
+  {
+    switch (aIndex) {
+    case DELAY:
+      MOZ_ASSERT(mSource && mDestination);
+      mDelay = aValue;
+      WebAudioUtils::ConvertAudioParamToTicks(mDelay, mSource, mDestination);
+      break;
+    default:
+      NS_ERROR("Bad DelayNodeEngine TimelineParameter");
+    }
+  }
+  void SetDoubleParameter(uint32_t aIndex, double aValue) MOZ_OVERRIDE
+  {
+    switch (aIndex) {
+    case MAX_DELAY: mMaxDelay = aValue; break;
+    default:
+      NS_ERROR("Bad DelayNodeEngine DoubleParameter");
+    }
+  }
+
+  bool EnsureBuffer(uint32_t aNumberOfChannels)
+  {
+    if (aNumberOfChannels == 0) {
+      return false;
+    }
+    if (mBuffer.Length() == 0) {
+      if (!mBuffer.SetLength(aNumberOfChannels)) {
+        return false;
+      }
+      const int32_t numFrames = NS_lround(mMaxDelay) * IdealAudioRate();
+      for (uint32_t channel = 0; channel < aNumberOfChannels; ++channel) {
+        if (!mBuffer[channel].SetLength(numFrames)) {
+          return false;
+        }
+        memset(mBuffer[channel].Elements(), 0, numFrames * sizeof(float));
+      }
+    } else if (mBuffer.Length() != aNumberOfChannels) {
+      // TODO: Handle changes in the channel count
+      return false;
+    }
+    return true;
+  }
+
+  virtual void ProduceAudioBlock(AudioNodeStream* aStream,
+                                 const AudioChunk& aInput,
+                                 AudioChunk* aOutput,
+                                 bool* aFinished)
+  {
+    MOZ_ASSERT(mSource == aStream, "Invalid source stream");
+
+    const bool firstTime = !!!mBuffer.Length();
+    const uint32_t numChannels = aInput.mChannelData.Length();
+
+    if (!EnsureBuffer(numChannels)) {
+      aOutput->SetNull(0);
+      return;
+    }
+
+    AllocateAudioBlock(numChannels, aOutput);
+
+    double delayTime = 0;
+    float computedDelay[WEBAUDIO_BLOCK_SIZE];
+    // Use a smoothing range of 20ms
+    const double smoothingRate = WebAudioUtils::ComputeSmoothingRate(0.02, IdealAudioRate());
+
+    if (mDelay.HasSimpleValue()) {
+      delayTime = std::max(0.0, std::min(mMaxDelay, double(mDelay.GetValue())));
+      if (firstTime) {
+        // Initialize this only the first time to make sure that mCurrentDelayTime
+        // has a valid value when we try to change the delay time further below.
+        mCurrentDelayTime = delayTime;
+      }
+    } else {
+      // Compute the delay values for the duration of the input AudioChunk
+      TrackTicks tick = aStream->GetCurrentPosition();
+      for (size_t counter = 0; counter < WEBAUDIO_BLOCK_SIZE; ++counter) {
+        computedDelay[counter] = std::max(0.0, std::min(mMaxDelay,
+                                   double(mDelay.GetValueAtTime<TrackTicks>(tick + counter))));
+      }
+    }
+
+    for (uint32_t channel = 0; channel < numChannels; ++channel) {
+      double currentDelayTime = mCurrentDelayTime;
+      uint32_t writeIndex = mWriteIndex;
+
+      float* buffer = mBuffer[channel].Elements();
+      const uint32_t bufferLength = mBuffer[channel].Length();
+      const float* input = static_cast<const float*>(aInput.mChannelData[channel]);
+      float* output = static_cast<float*>(const_cast<void*>(aOutput->mChannelData[channel]));
+
+      for (uint32_t i = 0; i < WEBAUDIO_BLOCK_SIZE; ++i) {
+        if (mDelay.HasSimpleValue()) {
+          // If the simple value has changed, smoothly approach it
+          currentDelayTime += (delayTime - currentDelayTime) * smoothingRate;
+        } else {
+          currentDelayTime = computedDelay[i];
+        }
+
+        // Write the input sample to the correct location in our buffer
+        buffer[writeIndex] = input[i];
+
+        // Now, determine the correct read position.  We adjust the read position to be
+        // from currentDelayTime seconds in the past.  We also interpolate the two input
+        // frames in case the read position does not match an integer index.
+        double readPosition = writeIndex + bufferLength -
+                              (currentDelayTime * IdealAudioRate());
+        if (readPosition >= bufferLength) {
+          readPosition -= bufferLength;
+        }
+        MOZ_ASSERT(readPosition >= 0.0, "Why are we reading before the beginning of the buffer?");
+
+        // Here is a the reason why readIndex1 and readIndex will never be out
+        // of bounds.  The maximum value for bufferLength is 180 * 48000 (see
+        // AudioContext::CreateDelay).  The maximum value for mCurrentDelay is
+        // 180.0, so initially readPosition cannot be more than bufferLength +
+        // a fraction less than 1.  Then we take care of that case by
+        // subtracting bufferLength from it if needed.  So, if
+        // |bufferLength-readPosition<1.0|, readIndex1 will end up being zero.
+        // If |1.0<=bufferLength-readPosition<2.0|, readIndex1 will be
+        // bufferLength-1 and readIndex2 will be 0.
+        int readIndex1 = int(readPosition);
+        int readIndex2 = (readIndex1 + 1) % bufferLength;
+        double interpolationFactor = readPosition - readIndex1;
+
+        output[i] = (1.0 - interpolationFactor) * buffer[readIndex1] +
+                           interpolationFactor  * buffer[readIndex2];
+        writeIndex = (writeIndex + 1) % bufferLength;
+      }
+
+      // Remember currentDelayTime and writeIndex for the next ProduceAudioBlock
+      // call when processing the last channel.
+      if (channel == numChannels - 1) {
+        mCurrentDelayTime = currentDelayTime;
+        mWriteIndex = writeIndex;
+      }
+    }
+  }
+
+  AudioNodeStream* mSource;
+  AudioNodeStream* mDestination;
+  AudioParamTimeline mDelay;
+  // Maximum delay time in seconds
+  double mMaxDelay;
+  // Circular buffer for capturing delayed samples.
+  AutoFallibleTArray<FallibleTArray<float>, 2> mBuffer;
+  // Write index for the buffer, to write the frames to the correct index of the buffer
+  // given the current delay.
+  uint32_t mWriteIndex;
+  // Current delay time, in seconds
+  double mCurrentDelayTime;
+};
+
 DelayNode::DelayNode(AudioContext* aContext, double aMaxDelay)
   : AudioNode(aContext)
-  , mDelay(new AudioParam(this, Callback, 0.0f, 0.0f, float(aMaxDelay)))
+  , mDelay(new AudioParam(this, SendDelayToStream, 0.0f, 0.0f, float(aMaxDelay)))
 {
+  DelayNodeEngine* engine = new DelayNodeEngine(aContext->Destination());
+  mStream = aContext->Graph()->CreateAudioNodeStream(engine, MediaStreamGraph::INTERNAL_STREAM);
+  engine->SetSourceStream(static_cast<AudioNodeStream*> (mStream.get()));
+  AudioNodeStream* ns = static_cast<AudioNodeStream*>(mStream.get());
+  ns->SetDoubleParameter(DelayNodeEngine::MAX_DELAY, aMaxDelay);
 }
 
 JSObject*
 DelayNode::WrapObject(JSContext* aCx, JSObject* aScope)
 {
   return DelayNodeBinding::Wrap(aCx, aScope, this);
 }
 
+void
+DelayNode::SendDelayToStream(AudioNode* aNode)
+{
+  DelayNode* This = static_cast<DelayNode*>(aNode);
+  SendTimelineParameterToStream(This, DelayNodeEngine::DELAY, *This->mDelay);
+}
+
 }
 }
 
--- a/content/media/webaudio/DelayNode.h
+++ b/content/media/webaudio/DelayNode.h
@@ -25,16 +25,24 @@ public:
 
   virtual JSObject* WrapObject(JSContext* aCx, JSObject* aScope);
 
   AudioParam* DelayTime() const
   {
     return mDelay;
   }
 
+  virtual bool SupportsMediaStreams() const MOZ_OVERRIDE
+  {
+    return true;
+  }
+
+private:
+  static void SendDelayToStream(AudioNode* aNode);
+
 private:
   nsRefPtr<AudioParam> mDelay;
 };
 
 }
 }
 
 #endif
--- a/content/media/webaudio/WebAudioUtils.h
+++ b/content/media/webaudio/WebAudioUtils.h
@@ -24,16 +24,25 @@ struct WebAudioUtils {
   }
   static bool FuzzyEqual(double v1, double v2)
   {
     using namespace std;
     return fabs(v1 - v2) < 1e-7;
   }
 
   /**
+   * Computes an exponential smoothing rate for a time based variable
+   * over aDuration seconds.
+   */
+  static double ComputeSmoothingRate(double aDuration, double aSampleRate)
+  {
+    return 1.0 - std::exp(-1.0 / (aDuration * aSampleRate));
+  }
+
+  /**
    * Converts AudioParamTimeline floating point time values to tick values
    * with respect to a source and a destination AudioNodeStream.
    *
    * This needs to be called for each AudioParamTimeline that gets sent to an
    * AudioNodeEngine on the engine side where the AudioParamTimeline is
    * received.  This means that such engines need to be aware of their source
    * and destination streams as well.
    */