Bug 694814: Patch 1: Add farend input to webrtc.org upstream rs=padenot
☠☠ backed out by b4cc87b3eeac ☠ ☠
authorRandell Jesup <rjesup@jesup.org>
Wed, 02 Apr 2014 13:58:19 -0400
changeset 195788 89a615263614916da84dbe1625d1e0244ad3668d
parent 195787 87f437be7de56c59093e4ba8c0104dce735a2e3e
child 195789 6922b1261595b36ce3d258d99f7f63af2c61c748
push id3624
push userasasaki@mozilla.com
push dateMon, 09 Jun 2014 21:49:01 +0000
treeherdermozilla-beta@b1a5da15899a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot
bugs694814
milestone31.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 694814: Patch 1: Add farend input to webrtc.org upstream rs=padenot
media/webrtc/trunk/webrtc/voice_engine/include/mock/fake_voe_external_media.h
media/webrtc/trunk/webrtc/voice_engine/include/voe_external_media.h
media/webrtc/trunk/webrtc/voice_engine/output_mixer.cc
media/webrtc/trunk/webrtc/voice_engine/output_mixer.h
media/webrtc/trunk/webrtc/voice_engine/voe_external_media_impl.cc
media/webrtc/trunk/webrtc/voice_engine/voe_external_media_impl.h
--- a/media/webrtc/trunk/webrtc/voice_engine/include/mock/fake_voe_external_media.h
+++ b/media/webrtc/trunk/webrtc/voice_engine/include/mock/fake_voe_external_media.h
@@ -38,16 +38,19 @@ class FakeVoEExternalMedia : public VoEE
   WEBRTC_STUB(SetExternalRecordingStatus, (bool enable));
   WEBRTC_STUB(SetExternalPlayoutStatus, (bool enable));
   WEBRTC_STUB(ExternalRecordingInsertData,
       (const int16_t speechData10ms[], int lengthSamples,
        int samplingFreqHz, int current_delay_ms));
   WEBRTC_STUB(ExternalPlayoutGetData,
       (int16_t speechData10ms[], int samplingFreqHz,
        int current_delay_ms, int& lengthSamples));
+  WEBRTC_STUB(ExternalPlayoutData,
+      (int16_t speechData10ms[], int samplingFreqHz,
+       int num_channels, int current_delay_ms, int& lengthSamples));
   WEBRTC_STUB(GetAudioFrame, (int channel, int desired_sample_rate_hz,
                               AudioFrame* frame));
   WEBRTC_STUB(SetExternalMixing, (int channel, bool enable));
 
   // Use this to trigger the Process() callback to a registered media processor.
   // If |audio| is NULL, a zero array of the correct length will be forwarded.
   void CallProcess(ProcessingTypes type, int16_t* audio,
                    int samples_per_channel, int sample_rate_hz,
--- a/media/webrtc/trunk/webrtc/voice_engine/include/voe_external_media.h
+++ b/media/webrtc/trunk/webrtc/voice_engine/include/voe_external_media.h
@@ -92,20 +92,28 @@ public:
     // This function accepts externally recorded audio. During transmission,
     // this method should be called at as regular an interval as possible
     // with frames of corresponding size.
     virtual int ExternalRecordingInsertData(
         const int16_t speechData10ms[], int lengthSamples,
         int samplingFreqHz, int current_delay_ms) = 0;
 
 
+    // This function inserts audio written to the OS audio drivers for use
+    // as the far-end signal for AEC processing.  The length of the block
+    // must be 160, 320, 441 or 480 samples (for 16000, 32000, 44100 or
+    // 48000 kHz sampling rates respectively).
+    virtual int ExternalPlayoutData(
+        int16_t speechData10ms[], int samplingFreqHz, int num_channels,
+        int current_delay_ms, int& lengthSamples) = 0;
+
     // This function gets audio for an external playout sink.
     // During transmission, this function should be called every ~10 ms
     // to obtain a new 10 ms frame of audio. The length of the block will
-    // be 160, 320, 440 or 480 samples (for 16000, 32000, 44100 or 48000
+    // be 160, 320, 441 or 480 samples (for 16000, 32000, 44100 or 48000
     // kHz sampling rates respectively).
     virtual int ExternalPlayoutGetData(
         int16_t speechData10ms[], int samplingFreqHz,
         int current_delay_ms, int& lengthSamples) = 0;
 
     // Pulls an audio frame from the specified |channel| for external mixing.
     // If the |desired_sample_rate_hz| is 0, the signal will be returned with
     // its native frequency, otherwise it will be resampled. Valid frequencies
--- a/media/webrtc/trunk/webrtc/voice_engine/output_mixer.cc
+++ b/media/webrtc/trunk/webrtc/voice_engine/output_mixer.cc
@@ -561,17 +561,17 @@ OutputMixer::DoOperationsOnCombinedSigna
         }
 
         assert(_audioFrame.num_channels_ == 2);
         AudioFrameOperations::Scale(_panLeft, _panRight, _audioFrame);
     }
 
     // --- Far-end Voice Quality Enhancement (AudioProcessing Module)
 
-    APMAnalyzeReverseStream();
+    APMAnalyzeReverseStream(_audioFrame);
 
     // --- External media processing
 
     if (_externalMedia)
     {
         CriticalSectionScoped cs(&_callbackCritSect);
         const bool isStereo = (_audioFrame.num_channels_ == 2);
         if (_externalMediaCallbackPtr)
@@ -587,35 +587,35 @@ OutputMixer::DoOperationsOnCombinedSigna
     }
 
     // --- Measure audio level (0-9) for the combined signal
     _audioLevel.ComputeLevel(_audioFrame);
 
     return 0;
 }
 
-// ----------------------------------------------------------------------------
-//                             Private methods
-// ----------------------------------------------------------------------------
-
-void OutputMixer::APMAnalyzeReverseStream() {
+void OutputMixer::APMAnalyzeReverseStream(AudioFrame &audioFrame) {
   // Convert from mixing to AudioProcessing sample rate, determined by the send
   // side. Downmix to mono.
   AudioFrame frame;
   frame.num_channels_ = 1;
   frame.sample_rate_hz_ = _audioProcessingModulePtr->sample_rate_hz();
-  if (RemixAndResample(_audioFrame, &audioproc_resampler_, &frame) == -1)
+  if (RemixAndResample(audioFrame, &audioproc_resampler_, &frame) == -1)
     return;
 
   if (_audioProcessingModulePtr->AnalyzeReverseStream(&frame) == -1) {
     WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId,-1),
                  "AudioProcessingModule::AnalyzeReverseStream() => error");
   }
 }
 
+// ----------------------------------------------------------------------------
+//                             Private methods
+// ----------------------------------------------------------------------------
+
 int
 OutputMixer::InsertInbandDtmfTone()
 {
     uint16_t sampleRate(0);
     _dtmfGenerator.GetSampleRate(sampleRate);
     if (sampleRate != _audioFrame.sample_rate_hz_)
     {
         // Update sample rate of Dtmf tone since the mixing frequency changed.
--- a/media/webrtc/trunk/webrtc/voice_engine/output_mixer.h
+++ b/media/webrtc/trunk/webrtc/voice_engine/output_mixer.h
@@ -113,19 +113,21 @@ public:
     // For file recording
     void PlayNotification(int32_t id, uint32_t durationMs);
 
     void RecordNotification(int32_t id, uint32_t durationMs);
 
     void PlayFileEnded(int32_t id);
     void RecordFileEnded(int32_t id);
 
+    // so ExternalPlayoutData() can insert far-end audio from the audio drivers
+    void APMAnalyzeReverseStream(AudioFrame &audioFrame);
+
 private:
     OutputMixer(uint32_t instanceId);
-    void APMAnalyzeReverseStream();
     int InsertInbandDtmfTone();
 
     // uses
     Statistics* _engineStatisticsPtr;
     AudioProcessing* _audioProcessingModulePtr;
 
     // owns
     CriticalSectionWrapper& _callbackCritSect;
--- a/media/webrtc/trunk/webrtc/voice_engine/voe_external_media_impl.cc
+++ b/media/webrtc/trunk/webrtc/voice_engine/voe_external_media_impl.cc
@@ -275,16 +275,78 @@ int VoEExternalMediaImpl::SetExternalPla
     return 0;
 #else
     shared_->SetLastError(VE_FUNC_NOT_SUPPORTED, kTraceError,
         "SetExternalPlayoutStatus() external playout is not supported");
     return -1;
 #endif
 }
 
+// This inserts a copy of the raw audio sent to the output drivers to use
+// as the "far end" signal for the AEC.  Currently only 10ms chunks are
+// supported unfortunately.  Since we have to rechunk to 10ms to call this,
+// thre isn't much gained by allowing N*10ms here; external code can loop
+// if needed.
+int VoEExternalMediaImpl::ExternalPlayoutData(
+    int16_t speechData10ms[],
+    int samplingFreqHz,
+    int num_channels,
+    int current_delay_ms,
+    int& lengthSamples)
+{
+    WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(shared_->instance_id(), -1),
+                 "ExternalPlayoutData(speechData10ms=0x%x,"
+                 " lengthSamples=%u, samplingFreqHz=%d, current_delay_ms=%d)",
+                 &speechData10ms[0], lengthSamples, samplingFreqHz,
+                 current_delay_ms);
+
+#ifdef WEBRTC_VOE_EXTERNAL_REC_AND_PLAYOUT
+    if (!shared_->statistics().Initialized())
+    {
+        shared_->SetLastError(VE_NOT_INITED, kTraceError);
+        return -1;
+    }
+    // FIX(jesup) - check if this is enabled?
+    if (shared_->NumOfSendingChannels() == 0)
+    {
+        shared_->SetLastError(VE_ALREADY_SENDING, kTraceError,
+            "SetExternalRecordingStatus() no channel is sending");
+        return -1;
+    }
+    if ((16000 != samplingFreqHz) && (32000 != samplingFreqHz) &&
+        (48000 != samplingFreqHz) && (44100 != samplingFreqHz))
+    {
+         shared_->SetLastError(VE_INVALID_ARGUMENT, kTraceError,
+             "SetExternalRecordingStatus() invalid sample rate");
+        return -1;
+    }
+    if (current_delay_ms < 0)
+    {
+        shared_->SetLastError(VE_INVALID_ARGUMENT, kTraceError,
+            "SetExternalRecordingStatus() invalid delay)");
+        return -1;
+    }
+
+    // Far-end data is inserted without going through neteq/etc.
+    // Only supports 10ms chunks; AnalyzeReverseStream() enforces that
+    // lower down.
+    AudioFrame audioFrame;
+    audioFrame.UpdateFrame(-1, 0xFFFFFFFF,
+                           speechData10ms,
+                           lengthSamples,
+                           samplingFreqHz,
+                           AudioFrame::kNormalSpeech,
+                           AudioFrame::kVadUnknown,
+                           num_channels);
+
+    shared_->output_mixer()->APMAnalyzeReverseStream(audioFrame);
+#endif
+    return 0;
+}
+
 int VoEExternalMediaImpl::ExternalPlayoutGetData(
     int16_t speechData10ms[],
     int samplingFreqHz,
     int current_delay_ms,
     int& lengthSamples)
 {
     WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(shared_->instance_id(), -1),
                  "ExternalPlayoutGetData(speechData10ms=0x%x, samplingFreqHz=%d"
--- a/media/webrtc/trunk/webrtc/voice_engine/voe_external_media_impl.h
+++ b/media/webrtc/trunk/webrtc/voice_engine/voe_external_media_impl.h
@@ -34,16 +34,24 @@ public:
     virtual int SetExternalPlayoutStatus(bool enable);
 
     virtual int ExternalRecordingInsertData(
         const int16_t speechData10ms[],
         int lengthSamples,
         int samplingFreqHz,
         int current_delay_ms);
 
+    // Insertion of far-end data as actually played out to the OS audio driver
+    virtual int ExternalPlayoutData(
+        int16_t speechData10ms[],
+        int samplingFreqHz,
+        int num_channels,
+        int current_delay_ms,
+        int& lengthSamples);
+
     virtual int ExternalPlayoutGetData(int16_t speechData10ms[],
                                        int samplingFreqHz,
                                        int current_delay_ms,
                                        int& lengthSamples);
 
     virtual int GetAudioFrame(int channel, int desired_sample_rate_hz,
                               AudioFrame* frame);