Bug 1444479 - P2. Make AudioConverter works with unknown layout. r=padenot
authorJean-Yves Avenard <jyavenard@mozilla.com>
Fri, 16 Mar 2018 16:45:13 +0100
changeset 463336 2ea728f4e688b877820403a452a31c4b25428120
parent 463335 63b5bd67f700c59859dba688fad8029dc359625c
child 463337 b821de55844fc0d8bf5def89110bfa55268ed166
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot
bugs1444479
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1444479 - P2. Make AudioConverter works with unknown layout. r=padenot If a channel layout is unsupported, the AudioConverter will instead just use the channel count information to leave the data as-is, only trimming extra channels, or inserting silence if needed. MozReview-Commit-ID: CXOjcSRsRwI
dom/media/AudioConfig.cpp
dom/media/AudioConfig.h
dom/media/AudioConverter.cpp
dom/media/AudioConverter.h
--- a/dom/media/AudioConfig.cpp
+++ b/dom/media/AudioConfig.cpp
@@ -241,29 +241,32 @@ AudioConfig::ChannelLayout::SMPTEDefault
     aMap >>= 1;
     i++;
   }
   return ChannelLayout(channels, layout.Elements());
 }
 
 bool
 AudioConfig::ChannelLayout::MappingTable(const ChannelLayout& aOther,
-                                         uint8_t* aMap) const
+                                         nsTArray<uint8_t>* aMap) const
 {
-  if (!IsValid() || !aOther.IsValid() ||
-      Map() != aOther.Map()) {
+  if (!IsValid() || !aOther.IsValid() || Map() != aOther.Map()) {
+    if (aMap) {
+      aMap->SetLength(0);
+    }
     return false;
   }
   if (!aMap) {
     return true;
   }
+  aMap->SetLength(Count());
   for (uint32_t i = 0; i < Count(); i++) {
     for (uint32_t j = 0; j < Count(); j++) {
       if (aOther[j] == mChannels[i]) {
-        aMap[j] = i;
+        (*aMap)[j] = i;
         break;
       }
     }
   }
   return true;
 }
 
 /**
--- a/dom/media/AudioConfig.h
+++ b/dom/media/AudioConfig.h
@@ -90,20 +90,19 @@ public:
     ChannelMap Map() const;
 
     // Calculate the mapping table from the current layout to aOther such that
     // one can easily go from one layout to the other by doing:
     // out[channel] = in[map[channel]].
     // Returns true if the reordering is possible or false otherwise.
     // If true, then aMap, if set, will be updated to contain the mapping table
     // allowing conversion from the current layout to aOther.
-    // If aMap is nullptr, then MappingTable can be used to simply determine if
+    // If aMap is empty, then MappingTable can be used to simply determine if
     // the current layout can be easily reordered to aOther.
-    // aMap must be an array of size MAX_AUDIO_CHANNELS.
-    bool MappingTable(const ChannelLayout& aOther, uint8_t* aMap = nullptr) const;
+    bool MappingTable(const ChannelLayout& aOther, nsTArray<uint8_t>* aMap = nullptr) const;
     bool IsValid() const { return mValid; }
     bool HasChannel(Channel aChannel) const
     {
       return mChannelMap & (1 << aChannel);
     }
 
     static ChannelLayout SMPTEDefault(
       const ChannelLayout& aChannelLayout);
--- a/dom/media/AudioConverter.cpp
+++ b/dom/media/AudioConverter.cpp
@@ -19,24 +19,25 @@
 
 namespace mozilla {
 
 AudioConverter::AudioConverter(const AudioConfig& aIn, const AudioConfig& aOut)
   : mIn(aIn)
   , mOut(aOut)
   , mResampler(nullptr)
 {
-  MOZ_DIAGNOSTIC_ASSERT(aIn.Format() == aOut.Format() &&
-                        aIn.Interleaved() == aOut.Interleaved(),
-                        "No format or rate conversion is supported at this stage");
-  MOZ_DIAGNOSTIC_ASSERT(aOut.Channels() <= 2 ||
-                        aIn.Channels() == aOut.Channels(),
-                        "Only down/upmixing to mono or stereo is supported at this stage");
-  MOZ_DIAGNOSTIC_ASSERT(aOut.Interleaved(), "planar audio format not supported");
-  mIn.Layout().MappingTable(mOut.Layout(), mChannelOrderMap);
+  MOZ_DIAGNOSTIC_ASSERT(
+    aIn.Format() == aOut.Format() && aIn.Interleaved() == aOut.Interleaved(),
+    "No format or rate conversion is supported at this stage");
+  MOZ_DIAGNOSTIC_ASSERT(
+    aOut.Channels() <= 2 || aIn.Channels() == aOut.Channels(),
+    "Only down/upmixing to mono or stereo is supported at this stage");
+  MOZ_DIAGNOSTIC_ASSERT(aOut.Interleaved(),
+                        "planar audio format not supported");
+  mIn.Layout().MappingTable(mOut.Layout(), &mChannelOrderMap);
   if (aIn.Rate() != aOut.Rate()) {
     RecreateResampler();
   }
 }
 
 AudioConverter::~AudioConverter()
 {
   if (mResampler) {
@@ -83,18 +84,18 @@ AudioConverter::ProcessInternal(void* aO
 // Reorder interleaved channels.
 // Can work in place (e.g aOut == aIn).
 template <class AudioDataType>
 void
 _ReOrderInterleavedChannels(AudioDataType* aOut, const AudioDataType* aIn,
                             uint32_t aFrames, uint32_t aChannels,
                             const uint8_t* aChannelOrderMap)
 {
-  MOZ_DIAGNOSTIC_ASSERT(aChannels <= MAX_AUDIO_CHANNELS);
-  AudioDataType val[MAX_AUDIO_CHANNELS];
+  MOZ_DIAGNOSTIC_ASSERT(aChannels <= AudioConfig::ChannelLayout::MAX_CHANNELS);
+  AudioDataType val[AudioConfig::ChannelLayout::MAX_CHANNELS];
   for (uint32_t i = 0; i < aFrames; i++) {
     for (uint32_t j = 0; j < aChannels; j++) {
       val[j] = aIn[aChannelOrderMap[j]];
     }
     for (uint32_t j = 0; j < aChannels; j++) {
       aOut[j] = val[j];
     }
     aOut += aChannels;
@@ -102,70 +103,115 @@ void
   }
 }
 
 void
 AudioConverter::ReOrderInterleavedChannels(void* aOut, const void* aIn,
                                            size_t aFrames) const
 {
   MOZ_DIAGNOSTIC_ASSERT(mIn.Channels() == mOut.Channels());
+  MOZ_DIAGNOSTIC_ASSERT(CanReorderAudio());
 
-  if (mOut.Channels() == 1 || mOut.Layout() == mIn.Layout()) {
-    // If channel count is 1, planar and non-planar formats are the same and
-    // there's nothing to reorder.
+  if (mChannelOrderMap.IsEmpty() || mOut.Channels() == 1 ||
+      mOut.Layout() == mIn.Layout()) {
+    // If channel count is 1, planar and non-planar formats are the same or
+    // there's nothing to reorder, or if we don't know how to re-order.
     if (aOut != aIn) {
       memmove(aOut, aIn, FramesOutToBytes(aFrames));
     }
     return;
   }
 
   uint32_t bits = AudioConfig::FormatToBits(mOut.Format());
   switch (bits) {
     case 8:
       _ReOrderInterleavedChannels((uint8_t*)aOut, (const uint8_t*)aIn,
-                                  aFrames, mIn.Channels(), mChannelOrderMap);
+                                  aFrames, mIn.Channels(), mChannelOrderMap.Elements());
       break;
     case 16:
       _ReOrderInterleavedChannels((int16_t*)aOut,(const int16_t*)aIn,
-                                  aFrames, mIn.Channels(), mChannelOrderMap);
+                                  aFrames, mIn.Channels(), mChannelOrderMap.Elements());
       break;
     default:
       MOZ_DIAGNOSTIC_ASSERT(AudioConfig::SampleSize(mOut.Format()) == 4);
       _ReOrderInterleavedChannels((int32_t*)aOut,(const int32_t*)aIn,
-                                  aFrames, mIn.Channels(), mChannelOrderMap);
+                                  aFrames, mIn.Channels(), mChannelOrderMap.Elements());
       break;
   }
 }
 
 static inline int16_t clipTo15(int32_t aX)
 {
   return aX < -32768 ? -32768 : aX <= 32767 ? aX : 32767;
 }
 
+template<typename TYPE>
+static void
+dumbUpDownMix(TYPE* aOut,
+              int32_t aOutChannels,
+              const TYPE* aIn,
+              int32_t aInChannels,
+              int32_t aFrames)
+{
+  if (aIn == aOut) {
+    return;
+  }
+  int32_t commonChannels = std::min(aInChannels, aOutChannels);
+
+  for (int32_t i = 0; i < aFrames; i++) {
+    for (int32_t j = 0; j < commonChannels; j++) {
+      aOut[i * aOutChannels + j] = aIn[i * aInChannels + j];
+    }
+    for (int32_t j = 0; j < aInChannels - aOutChannels; j++) {
+      aOut[i * aOutChannels + j] = 0;
+    }
+  }
+}
+
 size_t
 AudioConverter::DownmixAudio(void* aOut, const void* aIn, size_t aFrames) const
 {
   MOZ_ASSERT(mIn.Format() == AudioConfig::FORMAT_S16 ||
              mIn.Format() == AudioConfig::FORMAT_FLT);
   MOZ_ASSERT(mIn.Channels() >= mOut.Channels());
-  MOZ_ASSERT(mIn.Layout() ==
-             AudioConfig::ChannelLayout::SMPTEDefault(mIn.Layout()),
-             "Can only downmix input data in SMPTE layout");
   MOZ_ASSERT(mOut.Layout() == AudioConfig::ChannelLayout(2) ||
              mOut.Layout() == AudioConfig::ChannelLayout(1));
 
   uint32_t channels = mIn.Channels();
 
   if (channels == 1 && mOut.Channels() == 1) {
     if (aOut != aIn) {
       memmove(aOut, aIn, FramesOutToBytes(aFrames));
     }
     return aFrames;
   }
 
+  if (!mIn.Layout().IsValid() || !mOut.Layout().IsValid()) {
+    // Dumb copy dropping extra channels.
+    if (mIn.Format() == AudioConfig::FORMAT_FLT) {
+      dumbUpDownMix(static_cast<float*>(aOut),
+                    mOut.Channels(),
+                    static_cast<const float*>(aIn),
+                    mIn.Channels(),
+                    aFrames);
+    } else if (mIn.Format() == AudioConfig::FORMAT_S16) {
+      dumbUpDownMix(static_cast<int16_t*>(aOut),
+                    mOut.Channels(),
+                    static_cast<const int16_t*>(aIn),
+                    mIn.Channels(),
+                    aFrames);
+    } else {
+      MOZ_DIAGNOSTIC_ASSERT(false, "Unsupported data type");
+    }
+    return aFrames;
+  }
+
+  MOZ_ASSERT(mIn.Layout() ==
+             AudioConfig::ChannelLayout::SMPTEDefault(mIn.Layout()),
+             "Can only downmix input data in SMPTE layout");
   if (channels > 2) {
     if (mIn.Format() == AudioConfig::FORMAT_FLT) {
       // Downmix matrix. Per-row normalization 1 for rows 3,4 and 2 for rows 5-8.
       static const float dmatrix[6][8][2]= {
           /*3*/{{0.5858f,0},{0,0.5858f},{0.4142f,0.4142f}},
           /*4*/{{0.4226f,0},{0,0.4226f},{0.366f, 0.2114f},{0.2114f,0.366f}},
           /*5*/{{0.6510f,0},{0,0.6510f},{0.4600f,0.4600f},{0.5636f,0.3254f},{0.3254f,0.5636f}},
           /*6*/{{0.5290f,0},{0,0.5290f},{0.3741f,0.3741f},{0.3741f,0.3741f},{0.4582f,0.2645f},{0.2645f,0.4582f}},
@@ -324,18 +370,35 @@ size_t
 AudioConverter::UpmixAudio(void* aOut, const void* aIn, size_t aFrames) const
 {
   MOZ_ASSERT(mIn.Format() == AudioConfig::FORMAT_S16 ||
              mIn.Format() == AudioConfig::FORMAT_FLT);
   MOZ_ASSERT(mIn.Channels() < mOut.Channels());
   MOZ_ASSERT(mIn.Channels() == 1, "Can only upmix mono for now");
   MOZ_ASSERT(mOut.Channels() == 2, "Can only upmix to stereo for now");
 
-  if (mOut.Channels() != 2) {
-    return 0;
+  if (!mIn.Layout().IsValid() || !mOut.Layout().IsValid() ||
+      mOut.Channels() != 2) {
+    // Dumb copy the channels and insert silence for the extra channels.
+    if (mIn.Format() == AudioConfig::FORMAT_FLT) {
+      dumbUpDownMix(static_cast<float*>(aOut),
+                    mOut.Channels(),
+                    static_cast<const float*>(aIn),
+                    mIn.Channels(),
+                    aFrames);
+    } else if (mIn.Format() == AudioConfig::FORMAT_S16) {
+      dumbUpDownMix(static_cast<int16_t*>(aOut),
+                    mOut.Channels(),
+                    static_cast<const int16_t*>(aIn),
+                    mIn.Channels(),
+                    aFrames);
+    } else {
+      MOZ_DIAGNOSTIC_ASSERT(false, "Unsupported data type");
+    }
+    return aFrames;
   }
 
   // Upmix mono to stereo.
   // This is a very dumb mono to stereo upmixing, power levels are preserved
   // following the calculation: left = right = -3dB*mono.
   if (mIn.Format() == AudioConfig::FORMAT_FLT) {
     const float m3db = std::sqrt(0.5); // -3dB = sqrt(1/2)
     const float* in = static_cast<const float*>(aIn);
--- a/dom/media/AudioConverter.h
+++ b/dom/media/AudioConverter.h
@@ -230,17 +230,19 @@ public:
   }
 
   const AudioConfig& InputConfig() const { return mIn; }
   const AudioConfig& OutputConfig() const { return mOut; }
 
 private:
   const AudioConfig mIn;
   const AudioConfig mOut;
-  uint8_t mChannelOrderMap[MAX_AUDIO_CHANNELS];
+  // mChannelOrderMap will be empty if we do not know how to proceed with this
+  // channel layout.
+  AutoTArray<uint8_t, AudioConfig::ChannelLayout::MAX_CHANNELS> mChannelOrderMap;
   /**
    * ProcessInternal
    * Parameters:
    * aOut  : destination buffer where converted samples will be copied
    * aIn   : source buffer
    * aSamples: number of frames in source buffer
    *
    * Return Value: number of frames converted or 0 if error