Bug 1386957 - Uplift libcubeb fix for bug 1362764. r=padenot, a=lizzard
authorMatthew Gregan <kinetik@flim.org>
Tue, 08 Aug 2017 13:37:27 +1200
changeset 423732 00944b797230fc688374776aea588c6e48f6cc30
parent 423731 e33b936b6c2e3a26d93f045e772db9c187497ea9
child 423733 553d2a5f53a3da40f55babcd83d239939e0dda5b
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot, lizzard
bugs1386957, 1362764
milestone56.0
Bug 1386957 - Uplift libcubeb fix for bug 1362764. r=padenot, a=lizzard
media/libcubeb/bug1386957.patch
media/libcubeb/src/cubeb_wasapi.cpp
media/libcubeb/update.sh
new file mode 100644
--- /dev/null
+++ b/media/libcubeb/bug1386957.patch
@@ -0,0 +1,91 @@
+commit 9ccde3b1fa528ca42f72cf67a957665125579862
+Author: Matthew Gregan <kinetik@flim.org>
+Date:   Thu Aug 3 17:53:50 2017 +1200
+
+    wasapi: Handle GetMixFormat returning non-WAVEFORMATEXTENSIBLE pointers. (#343)
+    
+    This addresses BMO #1362764, which was detected as heap corruption resulting
+    in crashes when releasing IAudioClient or related interfaces on shutdown.
+
+diff --git a/src/cubeb_wasapi.cpp b/src/cubeb_wasapi.cpp
+index 9385e9f..5c742d9 100644
+--- a/src/cubeb_wasapi.cpp
++++ b/src/cubeb_wasapi.cpp
+@@ -451,8 +451,22 @@ channel_layout_to_mask(cubeb_channel_layout layout)
+ }
+ 
+ cubeb_channel_layout
+-mask_to_channel_layout(DWORD mask)
++mask_to_channel_layout(WAVEFORMATEX const * fmt)
+ {
++  DWORD mask = 0;
++
++  if (fmt->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {
++    WAVEFORMATEXTENSIBLE const * ext = reinterpret_cast<WAVEFORMATEXTENSIBLE const *>(fmt);
++    mask = ext->dwChannelMask;
++  } else if (fmt->wFormatTag == WAVE_FORMAT_PCM ||
++             fmt->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) {
++    if (fmt->nChannels == 1) {
++      mask = MASK_MONO;
++    } else if (fmt->nChannels == 2) {
++      mask = MASK_STEREO;
++    }
++  }
++
+   switch (mask) {
+     // MASK_DUAL_MONO(_LFE) is same as STEREO(_LFE), so we skip it.
+     case MASK_MONO: return CUBEB_LAYOUT_MONO;
+@@ -1368,8 +1382,7 @@ wasapi_get_preferred_channel_layout(cubeb * context, cubeb_channel_layout * layo
+   }
+   com_heap_ptr<WAVEFORMATEX> mix_format(tmp);
+ 
+-  WAVEFORMATEXTENSIBLE * format_pcm = reinterpret_cast<WAVEFORMATEXTENSIBLE *>(mix_format.get());
+-  *layout = mask_to_channel_layout(format_pcm->dwChannelMask);
++  *layout = mask_to_channel_layout(mix_format.get());
+ 
+   LOG("Preferred channel layout: %s", CUBEB_CHANNEL_LAYOUT_MAPS[*layout].name);
+ 
+@@ -1428,6 +1441,7 @@ handle_channel_layout(cubeb_stream * stm,  EDataFlow direction, com_heap_ptr<WAV
+        and handle the eventual upmix/downmix ourselves. Ignore the subformat of
+        the suggestion, since it seems to always be IEEE_FLOAT. */
+     LOG("Using WASAPI suggested format: channels: %d", closest->nChannels);
++    XASSERT(closest->wFormatTag == WAVE_FORMAT_EXTENSIBLE);
+     WAVEFORMATEXTENSIBLE * closest_pcm = reinterpret_cast<WAVEFORMATEXTENSIBLE *>(closest);
+     format_pcm->dwChannelMask = closest_pcm->dwChannelMask;
+     mix_format->nChannels = closest->nChannels;
+@@ -1436,6 +1450,7 @@ handle_channel_layout(cubeb_stream * stm,  EDataFlow direction, com_heap_ptr<WAV
+     /* Not supported, no suggestion. This should not happen, but it does in the
+        field with some sound cards. We restore the mix format, and let the rest
+        of the code figure out the right conversion path. */
++    XASSERT(mix_format->wFormatTag == WAVE_FORMAT_EXTENSIBLE);
+     *reinterpret_cast<WAVEFORMATEXTENSIBLE *>(mix_format.get()) = hw_mix_format;
+   } else if (hr == S_OK) {
+     LOG("Requested format accepted by WASAPI.");
+@@ -1514,21 +1529,23 @@ int setup_wasapi_stream_one_side(cubeb_stream * stm,
+   }
+   com_heap_ptr<WAVEFORMATEX> mix_format(tmp);
+ 
+-  WAVEFORMATEXTENSIBLE * format_pcm = reinterpret_cast<WAVEFORMATEXTENSIBLE *>(mix_format.get());
+   mix_format->wBitsPerSample = stm->bytes_per_sample * 8;
+-  format_pcm->SubFormat = stm->waveformatextensible_sub_format;
++  if (mix_format->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {
++    WAVEFORMATEXTENSIBLE * format_pcm = reinterpret_cast<WAVEFORMATEXTENSIBLE *>(mix_format.get());
++    format_pcm->SubFormat = stm->waveformatextensible_sub_format;
++  }
+   waveformatex_update_derived_properties(mix_format.get());
+   /* Set channel layout only when there're more than two channels. Otherwise,
+    * use the default setting retrieved from the stream format of the audio
+    * engine's internal processing by GetMixFormat. */
+   if (mix_format->nChannels > 2) {
+-    handle_channel_layout(stm, direction ,mix_format, stream_params);
++    handle_channel_layout(stm, direction, mix_format, stream_params);
+   }
+ 
+   mix_params->format = stream_params->format;
+   mix_params->rate = mix_format->nSamplesPerSec;
+   mix_params->channels = mix_format->nChannels;
+-  mix_params->layout = mask_to_channel_layout(format_pcm->dwChannelMask);
++  mix_params->layout = mask_to_channel_layout(mix_format.get());
+   if (mix_params->layout == CUBEB_LAYOUT_UNDEFINED) {
+     LOG("Output using undefined layout!\n");
+   } else if (mix_format->nChannels != CUBEB_CHANNEL_LAYOUT_MAPS[mix_params->layout].channels) {
--- a/media/libcubeb/src/cubeb_wasapi.cpp
+++ b/media/libcubeb/src/cubeb_wasapi.cpp
@@ -446,18 +446,32 @@ channel_layout_to_mask(cubeb_channel_lay
     MASK_3F2_LFE,         // CUBEB_LAYOUT_3F2_LFE
     MASK_3F3R_LFE,        // CUBEB_LAYOUT_3F3R_LFE
     MASK_3F4_LFE,         // CUBEB_LAYOUT_3F4_LFE
   };
   return map[layout];
 }
 
 cubeb_channel_layout
-mask_to_channel_layout(DWORD mask)
+mask_to_channel_layout(WAVEFORMATEX const * fmt)
 {
+  DWORD mask = 0;
+
+  if (fmt->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {
+    WAVEFORMATEXTENSIBLE const * ext = reinterpret_cast<WAVEFORMATEXTENSIBLE const *>(fmt);
+    mask = ext->dwChannelMask;
+  } else if (fmt->wFormatTag == WAVE_FORMAT_PCM ||
+             fmt->wFormatTag == WAVE_FORMAT_IEEE_FLOAT) {
+    if (fmt->nChannels == 1) {
+      mask = MASK_MONO;
+    } else if (fmt->nChannels == 2) {
+      mask = MASK_STEREO;
+    }
+  }
+
   switch (mask) {
     // MASK_DUAL_MONO(_LFE) is same as STEREO(_LFE), so we skip it.
     case MASK_MONO: return CUBEB_LAYOUT_MONO;
     case MASK_MONO_LFE: return CUBEB_LAYOUT_MONO_LFE;
     case MASK_STEREO: return CUBEB_LAYOUT_STEREO;
     case MASK_STEREO_LFE: return CUBEB_LAYOUT_STEREO_LFE;
     case MASK_3F: return CUBEB_LAYOUT_3F;
     case MASK_3F_LFE: return CUBEB_LAYOUT_3F_LFE;
@@ -1363,18 +1377,17 @@ wasapi_get_preferred_channel_layout(cube
 
   WAVEFORMATEX * tmp = nullptr;
   hr = client->GetMixFormat(&tmp);
   if (FAILED(hr)) {
     return CUBEB_ERROR;
   }
   com_heap_ptr<WAVEFORMATEX> mix_format(tmp);
 
-  WAVEFORMATEXTENSIBLE * format_pcm = reinterpret_cast<WAVEFORMATEXTENSIBLE *>(mix_format.get());
-  *layout = mask_to_channel_layout(format_pcm->dwChannelMask);
+  *layout = mask_to_channel_layout(mix_format.get());
 
   LOG("Preferred channel layout: %s", CUBEB_CHANNEL_LAYOUT_MAPS[*layout].name);
 
   return CUBEB_OK;
 }
 
 void wasapi_stream_destroy(cubeb_stream * stm);
 
@@ -1423,24 +1436,26 @@ handle_channel_layout(cubeb_stream * stm
   HRESULT hr = audio_client->IsFormatSupported(AUDCLNT_SHAREMODE_SHARED,
                                                mix_format.get(),
                                                &closest);
   if (hr == S_FALSE) {
     /* Channel layout not supported, but WASAPI gives us a suggestion. Use it,
        and handle the eventual upmix/downmix ourselves. Ignore the subformat of
        the suggestion, since it seems to always be IEEE_FLOAT. */
     LOG("Using WASAPI suggested format: channels: %d", closest->nChannels);
+    XASSERT(closest->wFormatTag == WAVE_FORMAT_EXTENSIBLE);
     WAVEFORMATEXTENSIBLE * closest_pcm = reinterpret_cast<WAVEFORMATEXTENSIBLE *>(closest);
     format_pcm->dwChannelMask = closest_pcm->dwChannelMask;
     mix_format->nChannels = closest->nChannels;
     waveformatex_update_derived_properties(mix_format.get());
   } else if (hr == AUDCLNT_E_UNSUPPORTED_FORMAT) {
     /* Not supported, no suggestion. This should not happen, but it does in the
        field with some sound cards. We restore the mix format, and let the rest
        of the code figure out the right conversion path. */
+    XASSERT(mix_format->wFormatTag == WAVE_FORMAT_EXTENSIBLE);
     *reinterpret_cast<WAVEFORMATEXTENSIBLE *>(mix_format.get()) = hw_mix_format;
   } else if (hr == S_OK) {
     LOG("Requested format accepted by WASAPI.");
   } else {
     LOG("IsFormatSupported unhandled error: %lx", hr);
   }
 }
 
@@ -1509,31 +1524,33 @@ int setup_wasapi_stream_one_side(cubeb_s
   hr = audio_client->GetMixFormat(&tmp);
   if (FAILED(hr)) {
     LOG("Could not fetch current mix format from the audio"
         " client for %s: error: %lx", DIRECTION_NAME, hr);
     return CUBEB_ERROR;
   }
   com_heap_ptr<WAVEFORMATEX> mix_format(tmp);
 
-  WAVEFORMATEXTENSIBLE * format_pcm = reinterpret_cast<WAVEFORMATEXTENSIBLE *>(mix_format.get());
   mix_format->wBitsPerSample = stm->bytes_per_sample * 8;
-  format_pcm->SubFormat = stm->waveformatextensible_sub_format;
+  if (mix_format->wFormatTag == WAVE_FORMAT_EXTENSIBLE) {
+    WAVEFORMATEXTENSIBLE * format_pcm = reinterpret_cast<WAVEFORMATEXTENSIBLE *>(mix_format.get());
+    format_pcm->SubFormat = stm->waveformatextensible_sub_format;
+  }
   waveformatex_update_derived_properties(mix_format.get());
   /* Set channel layout only when there're more than two channels. Otherwise,
    * use the default setting retrieved from the stream format of the audio
    * engine's internal processing by GetMixFormat. */
   if (mix_format->nChannels > 2) {
-    handle_channel_layout(stm, direction ,mix_format, stream_params);
+    handle_channel_layout(stm, direction, mix_format, stream_params);
   }
 
   mix_params->format = stream_params->format;
   mix_params->rate = mix_format->nSamplesPerSec;
   mix_params->channels = mix_format->nChannels;
-  mix_params->layout = mask_to_channel_layout(format_pcm->dwChannelMask);
+  mix_params->layout = mask_to_channel_layout(mix_format.get());
   if (mix_params->layout == CUBEB_LAYOUT_UNDEFINED) {
     LOG("Output using undefined layout!\n");
   } else if (mix_format->nChannels != CUBEB_CHANNEL_LAYOUT_MAPS[mix_params->layout].channels) {
     // The CUBEB_CHANNEL_LAYOUT_MAPS[mix_params->layout].channels may be
     // different from the mix_params->channels. 6 channel ouput with stereo
     // layout is acceptable in Windows. If this happens, it should not downmix
     // audio according to layout.
     LOG("Channel count is different from the layout standard!\n");
--- a/media/libcubeb/update.sh
+++ b/media/libcubeb/update.sh
@@ -68,8 +68,11 @@ else
   echo "Remember to update README_MOZILLA with the version details."
 fi
 
 echo "Applying disable-assert.patch on top of $rev"
 patch -p3 < disable-assert.patch
 
 echo "Applying prefer-pulse-rust.patch on top of $rev"
 patch -p3 < prefer-pulse-rust.patch
+
+echo "Applying bug1386957.patch on top of $rev"
+patch -p1 < bug1386957.patch