Bug 1266260: P1. Fix overflow is speexdsp. r=padenot
authorJean-Yves Avenard <jyavenard@mozilla.com>
Wed, 11 May 2016 12:15:13 +0200
changeset 296936 afe186de263fa4ddd827c5b9aa6520cc23a0042b
parent 296935 cb3ec6e11f93167c8d1f49e8e9e6bf9fa403bfd8
child 296937 b2ed54d8bba5e336a21bfc3237b07226314bcd7d
push id76530
push usercbook@mozilla.com
push dateWed, 11 May 2016 10:17:12 +0000
treeherdermozilla-inbound@9a00b5f6a6dc [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot
bugs1266260
milestone49.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1266260: P1. Fix overflow is speexdsp. r=padenot Fix submitted upstream Upstream version had a partial fix for overflow (though incomplete). I've included that fix here and completed it. MozReview-Commit-ID: 5RyIMI5qrvC
media/libspeex_resampler/fix-overflow.patch
media/libspeex_resampler/src/resample.c
media/libspeex_resampler/src/speex_resampler.h
media/libspeex_resampler/update.sh
new file mode 100644
--- /dev/null
+++ b/media/libspeex_resampler/fix-overflow.patch
@@ -0,0 +1,87 @@
+diff --git a/media/libspeex_resampler/fix-overflow.patch b/media/libspeex_resampler/fix-overflow.patch
+new file mode 100644
+index 0000000..e69de29
+diff --git a/media/libspeex_resampler/src/resample.c b/media/libspeex_resampler/src/resample.c
+index a3859e3..d99595a 100644
+--- a/media/libspeex_resampler/src/resample.c
++++ b/media/libspeex_resampler/src/resample.c
+@@ -98,6 +98,10 @@ static void speex_free (void *ptr) {free(ptr);}
+ #define NULL 0
+ #endif
+ 
++#ifndef UINT32_MAX
++#define UINT32_MAX 4294967296U
++#endif
++
+ #include "simd_detect.h"
+ 
+ /* Numer of elements to allocate on the stack */
+@@ -603,6 +607,22 @@ static int resampler_basic_zero(SpeexResamplerState *st, spx_uint32_t channel_in
+    return out_sample;
+ }
+ 
++static int _muldiv_safe(spx_uint32_t value, spx_uint32_t mul, spx_uint32_t div)
++{
++  /* TODO: Could be simplified with 64 bits operation. */
++  spx_uint32_t major = value / div;
++  spx_uint32_t remainder = value % div;
++  return remainder <= UINT32_MAX / mul && major <= UINT32_MAX / mul &&
++         major * mul <= UINT32_MAX - remainder * mul / div;
++}
++
++static spx_uint32_t _muldiv(spx_uint32_t value, spx_uint32_t mul, spx_uint32_t div)
++{
++  spx_uint32_t major = value / div;
++  spx_uint32_t remainder = value % div;
++  return remainder * mul / div + major * mul;
++}
++
+ static int update_filter(SpeexResamplerState *st)
+ {
+    spx_uint32_t old_length = st->filt_len;
+@@ -620,8 +640,9 @@ static int update_filter(SpeexResamplerState *st)
+    {
+       /* down-sampling */
+       st->cutoff = quality_map[st->quality].downsample_bandwidth * st->den_rate / st->num_rate;
+-      /* FIXME: divide the numerator and denominator by a certain amount if they're too large */
+-      st->filt_len = st->filt_len*st->num_rate / st->den_rate;
++      if (!_muldiv_safe(st->filt_len,st->num_rate,st->den_rate))
++         goto fail;
++      st->filt_len = _muldiv(st->filt_len,st->num_rate,st->den_rate);
+       /* Round up to make sure we have a multiple of 8 for SSE */
+       st->filt_len = ((st->filt_len-1)&(~0x7))+8;
+       if (2*st->den_rate < st->num_rate)
+@@ -1129,7 +1150,9 @@ EXPORT int speex_resampler_set_rate_frac(SpeexResamplerState *st, spx_uint32_t r
+    {
+       for (i=0;i<st->nb_channels;i++)
+       {
+-         st->samp_frac_num[i]=st->samp_frac_num[i]*st->den_rate/old_den;
++         if (!_muldiv_safe(st->samp_frac_num[i],st->den_rate,old_den))
++            return RESAMPLER_ERR_OVERFLOW;
++         st->samp_frac_num[i]= _muldiv(st->samp_frac_num[i],st->den_rate,old_den);
+          /* Safety net */
+          if (st->samp_frac_num[i] >= st->den_rate)
+             st->samp_frac_num[i] = st->den_rate-1;
+diff --git a/media/libspeex_resampler/src/speex_resampler.h b/media/libspeex_resampler/src/speex_resampler.h
+index 70abe52..1286872 100644
+--- a/media/libspeex_resampler/src/speex_resampler.h
++++ b/media/libspeex_resampler/src/speex_resampler.h
+@@ -106,7 +106,8 @@ enum {
+    RESAMPLER_ERR_BAD_STATE       = 2,
+    RESAMPLER_ERR_INVALID_ARG     = 3,
+    RESAMPLER_ERR_PTR_OVERLAP     = 4,
+-   
++   RESAMPLER_ERR_OVERFLOW        = 5,
++
+    RESAMPLER_ERR_MAX_ERROR
+ };
+ 
+diff --git a/media/libspeex_resampler/update.sh b/media/libspeex_resampler/update.sh
+index d4a025b..6950bc6 100644
+--- a/media/libspeex_resampler/update.sh
++++ b/media/libspeex_resampler/update.sh
+@@ -26,3 +26,4 @@ patch -p3 < set-skip-frac.patch
+ patch -p3 < hugemem.patch
+ patch -p3 < remove-empty-asm-clobber.patch
+ patch -p3 < handle-memory-error.patch
++patch -p3 < fix-overflow.patch
--- a/media/libspeex_resampler/src/resample.c
+++ b/media/libspeex_resampler/src/resample.c
@@ -93,16 +93,20 @@ static void speex_free (void *ptr) {free
                
 #define IMAX(a,b) ((a) > (b) ? (a) : (b))
 #define IMIN(a,b) ((a) < (b) ? (a) : (b))
 
 #ifndef NULL
 #define NULL 0
 #endif
 
+#ifndef UINT32_MAX
+#define UINT32_MAX 4294967296U
+#endif
+
 #include "simd_detect.h"
 
 /* Numer of elements to allocate on the stack */
 #ifdef VAR_ARRAYS
 #define FIXED_STACK_ALLOC 8192
 #else
 #define FIXED_STACK_ALLOC 1024
 #endif
@@ -598,16 +602,32 @@ static int resampler_basic_zero(SpeexRes
       }
    }
 
    st->last_sample[channel_index] = last_sample;
    st->samp_frac_num[channel_index] = samp_frac_num;
    return out_sample;
 }
 
+static int _muldiv_safe(spx_uint32_t value, spx_uint32_t mul, spx_uint32_t div)
+{
+  /* TODO: Could be simplified with 64 bits operation. */
+  spx_uint32_t major = value / div;
+  spx_uint32_t remainder = value % div;
+  return remainder <= UINT32_MAX / mul && major <= UINT32_MAX / mul &&
+         major * mul <= UINT32_MAX - remainder * mul / div;
+}
+
+static spx_uint32_t _muldiv(spx_uint32_t value, spx_uint32_t mul, spx_uint32_t div)
+{
+  spx_uint32_t major = value / div;
+  spx_uint32_t remainder = value % div;
+  return remainder * mul / div + major * mul;
+}
+
 static int update_filter(SpeexResamplerState *st)
 {
    spx_uint32_t old_length = st->filt_len;
    spx_uint32_t old_alloc_size = st->mem_alloc_size;
    int use_direct;
    spx_uint32_t min_sinc_table_length;
    spx_uint32_t min_alloc_size;
 
@@ -615,18 +635,19 @@ static int update_filter(SpeexResamplerS
    st->frac_advance = st->num_rate%st->den_rate;
    st->oversample = quality_map[st->quality].oversample;
    st->filt_len = quality_map[st->quality].base_length;
    
    if (st->num_rate > st->den_rate)
    {
       /* down-sampling */
       st->cutoff = quality_map[st->quality].downsample_bandwidth * st->den_rate / st->num_rate;
-      /* FIXME: divide the numerator and denominator by a certain amount if they're too large */
-      st->filt_len = st->filt_len*st->num_rate / st->den_rate;
+      if (!_muldiv_safe(st->filt_len,st->num_rate,st->den_rate))
+         goto fail;
+      st->filt_len = _muldiv(st->filt_len,st->num_rate,st->den_rate);
       /* Round up to make sure we have a multiple of 8 for SSE */
       st->filt_len = ((st->filt_len-1)&(~0x7))+8;
       if (2*st->den_rate < st->num_rate)
          st->oversample >>= 1;
       if (4*st->den_rate < st->num_rate)
          st->oversample >>= 1;
       if (8*st->den_rate < st->num_rate)
          st->oversample >>= 1;
@@ -1124,17 +1145,19 @@ EXPORT int speex_resampler_set_rate_frac
          st->den_rate /= fact;
       }
    }
       
    if (old_den > 0)
    {
       for (i=0;i<st->nb_channels;i++)
       {
-         st->samp_frac_num[i]=st->samp_frac_num[i]*st->den_rate/old_den;
+         if (!_muldiv_safe(st->samp_frac_num[i],st->den_rate,old_den))
+            return RESAMPLER_ERR_OVERFLOW;
+         st->samp_frac_num[i]= _muldiv(st->samp_frac_num[i],st->den_rate,old_den);
          /* Safety net */
          if (st->samp_frac_num[i] >= st->den_rate)
             st->samp_frac_num[i] = st->den_rate-1;
       }
    }
    
    if (st->initialised)
       return update_filter(st);
--- a/media/libspeex_resampler/src/speex_resampler.h
+++ b/media/libspeex_resampler/src/speex_resampler.h
@@ -101,17 +101,18 @@ extern "C" {
 #define SPEEX_RESAMPLER_QUALITY_DESKTOP 5
 
 enum {
    RESAMPLER_ERR_SUCCESS         = 0,
    RESAMPLER_ERR_ALLOC_FAILED    = 1,
    RESAMPLER_ERR_BAD_STATE       = 2,
    RESAMPLER_ERR_INVALID_ARG     = 3,
    RESAMPLER_ERR_PTR_OVERLAP     = 4,
-   
+   RESAMPLER_ERR_OVERFLOW        = 5,
+
    RESAMPLER_ERR_MAX_ERROR
 };
 
 struct SpeexResamplerState_;
 typedef struct SpeexResamplerState_ SpeexResamplerState;
 
 /** Create a new resampler with integer input and output rates.
  * @param nb_channels Number of channels to be processed
--- a/media/libspeex_resampler/update.sh
+++ b/media/libspeex_resampler/update.sh
@@ -21,8 +21,9 @@ cp $1/COPYING .
 
 # apply outstanding local patches
 patch -p3 < outside-speex.patch
 patch -p3 < simd-detect-runtime.patch
 patch -p3 < set-skip-frac.patch
 patch -p3 < hugemem.patch
 patch -p3 < remove-empty-asm-clobber.patch
 patch -p3 < handle-memory-error.patch
+patch -p3 < fix-overflow.patch