Bug 1349539 - Use CheckedInt in MediaPipeline.cpp; r=jesup
authorDan Minor <dminor@mozilla.com>
Wed, 31 May 2017 15:54:32 -0400
changeset 415621 56896524a81609ca509846235c104ef6b4c5c8d4
parent 415620 aa6b6b627a549c9ef04caf2ff04488084b84cf39
child 415622 02383934eea258ea88bf4b20a11e0dd9581a62ca
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjesup
bugs1349539
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1349539 - Use CheckedInt in MediaPipeline.cpp; r=jesup MozReview-Commit-ID: Lj9FlT3J42M
media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
@@ -95,17 +95,17 @@ public:
 
   virtual void OnVideoFrameConverted(webrtc::VideoFrame& aVideoFrame) = 0;
 
 protected:
   virtual ~VideoConverterListener() {}
 };
 
 // I420 buffer size macros
-#define YSIZE(x,y) ((x)*(y))
+#define YSIZE(x,y) (CheckedInt<int>(x)*(y))
 #define CRSIZE(x,y) ((((x)+1) >> 1) * (((y)+1) >> 1))
 #define I420SIZE(x,y) (YSIZE((x),(y)) + 2 * CRSIZE((x),(y)))
 
 // An async video frame format converter.
 //
 // Input is typically a MediaStream(Track)Listener driven by MediaStreamGraph.
 //
 // We keep track of the size of the TaskQueue so we can drop frames if
@@ -290,30 +290,36 @@ protected:
 
   void ProcessVideoFrame(Image* aImage, bool aForceBlack)
   {
     --mLength; // Atomic
     MOZ_ASSERT(mLength >= 0);
 
     if (aForceBlack) {
       IntSize size = aImage->GetSize();
-      uint32_t yPlaneLen = YSIZE(size.width, size.height);
-      uint32_t cbcrPlaneLen = 2 * CRSIZE(size.width, size.height);
-      uint32_t length = yPlaneLen + cbcrPlaneLen;
+      CheckedInt<int> yPlaneLen = YSIZE(size.width, size.height);
+      // doesn't need to be CheckedInt, any overflow will be caught by YSIZE
+      int cbcrPlaneLen = 2 * CRSIZE(size.width, size.height);
+      CheckedInt<int> length = yPlaneLen + cbcrPlaneLen;
+
+      if (!yPlaneLen.isValid() || !length.isValid()) {
+        return;
+      }
 
       // Send a black image.
-      auto pixelData = MakeUniqueFallible<uint8_t[]>(length);
+      auto pixelData = MakeUniqueFallible<uint8_t[]>(length.value());
       if (pixelData) {
         // YCrCb black = 0x10 0x80 0x80
-        memset(pixelData.get(), 0x10, yPlaneLen);
+        memset(pixelData.get(), 0x10, yPlaneLen.value());
         // Fill Cb/Cr planes
-        memset(pixelData.get() + yPlaneLen, 0x80, cbcrPlaneLen);
+        memset(pixelData.get() + yPlaneLen.value(), 0x80, cbcrPlaneLen);
 
         MOZ_MTLOG(ML_DEBUG, "Sending a black video frame");
-        VideoFrameConverted(Move(pixelData), length, size.width, size.height,
+        VideoFrameConverted(Move(pixelData), length.value(),
+                            size.width, size.height,
                             mozilla::kVideoI420, 0);
       }
       return;
     }
 
     ImageFormat format = aImage->GetFormat();
     if (format == ImageFormat::PLANAR_YCBCR) {
       // Cast away constness b/c some of the accessors are non-const
@@ -358,36 +364,42 @@ protected:
     if (!data) {
       MOZ_MTLOG(ML_ERROR, "Getting data surface from " << Stringify(format)
                           << " image with " << Stringify(surf->GetType()) << "("
                           << Stringify(surf->GetFormat()) << ") surface failed");
       return;
     }
 
     IntSize size = aImage->GetSize();
+    // these don't need to be CheckedInt, any overflow will be caught by YSIZE
     int half_width = (size.width + 1) >> 1;
     int half_height = (size.height + 1) >> 1;
     int c_size = half_width * half_height;
-    int buffer_size = YSIZE(size.width, size.height) + 2 * c_size;
-    auto yuv_scoped = MakeUniqueFallible<uint8[]>(buffer_size);
+    CheckedInt<int> buffer_size = YSIZE(size.width, size.height) + 2 * c_size;
+
+    if (!buffer_size.isValid()) {
+      return;
+    }
+
+    auto yuv_scoped = MakeUniqueFallible<uint8[]>(buffer_size.value());
     if (!yuv_scoped) {
       return;
     }
     uint8* yuv = yuv_scoped.get();
 
     DataSourceSurface::ScopedMap map(data, DataSourceSurface::READ);
     if (!map.IsMapped()) {
       MOZ_MTLOG(ML_ERROR, "Reading DataSourceSurface from " << Stringify(format)
                           << " image with " << Stringify(surf->GetType()) << "("
                           << Stringify(surf->GetFormat()) << ") surface failed");
       return;
     }
 
     int rv;
-    int cb_offset = YSIZE(size.width, size.height);
+    int cb_offset = YSIZE(size.width, size.height).value();
     int cr_offset = cb_offset + c_size;
     switch (surf->GetFormat()) {
       case SurfaceFormat::B8G8R8A8:
       case SurfaceFormat::B8G8R8X8:
         rv = libyuv::ARGBToI420(static_cast<uint8*>(map.GetData()),
                                 map.GetStride(),
                                 yuv, size.width,
                                 yuv + cb_offset, half_width,
@@ -408,17 +420,17 @@ protected:
         return;
     }
     if (rv != 0) {
       MOZ_MTLOG(ML_ERROR, Stringify(surf->GetFormat()) << " to I420 conversion failed");
       return;
     }
     MOZ_MTLOG(ML_DEBUG, "Sending an I420 video frame converted from " <<
                         Stringify(surf->GetFormat()));
-    VideoFrameConverted(Move(yuv_scoped), buffer_size, size.width, size.height, mozilla::kVideoI420, 0);
+    VideoFrameConverted(Move(yuv_scoped), buffer_size.value(), size.width, size.height, mozilla::kVideoI420, 0);
   }
 
   Atomic<int32_t, Relaxed> mLength;
   RefPtr<TaskQueue> mTaskQueue;
 
   // Written and read from the queueing thread (normally MSG).
   int32_t last_img_; // serial number of last Image
   TimeStamp disabled_frame_sent_; // The time we sent the last disabled frame.