Bug 1451297. r=nical, a=RyanVM
authorAndrew Osmond <aosmond@mozilla.com>
Tue, 22 May 2018 11:25:49 -0400
changeset 473463 a341ae7a49c05df8fd382e619f1ceb8958dffba9
parent 473462 5ff34e06b8e60b5a9b31dd0f06023db0538fc905
child 473464 e796c301b3ba87ea1a36bbc2f18d2bb442405fae
push id1728
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:12:27 +0000
treeherdermozilla-release@c296fde26f5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnical, RyanVM
bugs1451297
milestone61.0
Bug 1451297. r=nical, a=RyanVM
gfx/layers/ImageDataSerializer.cpp
gfx/layers/ImageDataSerializer.h
gfx/layers/composite/TextureHost.cpp
--- a/gfx/layers/ImageDataSerializer.cpp
+++ b/gfx/layers/ImageDataSerializer.cpp
@@ -74,16 +74,51 @@ ComputeYCbCrBufferSize(const gfx::IntSiz
   }
 
   // Overflow checks are performed in AllowedSurfaceSize
   return GetAlignedStride<4>(aYSize.height, aYStride) +
          2 * GetAlignedStride<4>(aCbCrSize.height, aCbCrStride);
 }
 
 uint32_t
+ComputeYCbCrBufferSize(const gfx::IntSize& aYSize, int32_t aYStride,
+                       const gfx::IntSize& aCbCrSize, int32_t aCbCrStride,
+                       uint32_t aYOffset, uint32_t aCbOffset,
+                       uint32_t aCrOffset)
+{
+  MOZ_ASSERT(aYSize.height >= 0 && aYSize.width >= 0);
+
+  if (aYSize.height < 0 || aYSize.width < 0 || aCbCrSize.height < 0 || aCbCrSize.width < 0 ||
+      !gfx::Factory::AllowedSurfaceSize(IntSize(aYStride, aYSize.height)) ||
+      !gfx::Factory::AllowedSurfaceSize(IntSize(aCbCrStride, aCbCrSize.height))) {
+    return 0;
+  }
+
+  uint32_t yLength = GetAlignedStride<4>(aYStride, aYSize.height);
+  uint32_t cbCrLength = GetAlignedStride<4>(aCbCrStride, aCbCrSize.height);
+  if (yLength == 0 || cbCrLength == 0) {
+    return 0;
+  }
+
+  CheckedInt<uint32_t> yEnd = aYOffset;
+  yEnd += yLength;
+  CheckedInt<uint32_t> cbEnd = aCbOffset;
+  cbEnd += cbCrLength;
+  CheckedInt<uint32_t> crEnd = aCrOffset;
+  crEnd += cbCrLength;
+
+  if (!yEnd.isValid() || !cbEnd.isValid() || !crEnd.isValid() ||
+      yEnd.value() > aCbOffset || cbEnd.value() > aCrOffset) {
+    return 0;
+  }
+
+  return crEnd.value();
+}
+
+uint32_t
 ComputeYCbCrBufferSize(uint32_t aBufferSize)
 {
   return GetAlignedStride<4>(aBufferSize, 1);
 }
 
 void ComputeYCbCrOffsets(int32_t yStride, int32_t yHeight,
                          int32_t cbCrStride, int32_t cbCrHeight,
                          uint32_t& outYOffset, uint32_t& outCbOffset,
--- a/gfx/layers/ImageDataSerializer.h
+++ b/gfx/layers/ImageDataSerializer.h
@@ -39,16 +39,23 @@ uint32_t ComputeRGBBufferSize(gfx::IntSi
 
 ///This function is meant as a helper to know how much shared memory we need
 ///to allocate in a shmem in order to place a shared YCbCr image blob of
 ///given dimensions.
 uint32_t ComputeYCbCrBufferSize(const gfx::IntSize& aYSize,
                                 int32_t aYStride,
                                 const gfx::IntSize& aCbCrSize,
                                 int32_t aCbCrStride);
+uint32_t ComputeYCbCrBufferSize(const gfx::IntSize& aYSize,
+                                int32_t aYStride,
+                                const gfx::IntSize& aCbCrSize,
+                                int32_t aCbCrStride,
+                                uint32_t aYOffset,
+                                uint32_t aCbOffset,
+                                uint32_t aCrOffset);
 uint32_t ComputeYCbCrBufferSize(uint32_t aBufferSize);
 
 void ComputeYCbCrOffsets(int32_t yStride, int32_t yHeight,
                          int32_t cbCrStride, int32_t cbCrHeight,
                          uint32_t& outYOffset, uint32_t& outCbOffset, uint32_t& outCrOffset);
 
 gfx::SurfaceFormat FormatFromBufferDescriptor(const BufferDescriptor& aDescriptor);
 
--- a/gfx/layers/composite/TextureHost.cpp
+++ b/gfx/layers/composite/TextureHost.cpp
@@ -287,30 +287,32 @@ CreateBackendIndependentTextureHost(cons
 
           size_t bufSize = shmem.Size<char>();
           size_t reqSize = SIZE_MAX;
           switch (desc.type()) {
             case BufferDescriptor::TYCbCrDescriptor: {
               const YCbCrDescriptor& ycbcr = desc.get_YCbCrDescriptor();
               reqSize =
                 ImageDataSerializer::ComputeYCbCrBufferSize(ycbcr.ySize(), ycbcr.yStride(),
-                                                            ycbcr.cbCrSize(), ycbcr.cbCrStride());
+                                                            ycbcr.cbCrSize(), ycbcr.cbCrStride(),
+                                                            ycbcr.yOffset(), ycbcr.cbOffset(),
+                                                            ycbcr.crOffset());
               break;
             }
             case BufferDescriptor::TRGBDescriptor: {
               const RGBDescriptor& rgb = desc.get_RGBDescriptor();
               reqSize = ImageDataSerializer::ComputeRGBBufferSize(rgb.size(), rgb.format());
               break;
             }
             default:
               gfxCriticalError() << "Bad buffer host descriptor " << (int)desc.type();
               MOZ_CRASH("GFX: Bad descriptor");
           }
 
-          if (bufSize < reqSize) {
+          if (reqSize == 0 || bufSize < reqSize) {
             NS_ERROR("A client process gave a shmem too small to fit for its descriptor!");
             return nullptr;
           }
 
           result = new ShmemTextureHost(shmem, desc, aDeallocator, aFlags);
           break;
         }
         case MemoryOrShmem::Tuintptr_t: {