Fix crash when ancestor masks have no overlapping area. (bug 1381479, r=bas)
authorDavid Anderson <danderson@mozilla.com>
Thu, 20 Jul 2017 15:10:40 -0700
changeset 418716 5df4ba09e140e576b9045e9ee7dc8b53f30d17c4
parent 418715 ea67db1628e0ad1f96bd67fc9f28197cecb71561
child 418717 a4c46cc48c7836ce767ab8897deea0299f61d79c
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbas
bugs1381479
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Fix crash when ancestor masks have no overlapping area. (bug 1381479, r=bas)
gfx/layers/mlgpu/LayerMLGPU.cpp
gfx/layers/mlgpu/MaskOperation.cpp
gfx/layers/mlgpu/MaskOperation.h
--- a/gfx/layers/mlgpu/LayerMLGPU.cpp
+++ b/gfx/layers/mlgpu/LayerMLGPU.cpp
@@ -46,28 +46,33 @@ LayerMLGPU::PrepareToRender(FrameBuilder
 
   Layer* layer = GetLayer();
 
   // Only container layers may have mixed blend modes.
   MOZ_ASSERT_IF(layer->GetMixBlendMode() != CompositionOp::OP_OVER,
                 layer->GetType() == Layer::TYPE_CONTAINER);
 
   mComputedClipRect = aClipRect;
+  mComputedOpacity = layer->GetEffectiveOpacity();
 
   if (layer->HasMaskLayers()) {
     mMask = aBuilder->AddMaskOperation(this);
+    // If the mask has no texture, the pixel shader can't read any non-zero
+    // values for the mask, so we can consider the whole thing invisible.
+    if (mMask && mMask->IsEmpty()) {
+      mComputedOpacity = 0.0f;
+    }
   } else {
     mMask = nullptr;
   }
 
   if (!OnPrepareToRender(aBuilder)) {
     return false;
   }
 
-  mComputedOpacity = layer->GetEffectiveOpacity();
   mPrepared = true;
   return true;
 }
 
 void
 LayerMLGPU::AssignToView(FrameBuilder* aBuilder,
                          RenderViewMLGPU* aView,
                          Maybe<gfx::Polygon>&& aGeometry)
--- a/gfx/layers/mlgpu/MaskOperation.cpp
+++ b/gfx/layers/mlgpu/MaskOperation.cpp
@@ -118,19 +118,25 @@ MaskCombineOperation::Init(const MaskTex
     mTextures.push_back(MaskTexture(rect, aTextures[i].mSource));
   }
 
   IntRect size;
   Rect bounds = area;
   bounds.RoundOut();
   bounds.ToIntRect(&size);
 
+  if (size.IsEmpty()) {
+    return;
+  }
+
   mTarget = mBuilder->GetDevice()->CreateRenderTarget(size.Size());
+  if (mTarget) {
+    mTexture = mTarget->GetTexture();
+  }
   mArea = area;
-  mTexture = mTarget->GetTexture();
 }
 
 void
 MaskCombineOperation::PrepareForRendering()
 {
   for (const auto& entry : mTextures) {
     Rect texCoords = TextureRectToCoords(entry.mRect, entry.mSource->GetSize());
 
@@ -142,16 +148,20 @@ MaskCombineOperation::PrepareForRenderin
     }
     mInputBuffers.push_back(section);
   }
 }
 
 void
 MaskCombineOperation::Render()
 {
+  if (!mTarget) {
+    return;
+  }
+
   RefPtr<MLGDevice> device = mBuilder->GetDevice();
 
   device->SetTopology(MLGPrimitiveTopology::UnitQuad);
   device->SetVertexShader(VertexShaderID::MaskCombiner);
 
   device->SetPixelShader(PixelShaderID::MaskCombiner);
   device->SetSamplerMode(0, SamplerMode::LinearClamp);
   device->SetBlendState(MLGBlendState::Min);
--- a/gfx/layers/mlgpu/MaskOperation.h
+++ b/gfx/layers/mlgpu/MaskOperation.h
@@ -33,16 +33,19 @@ public:
   // layer because a single-texture mask operation is not dependent on a
   // specific mask transform. (Multiple mask layer operations are, and they
   // ignore the layer parameter).
   virtual gfx::Rect ComputeMaskRect(Layer* aLayer) const;
 
   MLGTexture* GetTexture() const {
     return mTexture;
   }
+  bool IsEmpty() const {
+    return !mTexture;
+  }
 
 protected:
   explicit MaskOperation(FrameBuilder* aBuilder);
   virtual ~MaskOperation();
 
 protected:
   RefPtr<MLGTexture> mTexture;
 };