Bug 1148582 - Support multiple mask layers per layer in LayerManagerComposite.
authorMarkus Stange <mstange@themasta.com>
Wed, 01 Jul 2015 17:32:19 -0400
changeset 498089 4ffc0867750385059fa71b6e44da3487a99b20fb
parent 498088 9c4ed0cac6e8910dfbf9819f8df32581a9eac73e
child 498090 0a615031704f9f9f98e32aab7493959bc0f08c4b
child 498762 98012ca45674f995c01b87f94bc95f8591aecebb
push id76597
push usermstange@themasta.com
push dateThu, 02 Jul 2015 00:09:18 +0000
treeherdertry@0a615031704f [default view] [failures only]
bugs1148582
milestone42.0a1
Bug 1148582 - Support multiple mask layers per layer in LayerManagerComposite.
gfx/layers/composite/CanvasLayerComposite.cpp
gfx/layers/composite/ColorLayerComposite.cpp
gfx/layers/composite/ContainerLayerComposite.cpp
gfx/layers/composite/ImageLayerComposite.cpp
gfx/layers/composite/LayerManagerComposite.h
gfx/layers/composite/PaintedLayerComposite.cpp
--- a/gfx/layers/composite/CanvasLayerComposite.cpp
+++ b/gfx/layers/composite/CanvasLayerComposite.cpp
@@ -90,27 +90,25 @@ CanvasLayerComposite::RenderLayer(const 
 
 #ifdef MOZ_DUMP_PAINTING
   if (gfxUtils::sDumpPainting) {
     RefPtr<gfx::DataSourceSurface> surf = mCompositableHost->GetAsSurface();
     WriteSnapshotToDumpFile(this, surf);
   }
 #endif
 
-  EffectChain effectChain(this);
-  AddBlendModeEffect(effectChain);
-
-  LayerManagerComposite::AutoAddMaskEffect autoMaskEffect(mMaskLayer, effectChain);
-  gfx::Rect clipRect(aClipRect.x, aClipRect.y, aClipRect.width, aClipRect.height);
+  RenderWithAllMasks(this, mCompositor, aClipRect,
+                     [&](EffectChain& effectChain, const Rect& clipRect) {
+    mCompositableHost->Composite(effectChain,
+                          GetEffectiveOpacity(),
+                          GetEffectiveTransform(),
+                          GetEffectFilter(),
+                          clipRect);
+  });
 
-  mCompositableHost->Composite(effectChain,
-                        GetEffectiveOpacity(),
-                        GetEffectiveTransform(),
-                        GetEffectFilter(),
-                        clipRect);
   mCompositableHost->BumpFlashCounter();
 }
 
 CompositableHost*
 CanvasLayerComposite::GetCompositableHost()
 {
   if (mCompositableHost && mCompositableHost->IsAttached()) {
     return mCompositableHost.get();
--- a/gfx/layers/composite/ColorLayerComposite.cpp
+++ b/gfx/layers/composite/ColorLayerComposite.cpp
@@ -16,38 +16,27 @@
 #include "mozilla/mozalloc.h"           // for operator delete, etc
 
 namespace mozilla {
 namespace layers {
 
 void
 ColorLayerComposite::RenderLayer(const gfx::IntRect& aClipRect)
 {
-  EffectChain effects(this);
-
-  GenEffectChain(effects);
-
-  gfx::IntRect boundRect = GetBounds();
-
-  LayerManagerComposite::AutoAddMaskEffect autoMaskEffect(GetMaskLayer(),
-                                                          effects);
+  gfx::Rect rect(GetBounds());
+  const gfx::Matrix4x4& transform = GetEffectiveTransform();
 
-  gfx::Rect rect(boundRect.x, boundRect.y,
-                 boundRect.width, boundRect.height);
-  gfx::Rect clipRect(aClipRect.x, aClipRect.y,
-                     aClipRect.width, aClipRect.height);
-
-  float opacity = GetEffectiveOpacity();
+  RenderWithAllMasks(this, mCompositor, aClipRect,
+                     [&](EffectChain& effectChain, const Rect& clipRect) {
+    GenEffectChain(effectChain);
+    mCompositor->DrawQuad(rect, clipRect, effectChain, GetEffectiveOpacity(), transform);
+  });
 
-  AddBlendModeEffect(effects);
-
-  const gfx::Matrix4x4& transform = GetEffectiveTransform();
-  mCompositor->DrawQuad(rect, clipRect, effects, opacity, transform);
   mCompositor->DrawDiagnostics(DiagnosticFlags::COLOR,
-                               rect, clipRect,
+                               rect, Rect(aClipRect),
                                transform);
 }
 
 void
 ColorLayerComposite::GenEffectChain(EffectChain& aEffect)
 {
   aEffect.mLayerRef = this;
   gfxRGBA color(GetColor());
--- a/gfx/layers/composite/ContainerLayerComposite.cpp
+++ b/gfx/layers/composite/ContainerLayerComposite.cpp
@@ -508,44 +508,35 @@ ContainerRender(ContainerT* aContainer,
       surface = aContainer->mPrepared->mTmpTarget;
     }
 
     if (!surface) {
       aContainer->mPrepared = nullptr;
       return;
     }
 
-    float opacity = aContainer->GetEffectiveOpacity();
-
-    gfx::IntRect visibleRect = aContainer->GetEffectiveVisibleRegion().GetBounds();
+    gfx::Rect visibleRect(aContainer->GetEffectiveVisibleRegion().GetBounds());
+    nsRefPtr<Compositor> compositor = aManager->GetCompositor();
 #ifdef MOZ_DUMP_PAINTING
     if (gfxUtils::sDumpPainting) {
-      RefPtr<gfx::DataSourceSurface> surf = surface->Dump(aManager->GetCompositor());
+      RefPtr<gfx::DataSourceSurface> surf = surface->Dump(compositor);
       if (surf) {
         WriteSnapshotToDumpFile(aContainer, surf);
       }
     }
 #endif
 
-    EffectChain effectChain(aContainer);
-    LayerManagerComposite::AutoAddMaskEffect autoMaskEffect(aContainer->GetMaskLayer(),
-                                                            effectChain,
-                                                            !aContainer->GetTransform().CanDraw2D());
-    if (autoMaskEffect.Failed()) {
-      NS_WARNING("Failed to apply a mask effect.");
-      return;
-    }
-
-    aContainer->AddBlendModeEffect(effectChain);
-    effectChain.mPrimaryEffect = new EffectRenderTarget(surface);
-
-    gfx::Rect rect(visibleRect.x, visibleRect.y, visibleRect.width, visibleRect.height);
-    gfx::Rect clipRect(aClipRect.x, aClipRect.y, aClipRect.width, aClipRect.height);
-    aManager->GetCompositor()->DrawQuad(rect, clipRect, effectChain, opacity,
-                                        aContainer->GetEffectiveTransform());
+    nsRefPtr<ContainerT> container = aContainer;
+    RenderWithAllMasks(aContainer, compositor, aClipRect,
+                       [&, surface, compositor, container](EffectChain& effectChain, const Rect& clipRect) {
+      effectChain.mPrimaryEffect = new EffectRenderTarget(surface);
+      compositor->DrawQuad(visibleRect, clipRect, effectChain,
+                           container->GetEffectiveOpacity(),
+                           container->GetEffectiveTransform());
+    });
   } else {
     RenderLayers(aContainer, aManager, RenderTargetPixel::FromUntyped(aClipRect));
   }
   aContainer->mPrepared = nullptr;
 
   // If it is a scrollable container layer with no child layers, and one of the APZCs
   // attached to it has a nonempty async transform, then that transform is not applied
   // to any visible content. Display a warning box (conditioned on the FPS display being
--- a/gfx/layers/composite/ImageLayerComposite.cpp
+++ b/gfx/layers/composite/ImageLayerComposite.cpp
@@ -90,27 +90,25 @@ ImageLayerComposite::RenderLayer(const I
   if (gfxUtils::sDumpPainting) {
     RefPtr<gfx::DataSourceSurface> surf = mImageHost->GetAsSurface();
     WriteSnapshotToDumpFile(this, surf);
   }
 #endif
 
   mCompositor->MakeCurrent();
 
-  EffectChain effectChain(this);
-  LayerManagerComposite::AutoAddMaskEffect autoMaskEffect(mMaskLayer, effectChain);
-  AddBlendModeEffect(effectChain);
-
-  gfx::Rect clipRect(aClipRect.x, aClipRect.y, aClipRect.width, aClipRect.height);
-  mImageHost->SetCompositor(mCompositor);
-  mImageHost->Composite(effectChain,
-                        GetEffectiveOpacity(),
-                        GetEffectiveTransformForBuffer(),
-                        GetEffectFilter(),
-                        clipRect);
+  RenderWithAllMasks(this, mCompositor, aClipRect,
+                     [&](EffectChain& effectChain, const Rect& clipRect) {
+    mImageHost->SetCompositor(mCompositor);
+    mImageHost->Composite(effectChain,
+                          GetEffectiveOpacity(),
+                          GetEffectiveTransformForBuffer(),
+                          GetEffectFilter(),
+                          clipRect);
+  });
   mImageHost->BumpFlashCounter();
 }
 
 void
 ImageLayerComposite::ComputeEffectiveTransforms(const gfx::Matrix4x4& aTransformToSurface)
 {
   gfx::Matrix4x4 local = GetLocalTransform();
 
--- a/gfx/layers/composite/LayerManagerComposite.h
+++ b/gfx/layers/composite/LayerManagerComposite.h
@@ -13,16 +13,17 @@
 #include "mozilla/Assertions.h"         // for MOZ_ASSERT, etc
 #include "mozilla/Attributes.h"         // for override
 #include "mozilla/RefPtr.h"             // for RefPtr, already_AddRefed
 #include "mozilla/gfx/2D.h"
 #include "mozilla/gfx/Point.h"          // for IntSize
 #include "mozilla/gfx/Rect.h"           // for Rect
 #include "mozilla/gfx/Types.h"          // for SurfaceFormat
 #include "mozilla/layers/CompositorTypes.h"
+#include "mozilla/layers/Effects.h"     // for EffectChain
 #include "mozilla/layers/LayersTypes.h"  // for LayersBackend, etc
 #include "mozilla/Maybe.h"              // for Maybe
 #include "mozilla/RefPtr.h"
 #include "mozilla/UniquePtr.h"
 #include "nsAString.h"
 #include "nsRefPtr.h"                   // for nsRefPtr
 #include "nsCOMPtr.h"                   // for already_AddRefed
 #include "nsDebug.h"                    // for NS_ASSERTION
@@ -452,13 +453,145 @@ protected:
   RefPtr<Compositor> mCompositor;
   float mShadowOpacity;
   bool mShadowTransformSetByAnimation;
   bool mDestroyed;
   bool mLayerComposited;
   gfx::IntRect mClearRect;
 };
 
+// Render aLayer using aCompositor and apply all mask layers of aLayer: The
+// layer's own mask layer (aLayer->GetMaskLayer()), and any ancestor mask
+// layers.
+// If more than one mask layer needs to be applied, we use intermediate surfaces
+// (CompositingRenderTargets) for rendering, applying one mask layer at a time.
+// Callers need to provide a callback function aRenderCallback that does the
+// actual rendering of the source. It needs to have the following form:
+// void (EffectChain& effectChain, const Rect& clipRect)
+// aRenderCallback is called exactly once, inside this function, unless aLayer's
+// visible region is completely clipped out (in that case, aRenderCallback won't
+// be called at all).
+// This function calls aLayer->AsLayerComposite()->AddBlendModeEffect for the
+// final rendering pass.
+//
+// (This function should really live in LayerManagerComposite.cpp, but we
+// need to use templates for passing lambdas until bug 1164522 is resolved.)
+template<typename RenderCallbackType>
+void
+RenderWithAllMasks(Layer* aLayer, Compositor* aCompositor,
+                   const gfx::IntRect& aClipRect,
+                   RenderCallbackType aRenderCallback)
+{
+  Layer* firstMask = nullptr;
+  size_t maskLayerCount = 0;
+  size_t nextAncestorMaskLayer = 0;
+
+  size_t ancestorMaskLayerCount = aLayer->GetAncestorMaskLayerCount();
+  if (Layer* ownMask = aLayer->GetMaskLayer()) {
+    firstMask = ownMask;
+    maskLayerCount = ancestorMaskLayerCount + 1;
+    nextAncestorMaskLayer = 0;
+  } else if (ancestorMaskLayerCount > 0) {
+    firstMask = aLayer->GetAncestorMaskLayerAt(0);
+    maskLayerCount = ancestorMaskLayerCount;
+    nextAncestorMaskLayer = 1;
+  } else {
+    // no mask layers at all
+  }
+
+  bool firstMaskIs3D = false;
+  if (ContainerLayer* container = aLayer->AsContainerLayer()) {
+    firstMaskIs3D = !container->GetTransform().CanDraw2D();
+  }
+
+  if (maskLayerCount <= 1) {
+    // This is the common case. Render in one pass and return.
+    EffectChain effectChain(aLayer);
+    LayerManagerComposite::AutoAddMaskEffect
+      autoMaskEffect(firstMask, effectChain, firstMaskIs3D);
+    aLayer->AsLayerComposite()->AddBlendModeEffect(effectChain);
+    aRenderCallback(effectChain, gfx::Rect(aClipRect));
+    return;
+  }
+
+  // We have multiple mask layers.
+  // We split our list of mask layers into three parts:
+  //  (1) The first mask
+  //  (2) The list of intermediate masks (every mask except first and last)
+  //  (3) The final mask.
+  // Part (2) can be empty.
+  // For parts (1) and (2) we need to allocate intermediate surfaces to render
+  // into. The final mask gets rendered into the original render target.
+
+  // Calculate the size of the intermediate surfaces.
+  gfx::Rect visibleRect(aLayer->GetEffectiveVisibleRegion().GetBounds());
+  gfx::Matrix4x4 transform = aLayer->GetEffectiveTransform();
+  // TODO: Use RenderTargetIntRect and TransformTo<...> here
+  gfx::IntRect surfaceRect =
+    RoundedOut(transform.TransformBounds(visibleRect)).Intersect(aClipRect);
+  if (surfaceRect.IsEmpty()) {
+    return;
+  }
+
+  RefPtr<CompositingRenderTarget> originalTarget =
+    aCompositor->GetCurrentRenderTarget();
+
+  RefPtr<CompositingRenderTarget> firstTarget =
+    aCompositor->CreateRenderTarget(surfaceRect, INIT_MODE_CLEAR);
+  if (!firstTarget) {
+    return;
+  }
+
+  // Render the source while applying the first mask.
+  aCompositor->SetRenderTarget(firstTarget);
+  {
+    EffectChain firstEffectChain(aLayer);
+    LayerManagerComposite::AutoAddMaskEffect
+      firstMaskEffect(firstMask, firstEffectChain, firstMaskIs3D);
+    aRenderCallback(firstEffectChain, gfx::Rect(aClipRect - surfaceRect.TopLeft()));
+    // firstTarget now contains the transformed source with the first mask and
+    // opacity already applied.
+  }
+
+  // Apply the intermediate masks.
+  gfx::Rect intermediateClip(surfaceRect - surfaceRect.TopLeft());
+  RefPtr<CompositingRenderTarget> previousTarget = firstTarget;
+  for (size_t i = nextAncestorMaskLayer; i < ancestorMaskLayerCount - 1; i++) {
+    Layer* intermediateMask = aLayer->GetAncestorMaskLayerAt(i);
+    RefPtr<CompositingRenderTarget> intermediateTarget =
+      aCompositor->CreateRenderTarget(surfaceRect, INIT_MODE_CLEAR);
+    if (!intermediateTarget) {
+      break;
+    }
+    aCompositor->SetRenderTarget(intermediateTarget);
+    EffectChain intermediateEffectChain(aLayer);
+    LayerManagerComposite::AutoAddMaskEffect
+      intermediateMaskEffect(intermediateMask, intermediateEffectChain);
+    if (intermediateMaskEffect.Failed()) {
+      continue;
+    }
+    intermediateEffectChain.mPrimaryEffect = new EffectRenderTarget(previousTarget);
+    aCompositor->DrawQuad(gfx::Rect(surfaceRect), intermediateClip,
+                          intermediateEffectChain, 1.0, gfx::Matrix4x4());
+    previousTarget = intermediateTarget;
+  }
+
+  aCompositor->SetRenderTarget(originalTarget);
+
+  // Apply the final mask, rendering into originalTarget.
+  EffectChain finalEffectChain(aLayer);
+  finalEffectChain.mPrimaryEffect = new EffectRenderTarget(previousTarget);
+  Layer* finalMask = aLayer->GetAncestorMaskLayerAt(ancestorMaskLayerCount - 1);
+
+  // The blend mode needs to be applied in this final step, because this is
+  // where we're blending with the actual background (which is in originalTarget).
+  aLayer->AsLayerComposite()->AddBlendModeEffect(finalEffectChain);
+  LayerManagerComposite::AutoAddMaskEffect autoMaskEffect(finalMask, finalEffectChain);
+  if (!autoMaskEffect.Failed()) {
+    aCompositor->DrawQuad(gfx::Rect(surfaceRect), gfx::Rect(aClipRect),
+                          finalEffectChain, 1.0, gfx::Matrix4x4());
+  }
+}
 
 } /* layers */
 } /* mozilla */
 
 #endif /* GFX_LayerManagerComposite_H */
--- a/gfx/layers/composite/PaintedLayerComposite.cpp
+++ b/gfx/layers/composite/PaintedLayerComposite.cpp
@@ -114,47 +114,49 @@ void
 PaintedLayerComposite::RenderLayer(const gfx::IntRect& aClipRect)
 {
   if (!mBuffer || !mBuffer->IsAttached()) {
     return;
   }
   PROFILER_LABEL("PaintedLayerComposite", "RenderLayer",
     js::ProfileEntry::Category::GRAPHICS);
 
-  MOZ_ASSERT(mBuffer->GetCompositor() == mCompositeManager->GetCompositor() &&
+  Compositor* compositor = mCompositeManager->GetCompositor();
+
+  MOZ_ASSERT(mBuffer->GetCompositor() == compositor &&
              mBuffer->GetLayer() == this,
              "buffer is corrupted");
 
   const nsIntRegion& visibleRegion = GetEffectiveVisibleRegion();
-  gfx::Rect clipRect(aClipRect.x, aClipRect.y, aClipRect.width, aClipRect.height);
 
 #ifdef MOZ_DUMP_PAINTING
   if (gfxUtils::sDumpPainting) {
     RefPtr<gfx::DataSourceSurface> surf = mBuffer->GetAsSurface();
     if (surf) {
       WriteSnapshotToDumpFile(this, surf);
     }
   }
 #endif
 
-  EffectChain effectChain(this);
-  LayerManagerComposite::AutoAddMaskEffect autoMaskEffect(mMaskLayer, effectChain);
-  AddBlendModeEffect(effectChain);
 
-  mBuffer->SetPaintWillResample(MayResample());
+  RenderWithAllMasks(this, compositor, aClipRect,
+                     [&](EffectChain& effectChain, const Rect& clipRect) {
+    mBuffer->SetPaintWillResample(MayResample());
 
-  mBuffer->Composite(effectChain,
-                     GetEffectiveOpacity(),
-                     GetEffectiveTransform(),
-                     GetEffectFilter(),
-                     clipRect,
-                     &visibleRegion);
+    mBuffer->Composite(effectChain,
+                       GetEffectiveOpacity(),
+                       GetEffectiveTransform(),
+                       GetEffectFilter(),
+                       clipRect,
+                       &visibleRegion);
+  });
+
   mBuffer->BumpFlashCounter();
 
-  mCompositeManager->GetCompositor()->MakeCurrent();
+  compositor->MakeCurrent();
 }
 
 CompositableHost*
 PaintedLayerComposite::GetCompositableHost()
 {
   if (mBuffer && mBuffer->IsAttached()) {
     return mBuffer.get();
   }