Disable device access on textures created against stale layers. (bug 1256517 part 3, r=nical, a=rkothari)
☠☠ backed out by 76a54862123c ☠ ☠
authorDavid Anderson <danderson@mozilla.com>
Fri, 25 Mar 2016 01:36:17 -0700
changeset 324106 a24cd849856c
parent 324105 8be24a8a9cf6
child 324107 18a82d839878
push id5913
push userjlund@mozilla.com
push dateMon, 25 Apr 2016 16:57:49 +0000
treeherdermozilla-beta@dcaf0a6fa115 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnical, rkothari
bugs1256517
milestone47.0a2
Disable device access on textures created against stale layers. (bug 1256517 part 3, r=nical, a=rkothari)
gfx/2d/Logging.h
gfx/layers/CompositorTypes.h
gfx/layers/d3d11/TextureD3D11.cpp
gfx/layers/d3d9/TextureD3D9.cpp
gfx/layers/ipc/LayerTransactionParent.cpp
--- a/gfx/2d/Logging.h
+++ b/gfx/2d/Logging.h
@@ -127,16 +127,17 @@ enum class LogReason : int {
   FilterNodeD2D1Backend,
   SourceSurfaceIncompatible,
   GlyphAllocFailedCairo,
   GlyphAllocFailedCG,
   InvalidRect,
   CannotDraw3D, // 20
   IncompatibleBasicTexturedEffect,
   InvalidFont,
+  PAllocTextureBackendMismatch,
   // End
   MustBeLessThanThis = 101,
 };
 
 struct BasicLogger
 {
   // For efficiency, this method exists and copies the logic of the
   // OutputMessage below.  If making any changes here, also make it
--- a/gfx/layers/CompositorTypes.h
+++ b/gfx/layers/CompositorTypes.h
@@ -58,19 +58,22 @@ enum class TextureFlags : uint32_t {
   // textures.
   IMMUTABLE          = 1 << 9,
   // The contents of the texture must be uploaded or copied immediately
   // during the transaction, because the producer may want to write
   // to it again.
   IMMEDIATE_UPLOAD   = 1 << 10,
   // The texture is part of a component-alpha pair
   COMPONENT_ALPHA    = 1 << 11,
+  // The texture is being allocated for a compositor that no longer exists.
+  // This flag is only used in the parent process.
+  INVALID_COMPOSITOR = 1 << 12,
 
   // OR union of all valid bits
-  ALL_BITS           = (1 << 12) - 1,
+  ALL_BITS           = (1 << 13) - 1,
   // the default flags
   DEFAULT = NO_FLAGS
 };
 MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(TextureFlags)
 
 static inline bool
 TextureRequiresLocking(TextureFlags aFlags)
 {
--- a/gfx/layers/d3d11/TextureD3D11.cpp
+++ b/gfx/layers/d3d11/TextureD3D11.cpp
@@ -624,16 +624,20 @@ DXGITextureHostD3D11::OpenSharedHandle()
   mTexture->GetDesc(&desc);
   mSize = IntSize(desc.Width, desc.Height);
   return true;
 }
 
 ID3D11Device*
 DXGITextureHostD3D11::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
+
   return gfxWindowsPlatform::GetPlatform()->GetD3D11Device();
 }
 
 void
 DXGITextureHostD3D11::SetCompositor(Compositor* aCompositor)
 {
   MOZ_ASSERT(aCompositor);
   mCompositor = static_cast<CompositorD3D11*>(aCompositor);
@@ -728,16 +732,20 @@ DXGIYCbCrTextureHostD3D11::OpenSharedHan
   mTextures[2] = textures[2].forget();
 
   return true;
 }
 
 ID3D11Device*
 DXGIYCbCrTextureHostD3D11::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
+
   return gfxWindowsPlatform::GetPlatform()->GetD3D11Device();
 }
 
 void
 DXGIYCbCrTextureHostD3D11::SetCompositor(Compositor* aCompositor)
 {
   MOZ_ASSERT(aCompositor);
   mCompositor = static_cast<CompositorD3D11*>(aCompositor);
--- a/gfx/layers/d3d9/TextureD3D9.cpp
+++ b/gfx/layers/d3d9/TextureD3D9.cpp
@@ -896,16 +896,19 @@ TextureHostD3D9::UpdatedInternal(const n
   if (!mTextureSource->UpdateFromTexture(mTexture, regionToUpdate)) {
     gfxCriticalError() << "[D3D9] DataTextureSourceD3D9::UpdateFromTexture failed";
   }
 }
 
 IDirect3DDevice9*
 TextureHostD3D9::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
   return mCompositor ? mCompositor->device() : nullptr;
 }
 
 void
 TextureHostD3D9::SetCompositor(Compositor* aCompositor)
 {
   mCompositor = static_cast<CompositorD3D9*>(aCompositor);
   if (mTextureSource) {
@@ -956,16 +959,19 @@ DXGITextureHostD3D9::DXGITextureHostD3D9
 {
   MOZ_ASSERT(mHandle);
   OpenSharedHandle();
 }
 
 IDirect3DDevice9*
 DXGITextureHostD3D9::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
   return mCompositor ? mCompositor->device() : nullptr;
 }
 
 void
 DXGITextureHostD3D9::OpenSharedHandle()
 {
   MOZ_ASSERT(!mTextureSource);
 
@@ -1046,16 +1052,19 @@ DXGIYCbCrTextureHostD3D9::DXGIYCbCrTextu
   mHandles[0] = reinterpret_cast<HANDLE>(aDescriptor.handleY());
   mHandles[1] = reinterpret_cast<HANDLE>(aDescriptor.handleCb());
   mHandles[2] = reinterpret_cast<HANDLE>(aDescriptor.handleCr());
 }
 
 IDirect3DDevice9*
 DXGIYCbCrTextureHostD3D9::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
   return mCompositor ? mCompositor->device() : nullptr;
 }
 
 void
 DXGIYCbCrTextureHostD3D9::SetCompositor(Compositor* aCompositor)
 {
   MOZ_ASSERT(aCompositor);
   mCompositor = static_cast<CompositorD3D9*>(aCompositor);
--- a/gfx/layers/ipc/LayerTransactionParent.cpp
+++ b/gfx/layers/ipc/LayerTransactionParent.cpp
@@ -945,18 +945,30 @@ LayerTransactionParent::DeallocPComposit
   return CompositableHost::DestroyIPDLActor(aActor);
 }
 
 PTextureParent*
 LayerTransactionParent::AllocPTextureParent(const SurfaceDescriptor& aSharedData,
                                             const LayersBackend& aLayersBackend,
                                             const TextureFlags& aFlags)
 {
-  MOZ_ASSERT(aLayersBackend == mLayerManager->GetCompositor()->GetBackendType());
-  return TextureHost::CreateIPDLActor(this, aSharedData, aLayersBackend, aFlags);
+  TextureFlags flags = aFlags;
+
+  if (mPendingCompositorUpdates) {
+    // The compositor was recreated, and we're receiving layers updates for a
+    // a layer manager that will soon be discarded or invalidated. We can't
+    // return null because this will mess up deserialization later and we'll
+    // kill the content process. Instead, we signal that the underlying
+    // TextureHost should not attempt to access the compositor.
+    flags |= TextureFlags::INVALID_COMPOSITOR;
+  } else if (aLayersBackend != mLayerManager->GetCompositor()->GetBackendType()) {
+    gfxDevCrash(gfx::LogReason::PAllocTextureBackendMismatch) << "Texture backend is wrong";
+  }
+
+  return TextureHost::CreateIPDLActor(this, aSharedData, aLayersBackend, flags);
 }
 
 bool
 LayerTransactionParent::DeallocPTextureParent(PTextureParent* actor)
 {
   return TextureHost::DestroyIPDLActor(actor);
 }