Disable device access on textures created against stale layers. (bug 1256517 part 3, r=nical)
☠☠ backed out by 712d5ccb76cd ☠ ☠
authorDavid Anderson <danderson@mozilla.com>
Wed, 23 Mar 2016 10:32:21 -0700
changeset 290094 9d33171eb4e91554b3d7f07da5dd737fc2c86c53
parent 290093 8e5cb6d1d1a04e2d3faab4567e93b077515fa3cc
child 290095 27a8a01abf66e8794d52ef84ef7ad84db2ca2f2d
push id30114
push usercbook@mozilla.com
push dateThu, 24 Mar 2016 15:15:54 +0000
treeherdermozilla-central@24c5fbde4488 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnical
bugs1256517
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Disable device access on textures created against stale layers. (bug 1256517 part 3, r=nical)
gfx/2d/Logging.h
gfx/layers/CompositorTypes.h
gfx/layers/d3d11/TextureD3D11.cpp
gfx/layers/d3d9/TextureD3D9.cpp
gfx/layers/ipc/LayerTransactionParent.cpp
--- a/gfx/2d/Logging.h
+++ b/gfx/2d/Logging.h
@@ -128,16 +128,17 @@ enum class LogReason : int {
   SourceSurfaceIncompatible,
   GlyphAllocFailedCairo,
   GlyphAllocFailedCG,
   InvalidRect,
   CannotDraw3D, // 20
   IncompatibleBasicTexturedEffect,
   InvalidFont,
   AsyncTransactionTimeout,
+  PAllocTextureBackendMismatch,
   // End
   MustBeLessThanThis = 101,
 };
 
 struct BasicLogger
 {
   // For efficiency, this method exists and copies the logic of the
   // OutputMessage below.  If making any changes here, also make it
--- a/gfx/layers/CompositorTypes.h
+++ b/gfx/layers/CompositorTypes.h
@@ -58,19 +58,22 @@ enum class TextureFlags : uint32_t {
   // textures.
   IMMUTABLE          = 1 << 9,
   // The contents of the texture must be uploaded or copied immediately
   // during the transaction, because the producer may want to write
   // to it again.
   IMMEDIATE_UPLOAD   = 1 << 10,
   // The texture is part of a component-alpha pair
   COMPONENT_ALPHA    = 1 << 11,
+  // The texture is being allocated for a compositor that no longer exists.
+  // This flag is only used in the parent process.
+  INVALID_COMPOSITOR = 1 << 12,
 
   // OR union of all valid bits
-  ALL_BITS           = (1 << 12) - 1,
+  ALL_BITS           = (1 << 13) - 1,
   // the default flags
   DEFAULT = NO_FLAGS
 };
 MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(TextureFlags)
 
 static inline bool
 TextureRequiresLocking(TextureFlags aFlags)
 {
--- a/gfx/layers/d3d11/TextureD3D11.cpp
+++ b/gfx/layers/d3d11/TextureD3D11.cpp
@@ -624,16 +624,20 @@ DXGITextureHostD3D11::OpenSharedHandle()
   mTexture->GetDesc(&desc);
   mSize = IntSize(desc.Width, desc.Height);
   return true;
 }
 
 RefPtr<ID3D11Device>
 DXGITextureHostD3D11::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
+
   RefPtr<ID3D11Device> device;
   gfxWindowsPlatform::GetPlatform()->GetD3D11Device(&device);
   return device;
 }
 
 static bool AssertD3D11Compositor(Compositor* aCompositor)
 {
   bool ok = aCompositor && aCompositor->GetBackendType() == LayersBackend::LAYERS_D3D11;
@@ -742,16 +746,20 @@ DXGIYCbCrTextureHostD3D11::OpenSharedHan
   mTextures[2] = textures[2].forget();
 
   return true;
 }
 
 RefPtr<ID3D11Device>
 DXGIYCbCrTextureHostD3D11::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
+
   RefPtr<ID3D11Device> device;
   gfxWindowsPlatform::GetPlatform()->GetD3D11Device(&device);
   return device;
 }
 
 void
 DXGIYCbCrTextureHostD3D11::SetCompositor(Compositor* aCompositor)
 {
--- a/gfx/layers/d3d9/TextureD3D9.cpp
+++ b/gfx/layers/d3d9/TextureD3D9.cpp
@@ -906,16 +906,19 @@ TextureHostD3D9::UpdatedInternal(const n
   if (!mTextureSource->UpdateFromTexture(mTexture, regionToUpdate)) {
     gfxCriticalError() << "[D3D9] DataTextureSourceD3D9::UpdateFromTexture failed";
   }
 }
 
 IDirect3DDevice9*
 TextureHostD3D9::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
   return mCompositor ? mCompositor->device() : nullptr;
 }
 
 void
 TextureHostD3D9::SetCompositor(Compositor* aCompositor)
 {
   if (!AssertD3D9Compositor(aCompositor)) {
     mCompositor = nullptr;
@@ -971,16 +974,19 @@ DXGITextureHostD3D9::DXGITextureHostD3D9
 {
   MOZ_ASSERT(mHandle);
   OpenSharedHandle();
 }
 
 IDirect3DDevice9*
 DXGITextureHostD3D9::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
   return mCompositor ? mCompositor->device() : nullptr;
 }
 
 void
 DXGITextureHostD3D9::OpenSharedHandle()
 {
   MOZ_ASSERT(!mTextureSource);
 
@@ -1065,16 +1071,19 @@ DXGIYCbCrTextureHostD3D9::DXGIYCbCrTextu
   mHandles[0] = reinterpret_cast<HANDLE>(aDescriptor.handleY());
   mHandles[1] = reinterpret_cast<HANDLE>(aDescriptor.handleCb());
   mHandles[2] = reinterpret_cast<HANDLE>(aDescriptor.handleCr());
 }
 
 IDirect3DDevice9*
 DXGIYCbCrTextureHostD3D9::GetDevice()
 {
+  if (mFlags & TextureFlags::INVALID_COMPOSITOR) {
+    return nullptr;
+  }
   return mCompositor ? mCompositor->device() : nullptr;
 }
 
 void
 DXGIYCbCrTextureHostD3D9::SetCompositor(Compositor* aCompositor)
 {
   if (!AssertD3D9Compositor(aCompositor)) {
     mCompositor = nullptr;
--- a/gfx/layers/ipc/LayerTransactionParent.cpp
+++ b/gfx/layers/ipc/LayerTransactionParent.cpp
@@ -964,18 +964,30 @@ LayerTransactionParent::DeallocPComposit
   return CompositableHost::DestroyIPDLActor(aActor);
 }
 
 PTextureParent*
 LayerTransactionParent::AllocPTextureParent(const SurfaceDescriptor& aSharedData,
                                             const LayersBackend& aLayersBackend,
                                             const TextureFlags& aFlags)
 {
-  MOZ_ASSERT(aLayersBackend == mLayerManager->GetCompositor()->GetBackendType());
-  return TextureHost::CreateIPDLActor(this, aSharedData, aLayersBackend, aFlags);
+  TextureFlags flags = aFlags;
+
+  if (mPendingCompositorUpdates) {
+    // The compositor was recreated, and we're receiving layers updates for a
+    // a layer manager that will soon be discarded or invalidated. We can't
+    // return null because this will mess up deserialization later and we'll
+    // kill the content process. Instead, we signal that the underlying
+    // TextureHost should not attempt to access the compositor.
+    flags |= TextureFlags::INVALID_COMPOSITOR;
+  } else if (aLayersBackend != mLayerManager->GetCompositor()->GetBackendType()) {
+    gfxDevCrash(LogReason::PAllocTextureBackendMismatch, "Texture backend is wrong");
+  }
+
+  return TextureHost::CreateIPDLActor(this, aSharedData, aLayersBackend, flags);
 }
 
 bool
 LayerTransactionParent::DeallocPTextureParent(PTextureParent* actor)
 {
   return TextureHost::DestroyIPDLActor(actor);
 }