Bug 716438; using flags to select a shader. r=Bas
authorNicholas Cameron <ncameron@mozilla.com>
Thu, 01 Mar 2012 17:29:30 +0000
changeset 94298 d2eefdb6f9d40047489d049b544f49c56e557057
parent 94297 16f39df9be8b5d7b373827e7a4a8ff8bd0f10bba
child 94299 82ad8aacc6695fbed8d22288fcd2f7d99b52873c
push idunknown
push userunknown
push dateunknown
reviewersBas
bugs716438
milestone15.0a1
Bug 716438; using flags to select a shader. r=Bas
gfx/layers/d3d10/CanvasLayerD3D10.cpp
gfx/layers/d3d10/ColorLayerD3D10.cpp
gfx/layers/d3d10/ContainerLayerD3D10.cpp
gfx/layers/d3d10/ImageLayerD3D10.cpp
gfx/layers/d3d10/LayerManagerD3D10.cpp
gfx/layers/d3d10/LayerManagerD3D10.h
gfx/layers/d3d10/ThebesLayerD3D10.cpp
--- a/gfx/layers/d3d10/CanvasLayerD3D10.cpp
+++ b/gfx/layers/d3d10/CanvasLayerD3D10.cpp
@@ -259,63 +259,24 @@ CanvasLayerD3D10::RenderLayer()
 
   if (!mTexture)
     return;
 
   nsIntRect visibleRect = mVisibleRegion.GetBounds();
 
   SetEffectTransformAndOpacity();
 
-  ID3D10EffectTechnique *technique;
-
-  if (LoadMaskTexture()) {
-    if (mDataIsPremultiplied) {
-      if (!mHasAlpha) {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulPointMask");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
-        }
-      } else {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulPointMask");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
-        }
-      }
-    } else {
-      if (mFilter == gfxPattern::FILTER_NEAREST) {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerNonPremulPointMask");
-      } else {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerNonPremulMask");
-      }
-    }
-  } else { // no mask
-    if (mDataIsPremultiplied) {
-      if (!mHasAlpha) {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulPoint");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremul");
-        }
-      } else {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulPoint");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremul");
-        }
-      }
-    } else {
-      if (mFilter == gfxPattern::FILTER_NEAREST) {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerNonPremulPoint");
-      } else {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerNonPremul");
-      }
-    }
-  }
+  PRUint8 shaderFlags = 0;
+  shaderFlags |= LoadMaskTexture();
+  shaderFlags |= mDataIsPremultiplied
+                ? SHADER_PREMUL : SHADER_NON_PREMUL | SHADER_RGBA;
+  shaderFlags |= mHasAlpha ? SHADER_RGBA : SHADER_RGB;
+  shaderFlags |= mFilter == gfxPattern::FILTER_NEAREST
+                ? SHADER_POINT : SHADER_LINEAR;
+  ID3D10EffectTechnique* technique = SelectShader(shaderFlags);
 
   if (mSRView) {
     effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(mSRView);
   }
 
   effect()->GetVariableByName("vLayerQuad")->AsVector()->SetFloatVector(
     ShaderConstantRectD3D10(
       (float)mBounds.x,
--- a/gfx/layers/d3d10/ColorLayerD3D10.cpp
+++ b/gfx/layers/d3d10/ColorLayerD3D10.cpp
@@ -68,22 +68,17 @@ ColorLayerD3D10::RenderLayer()
   color[2] = (float)(mColor.b * opacity);
   color[3] = opacity;
 
   const gfx3DMatrix& transform = GetEffectiveTransform();
   void* raw = &const_cast<gfx3DMatrix&>(transform)._11;
   effect()->GetVariableByName("mLayerTransform")->SetRawValue(raw, 0, 64);
   effect()->GetVariableByName("fLayerColor")->AsVector()->SetFloatVector(color);
 
-  ID3D10EffectTechnique *technique;
-  if (LoadMaskTexture()) {
-    technique = effect()->GetTechniqueByName("RenderSolidColorLayerMask");
-  } else {
-    technique = effect()->GetTechniqueByName("RenderSolidColorLayer");
-  }
+  ID3D10EffectTechnique *technique = SelectShader(SHADER_SOLID | LoadMaskTexture());
 
   nsIntRegionRectIterator iter(mVisibleRegion);
 
   const nsIntRect *iterRect;
   while ((iterRect = iter.Next())) {
     effect()->GetVariableByName("vLayerQuad")->AsVector()->SetFloatVector(
       ShaderConstantRectD3D10(
         (float)iterRect->x,
--- a/gfx/layers/d3d10/ContainerLayerD3D10.cpp
+++ b/gfx/layers/d3d10/ContainerLayerD3D10.cpp
@@ -311,22 +311,22 @@ ContainerLayerD3D10::RenderLayer()
     effect()->GetVariableByName("vRenderTargetOffset")->
       SetRawValue(previousRenderTargetOffset, 0, 8);
 
     SetEffectTransformAndOpacity();
 
     ID3D10EffectTechnique *technique;
     if (LoadMaskTexture()) {
       if (GetTransform().CanDraw2D()) {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
+        technique = SelectShader(SHADER_RGBA | SHADER_PREMUL | SHADER_MASK);
       } else {
-        technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask3D");
+        technique = SelectShader(SHADER_RGBA | SHADER_PREMUL | SHADER_MASK_3D);
       }
     } else {
-      technique = effect()->GetTechniqueByName("RenderRGBALayerPremul");
+        technique = SelectShader(SHADER_RGBA | SHADER_PREMUL | SHADER_NO_MASK);
     }
 
     effect()->GetVariableByName("vLayerQuad")->AsVector()->SetFloatVector(
       ShaderConstantRectD3D10(
         (float)visibleRect.x,
         (float)visibleRect.y,
         (float)visibleRect.width,
         (float)visibleRect.height)
--- a/gfx/layers/d3d10/ImageLayerD3D10.cpp
+++ b/gfx/layers/d3d10/ImageLayerD3D10.cpp
@@ -206,45 +206,24 @@ ImageLayerD3D10::RenderLayer()
   {
     bool hasAlpha = false;
 
     nsRefPtr<ID3D10ShaderResourceView> srView = GetImageSRView(image, hasAlpha);
     if (!srView) {
       return;
     }
     
-    if (LoadMaskTexture()) {
-      if (hasAlpha) {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulPointMask");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
-        }
-       } else {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulPointMask");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
-        }
-       }
-     } else {
-      if (hasAlpha) {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremulPoint");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBALayerPremul");
-        }
-       } else {
-        if (mFilter == gfxPattern::FILTER_NEAREST) {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremulPoint");
-        } else {
-          technique = effect()->GetTechniqueByName("RenderRGBLayerPremul");
-        }
-      }
-    }
+    PRUint8 shaderFlags = SHADER_PREMUL;
+    shaderFlags |= LoadMaskTexture();
+    shaderFlags |= hasAlpha
+                  ? SHADER_RGBA : SHADER_RGB;
+    shaderFlags |= mFilter == gfxPattern::FILTER_NEAREST
+                  ? SHADER_POINT : SHADER_LINEAR;
+    technique = SelectShader(shaderFlags);
+
 
     effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(srView);
 
     effect()->GetVariableByName("vLayerQuad")->AsVector()->SetFloatVector(
       ShaderConstantRectD3D10(
         (float)0,
         (float)0,
         (float)size.width,
@@ -270,28 +249,22 @@ ImageLayerD3D10::RenderLayer()
     }
 
     nsRefPtr<ID3D10Device> dev;
     data->mYTexture->GetDevice(getter_AddRefs(dev));
     if (dev != device()) {
       return;
     }
 
-    bool useMask = LoadMaskTexture();
-
     // TODO: At some point we should try to deal with mFilter here, you don't
     // really want to use point filtering in the case of NEAREST, since that
     // would also use point filtering for Chroma upsampling. Where most likely
     // the user would only want point filtering for final RGB image upsampling.
 
-    if (useMask) {
-      technique = effect()->GetTechniqueByName("RenderYCbCrLayerMask");
-    } else {
-      technique = effect()->GetTechniqueByName("RenderYCbCrLayer");
-    }
+    technique = SelectShader(SHADER_YCBCR | LoadMaskTexture());
 
     effect()->GetVariableByName("tY")->AsShaderResource()->SetResource(data->mYView);
     effect()->GetVariableByName("tCb")->AsShaderResource()->SetResource(data->mCbView);
     effect()->GetVariableByName("tCr")->AsShaderResource()->SetResource(data->mCrView);
 
     /*
      * Send 3d control data and metadata to NV3DVUtils
      */
--- a/gfx/layers/d3d10/LayerManagerD3D10.cpp
+++ b/gfx/layers/d3d10/LayerManagerD3D10.cpp
@@ -837,45 +837,95 @@ LayerManagerD3D10::ReportFailure(const n
   gfx::LogFailure(msg);
 }
 
 LayerD3D10::LayerD3D10(LayerManagerD3D10 *aManager)
   : mD3DManager(aManager)
 {
 }
 
-bool LayerD3D10::LoadMaskTexture()
+ID3D10EffectTechnique*
+LayerD3D10::SelectShader(PRUint8 aFlags)
+{
+  switch (aFlags) {
+  case (SHADER_RGBA | SHADER_NON_PREMUL | SHADER_LINEAR | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerNonPremulMask");
+  case (SHADER_RGBA | SHADER_NON_PREMUL | SHADER_LINEAR | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerNonPremul");
+  case (SHADER_RGBA | SHADER_NON_PREMUL | SHADER_POINT | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerNonPremulPoint");
+  case (SHADER_RGBA | SHADER_NON_PREMUL | SHADER_POINT | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerNonPremulPointMask");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_LINEAR | SHADER_MASK_3D):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremulMask3D");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_LINEAR | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_LINEAR | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremul");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_POINT | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremulPointMask");
+  case (SHADER_RGBA | SHADER_PREMUL | SHADER_POINT | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBALayerPremulPoint");
+  case (SHADER_RGB | SHADER_PREMUL | SHADER_POINT | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBLayerPremulPointMask");
+  case (SHADER_RGB | SHADER_PREMUL | SHADER_POINT | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBLayerPremulPoint");
+  case (SHADER_RGB | SHADER_PREMUL | SHADER_LINEAR | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
+  case (SHADER_RGB | SHADER_PREMUL | SHADER_LINEAR | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderRGBLayerPremul");
+  case (SHADER_SOLID | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderSolidColorLayerMask");
+  case (SHADER_SOLID | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderSolidColorLayer");
+  case (SHADER_COMPONENT_ALPHA | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderComponentAlphaLayerMask");
+  case (SHADER_COMPONENT_ALPHA | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderComponentAlphaLayer");
+  case (SHADER_YCBCR | SHADER_MASK):
+    return effect()->GetTechniqueByName("RenderYCbCrLayerMask");
+  case (SHADER_YCBCR | SHADER_NO_MASK):
+    return effect()->GetTechniqueByName("RenderYCbCrLayer");
+  default:
+    NS_ERROR("Invalid shader.");
+    return nsnull;
+  }
+}
+
+PRUint8
+LayerD3D10::LoadMaskTexture()
 {
   if (Layer* maskLayer = GetLayer()->GetMaskLayer()) {
     gfxIntSize size;
     nsRefPtr<ID3D10ShaderResourceView> maskSRV =
       static_cast<LayerD3D10*>(maskLayer->ImplData())->GetAsTexture(&size);
   
     if (!maskSRV) {
-      return false;
+      return SHADER_NO_MASK;
     }
 
     gfxMatrix maskTransform;
-    bool maskIs2D = maskLayer->GetTransform().CanDraw2D(&maskTransform);
+    bool maskIs2D = maskLayer->GetEffectiveTransform().CanDraw2D(&maskTransform);
     NS_ASSERTION(maskIs2D, "How did we end up with a 3D transform here?!");
-    gfxRect bounds = gfxRect(maskTransform.GetTranslation(), size);
+    gfxRect bounds = gfxRect(gfxPoint(), size);
+    bounds = maskTransform.TransformBounds(bounds);
 
     effect()->GetVariableByName("vMaskQuad")->AsVector()->SetFloatVector(
       ShaderConstantRectD3D10(
         (float)bounds.x,
         (float)bounds.y,
         (float)bounds.width,
         (float)bounds.height)
       );
 
     effect()->GetVariableByName("tMask")->AsShaderResource()->SetResource(maskSRV);
-    return true;
+    return SHADER_MASK;
   }
 
-  return false; 
+  return SHADER_NO_MASK; 
 }
 
 WindowLayer::WindowLayer(LayerManagerD3D10* aManager)
   : ThebesLayer(aManager, nsnull)
 {
  }
 
 WindowLayer::~WindowLayer()
--- a/gfx/layers/d3d10/LayerManagerD3D10.h
+++ b/gfx/layers/d3d10/LayerManagerD3D10.h
@@ -314,20 +314,43 @@ public:
     effect()->GetVariableByName("mLayerTransform")->SetRawValue(raw, 0, 64);
     effect()->GetVariableByName("fLayerOpacity")->AsScalar()->SetFloat(layer->GetEffectiveOpacity());
   }
 
 protected:
   /*
    * Finds a texture for this layer's mask layer (if it has one) and sets it
    * as an input to the shaders.
-   * Returns true if a texture is loaded, false if there was no mask layer, or
-   * a texture for the mask layer could not be loaded.
+   * Returns SHADER_MASK if a texture is loaded, SHADER_NO_MASK if there was no 
+   * mask layer, or a texture for the mask layer could not be loaded.
+   */
+  PRUint8 LoadMaskTexture();
+
+  /**
+   * Select a shader technique using a combination of the following flags.
+   * Not all combinations of flags are supported, and might cause an error,
+   * check the fx file to see which shaders exist. In particular, aFlags should
+   * include any combination of the 0x20 bit = 0 flags OR one of the 0x20 bit = 1
+   * flags. Mask flags can be used in either case.
    */
-  bool LoadMaskTexture();
+  ID3D10EffectTechnique* SelectShader(PRUint8 aFlags);
+  const static PRUint8 SHADER_NO_MASK = 0;
+  const static PRUint8 SHADER_MASK = 0x1;
+  const static PRUint8 SHADER_MASK_3D = 0x2;
+  // 0x20 bit = 0
+  const static PRUint8 SHADER_RGB = 0;
+  const static PRUint8 SHADER_RGBA = 0x4;
+  const static PRUint8 SHADER_NON_PREMUL = 0;
+  const static PRUint8 SHADER_PREMUL = 0x8;
+  const static PRUint8 SHADER_LINEAR = 0;
+  const static PRUint8 SHADER_POINT = 0x10;
+  // 0x20 bit = 1
+  const static PRUint8 SHADER_YCBCR = 0x20;
+  const static PRUint8 SHADER_COMPONENT_ALPHA = 0x24;
+  const static PRUint8 SHADER_SOLID = 0x28;
 
   LayerManagerD3D10 *mD3DManager;
 };
 
 /**
  * WindowLayer is a simple, special kinds of shadowable layer into
  * which layer trees are rendered.  It represents something like an OS
  * window.  It exists only to allow sharing textures with the
--- a/gfx/layers/d3d10/ThebesLayerD3D10.cpp
+++ b/gfx/layers/d3d10/ThebesLayerD3D10.cpp
@@ -127,46 +127,29 @@ ThebesLayerD3D10::RenderLayer()
 {
   if (!mTexture) {
     return;
   }
 
   SetEffectTransformAndOpacity();
 
   ID3D10EffectTechnique *technique;
-  if (LoadMaskTexture()) {
-    switch (mCurrentSurfaceMode) {
-    case SURFACE_COMPONENT_ALPHA:
-      technique = effect()->GetTechniqueByName("RenderComponentAlphaLayerMask");
-      break;
-    case SURFACE_OPAQUE:
-      technique = effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
-      break;
-    case SURFACE_SINGLE_CHANNEL_ALPHA:
-      technique = effect()->GetTechniqueByName("RenderRGBALayerPremulMask");
-      break;
-    default:
-      NS_ERROR("Unknown mode");
-      return;
-    }
-  } else {
-    switch (mCurrentSurfaceMode) {
-    case SURFACE_COMPONENT_ALPHA:
-      technique = effect()->GetTechniqueByName("RenderComponentAlphaLayer");
-      break;
-    case SURFACE_OPAQUE:
-      technique = effect()->GetTechniqueByName("RenderRGBLayerPremul");
-      break;
-    case SURFACE_SINGLE_CHANNEL_ALPHA:
-      technique = effect()->GetTechniqueByName("RenderRGBALayerPremul");
-      break;
-    default:
-      NS_ERROR("Unknown mode");
-      return;
-    }
+  switch (mCurrentSurfaceMode) {
+  case SURFACE_COMPONENT_ALPHA:
+    technique = SelectShader(SHADER_COMPONENT_ALPHA | LoadMaskTexture());
+    break;
+  case SURFACE_OPAQUE:
+    technique = SelectShader(SHADER_RGB | SHADER_PREMUL | LoadMaskTexture());
+    break;
+  case SURFACE_SINGLE_CHANNEL_ALPHA:
+    technique = SelectShader(SHADER_RGBA | SHADER_PREMUL | LoadMaskTexture());
+    break;
+  default:
+    NS_ERROR("Unknown mode");
+    return;
   }
 
   nsIntRegionRectIterator iter(mVisibleRegion);
 
   const nsIntRect *iterRect;
   if (mSRView) {
     effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(mSRView);
   }
@@ -647,22 +630,17 @@ ShadowThebesLayerD3D10::RenderLayer()
   HRESULT hr = device()->CreateShaderResourceView(mTexture, NULL, getter_AddRefs(srView));
   if (FAILED(hr)) {
       NS_WARNING("Failed to create shader resource view for ThebesLayerD3D10.");
   }
 
 
   SetEffectTransformAndOpacity();
 
-  ID3D10EffectTechnique *technique;
-  if (LoadMaskTexture()) {
-    technique = effect()->GetTechniqueByName("RenderRGBLayerPremulMask");
-  } else {
-    technique = effect()->GetTechniqueByName("RenderRGBLayerPremul");
-  }
+  ID3D10EffectTechnique *technique = SelectShader(SHADER_RGB | SHADER_PREMUL | LoadMaskTexture());
 
   effect()->GetVariableByName("tRGB")->AsShaderResource()->SetResource(srView);
 
   nsIntRect textureRect = GetVisibleRegion().GetBounds();
 
   nsIntRegionRectIterator iter(mVisibleRegion);
   while (const nsIntRect *iterRect = iter.Next()) {
     effect()->GetVariableByName("vLayerQuad")->AsVector()->SetFloatVector(