Bug 1689978 - Fill out-of-bounds texelFetchPtr with zeroes rather than clamping. r=jrmuizel, a=RyanVM
authorLee Salzman <lsalzman@mozilla.com>
Wed, 01 Dec 2021 08:24:06 +0000
changeset 657557 aae8c64f2a1953d448b394c97840ca6c8c6a3dee
parent 657556 3701f969c754cad3546d48dd68150e8b32cd18e8
child 657558 e599b5e0957ea4f11c93c3bd737e1b06c685f827
push id160
push userryanvm@gmail.com
push dateThu, 27 Jan 2022 15:25:39 +0000
treeherdermozilla-esr91@386b63ddcd80 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjrmuizel, RyanVM
bugs1689978
milestone91.6.0
Bug 1689978 - Fill out-of-bounds texelFetchPtr with zeroes rather than clamping. r=jrmuizel, a=RyanVM Attempting to just clamping the base address returning from texelFetchPtr might be causing some crashes in the case the texture is actually smaller than the offset area. Instead, switch out the sampler with a zero buffer to ensure we have something sane to sample without having to do slow bounds checking on everything. Differential Revision: https://phabricator.services.mozilla.com/D132508
gfx/wr/glsl-to-cxx/src/lib.rs
gfx/wr/swgl/src/gl.cc
gfx/wr/swgl/src/gl_defs.h
gfx/wr/swgl/src/texture.h
--- a/gfx/wr/glsl-to-cxx/src/lib.rs
+++ b/gfx/wr/glsl-to-cxx/src/lib.rs
@@ -2605,34 +2605,19 @@ pub fn has_conditional_return(state: &mu
 
 fn define_texel_fetch_ptr(
     state: &OutputState,
     base_sym: &hir::Symbol,
     sampler_sym: &hir::Symbol,
     offsets: &hir::TexelFetchOffsets,
 ) {
     show_indent(state);
-    let ptr_type = if let hir::SymDecl::Global(_, _, ty, _) = &sampler_sym.decl {
-        if symbol_run_class(&base_sym.decl, state.vector_mask) == hir::RunClass::Scalar {
-            match ty.kind {
-                hir::TypeKind::Sampler2D
-                | hir::TypeKind::Sampler2DRect => "vec4_scalar*",
-                hir::TypeKind::ISampler2D => "ivec4_scalar*",
-                _ => panic!(),
-            }
-        } else {
-            "I32"
-        }
-    } else {
-        panic!();
-    };
     write!(
         state,
-        "{} {}_{}_fetch = texelFetchPtr({}, {}, {}, {}, {}, {});\n",
-        ptr_type,
+        "auto {}_{}_fetch = texelFetchPtr({}, {}, {}, {}, {}, {});\n",
         sampler_sym.name,
         base_sym.name,
         sampler_sym.name,
         base_sym.name,
         offsets.min_x,
         offsets.max_x,
         offsets.min_y,
         offsets.max_y,
--- a/gfx/wr/swgl/src/gl.cc
+++ b/gfx/wr/swgl/src/gl.cc
@@ -1146,16 +1146,22 @@ void GetIntegerv(GLenum pname, GLint* pa
       params[0] = sizeof(extensions) / sizeof(extensions[0]);
       break;
     case GL_MAJOR_VERSION:
       params[0] = 3;
       break;
     case GL_MINOR_VERSION:
       params[0] = 2;
       break;
+    case GL_MIN_PROGRAM_TEXEL_OFFSET:
+      params[0] = 0;
+      break;
+    case GL_MAX_PROGRAM_TEXEL_OFFSET:
+      params[0] = MAX_TEXEL_OFFSET;
+      break;
     default:
       debugf("unhandled glGetIntegerv parameter %x\n", pname);
       assert(false);
   }
 }
 
 void GetBooleanv(GLenum pname, GLboolean* params) {
   assert(params);
@@ -2788,17 +2794,17 @@ void DestroyContext(Context* c) {
     return;
   }
   if (ctx == c) {
     MakeCurrent(nullptr);
   }
   delete c;
 }
 
-size_t ReportMemory(Context *ctx, size_t (*size_of_op)(void*)) {
+size_t ReportMemory(Context* ctx, size_t (*size_of_op)(void*)) {
   size_t size = 0;
   if (ctx) {
     for (auto& t : ctx->textures) {
       if (t && t->should_free()) {
         size += size_of_op(t->buf);
       }
     }
   }
--- a/gfx/wr/swgl/src/gl_defs.h
+++ b/gfx/wr/swgl/src/gl_defs.h
@@ -125,16 +125,18 @@ typedef intptr_t GLintptr;
 #define GL_TEXTURE12 0x84CC
 #define GL_TEXTURE13 0x84CD
 #define GL_TEXTURE14 0x84CE
 #define GL_TEXTURE15 0x84CF
 #define GL_MAX_TEXTURE_UNITS 0x84E2
 #define GL_MAX_TEXTURE_IMAGE_UNITS 0x8872
 #define GL_MAX_TEXTURE_SIZE 0x0D33
 #define GL_MAX_ARRAY_TEXTURE_LAYERS 0x88FF
+#define GL_MIN_PROGRAM_TEXEL_OFFSET 0x8904
+#define GL_MAX_PROGRAM_TEXEL_OFFSET 0x8905
 
 #define GL_VERTEX_SHADER 0x8B31
 #define GL_FRAGMENT_SHADER 0x8B30
 
 #define GL_BLEND 0x0BE2
 #define GL_ZERO 0
 #define GL_ONE 1
 #define GL_SRC_COLOR 0x0300
--- a/gfx/wr/swgl/src/texture.h
+++ b/gfx/wr/swgl/src/texture.h
@@ -155,21 +155,23 @@ SI Float fetchOffsetsR16(S sampler, I32 
 }
 
 template <typename S>
 vec4 texelFetchR16(S sampler, ivec2 P) {
   I32 offset = P.x + P.y * sampler->stride;
   return vec4(fetchOffsetsR16(sampler, offset), 0.0f, 0.0f, 1.0f);
 }
 
-template <typename S>
-SI vec4 fetchOffsetsFloat(S sampler, I32 offset) {
-  return pixel_float_to_vec4(
-      *(Float*)&sampler->buf[offset.x], *(Float*)&sampler->buf[offset.y],
-      *(Float*)&sampler->buf[offset.z], *(Float*)&sampler->buf[offset.w]);
+SI vec4 fetchOffsetsFloat(const uint32_t* buf, I32 offset) {
+  return pixel_float_to_vec4(*(Float*)&buf[offset.x], *(Float*)&buf[offset.y],
+                             *(Float*)&buf[offset.z], *(Float*)&buf[offset.w]);
+}
+
+SI vec4 fetchOffsetsFloat(samplerCommon* sampler, I32 offset) {
+  return fetchOffsetsFloat(sampler->buf, offset);
 }
 
 vec4 texelFetchFloat(sampler2D sampler, ivec2 P) {
   I32 offset = P.x * 4 + P.y * sampler->stride;
   return fetchOffsetsFloat(sampler, offset);
 }
 
 template <typename S>
@@ -302,21 +304,23 @@ vec4 texelFetch(sampler2DRect sampler, i
     case TextureFormat::YUV422:
       return texelFetchYUV422(sampler, P);
     default:
       assert(false);
       return vec4();
   }
 }
 
-template <typename S>
-SI ivec4 fetchOffsetsInt(S sampler, I32 offset) {
-  return pixel_int_to_ivec4(
-      *(I32*)&sampler->buf[offset.x], *(I32*)&sampler->buf[offset.y],
-      *(I32*)&sampler->buf[offset.z], *(I32*)&sampler->buf[offset.w]);
+SI ivec4 fetchOffsetsInt(const uint32_t* buf, I32 offset) {
+  return pixel_int_to_ivec4(*(I32*)&buf[offset.x], *(I32*)&buf[offset.y],
+                            *(I32*)&buf[offset.z], *(I32*)&buf[offset.w]);
+}
+
+SI ivec4 fetchOffsetsInt(samplerCommon* sampler, I32 offset) {
+  return fetchOffsetsInt(sampler->buf, offset);
 }
 
 ivec4 texelFetch(isampler2D sampler, ivec2 P, int lod) {
   assert(lod == 0);
   P = clamp2D(P, sampler);
   assert(sampler->format == TextureFormat::RGBA32I);
   I32 offset = P.x * 4 + P.y * sampler->stride;
   return fetchOffsetsInt(sampler, offset);
@@ -324,53 +328,80 @@ ivec4 texelFetch(isampler2D sampler, ive
 
 ivec4_scalar texelFetch(isampler2D sampler, ivec2_scalar P, int lod) {
   assert(lod == 0);
   P = clamp2D(P, sampler);
   assert(sampler->format == TextureFormat::RGBA32I);
   return *(ivec4_scalar*)&sampler->buf[P.x * 4 + P.y * sampler->stride];
 }
 
-SI vec4_scalar* texelFetchPtr(sampler2D sampler, ivec2_scalar P, int min_x,
-                              int max_x, int min_y, int max_y) {
-  P.x = min(max(P.x, -min_x), int(sampler->width) - 1 - max_x);
-  P.y = min(max(P.y, -min_y), int(sampler->height) - 1 - max_y);
-  assert(sampler->format == TextureFormat::RGBA32F);
-  return (vec4_scalar*)&sampler->buf[P.x * 4 + P.y * sampler->stride];
-}
+constexpr int MAX_TEXEL_OFFSET = 8;
 
-SI ivec4_scalar* texelFetchPtr(isampler2D sampler, ivec2_scalar P, int min_x,
-                               int max_x, int min_y, int max_y) {
-  P.x = min(max(P.x, -min_x), int(sampler->width) - 1 - max_x);
-  P.y = min(max(P.y, -min_y), int(sampler->height) - 1 - max_y);
-  assert(sampler->format == TextureFormat::RGBA32I);
-  return (ivec4_scalar*)&sampler->buf[P.x * 4 + P.y * sampler->stride];
-}
+// Fill texelFetchOffset outside the valid texture bounds with zeroes. The
+// stride will be set to 0 so that only one row of zeroes is needed.
+static const uint32_t
+    zeroFetchBuf[MAX_TEXEL_OFFSET * sizeof(Float) / sizeof(uint32_t)] = {0};
+
+struct FetchScalar {
+  const uint32_t* buf;
+  uint32_t stride;
+};
 
 template <typename S>
-SI I32 texelFetchPtr(S sampler, ivec2 P, int min_x, int max_x, int min_y,
-                     int max_y) {
-  P.x = clampCoord(P.x, int(sampler->width) - max_x, -min_x);
-  P.y = clampCoord(P.y, int(sampler->height) - max_y, -min_y);
-  return P.x * 4 + P.y * sampler->stride;
+SI FetchScalar texelFetchPtr(S sampler, ivec2_scalar P, int min_x, int max_x,
+                             int min_y, int max_y) {
+  assert(max_x < MAX_TEXEL_OFFSET);
+  if (P.x < -min_x || P.x >= int(sampler->width) - max_x || P.y < -min_y ||
+      P.y >= int(sampler->height) - max_y) {
+    return FetchScalar{zeroFetchBuf, 0};
+  }
+  return FetchScalar{&sampler->buf[P.x * 4 + P.y * sampler->stride],
+                     sampler->stride};
+}
+
+SI vec4_scalar texelFetchUnchecked(sampler2D sampler, FetchScalar ptr, int x,
+                                   int y = 0) {
+  assert(sampler->format == TextureFormat::RGBA32F);
+  return *(vec4_scalar*)&ptr.buf[x * 4 + y * ptr.stride];
+}
+
+SI ivec4_scalar texelFetchUnchecked(isampler2D sampler, FetchScalar ptr, int x,
+                                    int y = 0) {
+  assert(sampler->format == TextureFormat::RGBA32I);
+  return *(ivec4_scalar*)&ptr.buf[x * 4 + y * ptr.stride];
 }
 
-template <typename S, typename P>
-SI P texelFetchUnchecked(S sampler, P* ptr, int x, int y = 0) {
-  return ptr[x + y * (sampler->stride >> 2)];
+struct FetchVector {
+  const uint32_t* buf;
+  I32 offset;
+  uint32_t stride;
+};
+
+template <typename S>
+SI FetchVector texelFetchPtr(S sampler, ivec2 P, int min_x, int max_x,
+                             int min_y, int max_y) {
+  assert(max_x < MAX_TEXEL_OFFSET);
+  if (test_any(P.x < -min_x || P.x >= int(sampler->width) - max_x ||
+               P.y < -min_y || P.y >= int(sampler->height) - max_y)) {
+    return FetchVector{zeroFetchBuf, I32(0), 0};
+  }
+  return FetchVector{sampler->buf, P.x * 4 + P.y * sampler->stride,
+                     sampler->stride};
 }
 
-SI vec4 texelFetchUnchecked(sampler2D sampler, I32 offset, int x, int y = 0) {
+SI vec4 texelFetchUnchecked(sampler2D sampler, FetchVector ptr, int x,
+                            int y = 0) {
   assert(sampler->format == TextureFormat::RGBA32F);
-  return fetchOffsetsFloat(sampler, offset + (x * 4 + y * sampler->stride));
+  return fetchOffsetsFloat(&ptr.buf[x * 4 + y * ptr.stride], ptr.offset);
 }
 
-SI ivec4 texelFetchUnchecked(isampler2D sampler, I32 offset, int x, int y = 0) {
+SI ivec4 texelFetchUnchecked(isampler2D sampler, FetchVector ptr, int x,
+                             int y = 0) {
   assert(sampler->format == TextureFormat::RGBA32I);
-  return fetchOffsetsInt(sampler, offset + (x * 4 + y * sampler->stride));
+  return fetchOffsetsInt(&ptr.buf[x * 4 + y * ptr.stride], ptr.offset);
 }
 
 #define texelFetchOffset(sampler, P, lod, offset) \
   texelFetch(sampler, (P) + (offset), lod)
 
 // Scale texture coords for quantization, subtract offset for filtering
 // (assuming coords already offset to texel centers), and round to nearest
 // 1/scale increment