Bug 1689978 - Fill out-of-bounds texelFetchPtr with zeroes rather than clamping. r=jrmuizel
authorLee Salzman <lsalzman@mozilla.com>
Wed, 01 Dec 2021 08:24:06 +0000
changeset 600753 2d1f7196e88a3aee7d084225f4381b6952b545ce
parent 600752 1893ac7500dc3ed86746da17cb1668c147705b23
child 600754 8c5b98f3c6077c8d91d3b7a1e4c45702921be63c
push id153947
push userlsalzman@mozilla.com
push dateWed, 01 Dec 2021 09:50:08 +0000
treeherderautoland@2d1f7196e88a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjrmuizel
bugs1689978
milestone96.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1689978 - Fill out-of-bounds texelFetchPtr with zeroes rather than clamping. r=jrmuizel Attempting to just clamping the base address returning from texelFetchPtr might be causing some crashes in the case the texture is actually smaller than the offset area. Instead, switch out the sampler with a zero buffer to ensure we have something sane to sample without having to do slow bounds checking on everything. Differential Revision: https://phabricator.services.mozilla.com/D132508
gfx/wr/glsl-to-cxx/src/lib.rs
gfx/wr/swgl/src/gl.cc
gfx/wr/swgl/src/gl_defs.h
gfx/wr/swgl/src/texture.h
--- a/gfx/wr/glsl-to-cxx/src/lib.rs
+++ b/gfx/wr/glsl-to-cxx/src/lib.rs
@@ -2598,34 +2598,19 @@ pub fn has_conditional_return(state: &mu
 
 fn define_texel_fetch_ptr(
     state: &OutputState,
     base_sym: &hir::Symbol,
     sampler_sym: &hir::Symbol,
     offsets: &hir::TexelFetchOffsets,
 ) {
     show_indent(state);
-    let ptr_type = if let hir::SymDecl::Global(_, _, ty, _) = &sampler_sym.decl {
-        if symbol_run_class(&base_sym.decl, state.vector_mask) == hir::RunClass::Scalar {
-            match ty.kind {
-                hir::TypeKind::Sampler2D
-                | hir::TypeKind::Sampler2DRect => "vec4_scalar*",
-                hir::TypeKind::ISampler2D => "ivec4_scalar*",
-                _ => panic!(),
-            }
-        } else {
-            "I32"
-        }
-    } else {
-        panic!();
-    };
     write!(
         state,
-        "{} {}_{}_fetch = texelFetchPtr({}, {}, {}, {}, {}, {});\n",
-        ptr_type,
+        "auto {}_{}_fetch = texelFetchPtr({}, {}, {}, {}, {}, {});\n",
         sampler_sym.name,
         base_sym.name,
         sampler_sym.name,
         base_sym.name,
         offsets.min_x,
         offsets.max_x,
         offsets.min_y,
         offsets.max_y,
--- a/gfx/wr/swgl/src/gl.cc
+++ b/gfx/wr/swgl/src/gl.cc
@@ -1118,19 +1118,17 @@ void Disable(GLenum cap) {
 GLenum GetError() {
   GLenum error = ctx->last_error;
   ctx->last_error = GL_NO_ERROR;
   return error;
 }
 
 // Sets the error status to out-of-memory to indicate that a buffer
 // or texture re-allocation failed.
-static void out_of_memory() {
-  ctx->last_error = GL_OUT_OF_MEMORY;
-}
+static void out_of_memory() { ctx->last_error = GL_OUT_OF_MEMORY; }
 
 static const char* const extensions[] = {
     "GL_ARB_blend_func_extended",
     "GL_ARB_clear_texture",
     "GL_ARB_copy_image",
     "GL_ARB_draw_instanced",
     "GL_ARB_explicit_attrib_location",
     "GL_ARB_instanced_arrays",
@@ -1171,16 +1169,22 @@ void GetIntegerv(GLenum pname, GLint* pa
       params[0] = sizeof(extensions) / sizeof(extensions[0]);
       break;
     case GL_MAJOR_VERSION:
       params[0] = 3;
       break;
     case GL_MINOR_VERSION:
       params[0] = 2;
       break;
+    case GL_MIN_PROGRAM_TEXEL_OFFSET:
+      params[0] = 0;
+      break;
+    case GL_MAX_PROGRAM_TEXEL_OFFSET:
+      params[0] = MAX_TEXEL_OFFSET;
+      break;
     default:
       debugf("unhandled glGetIntegerv parameter %x\n", pname);
       assert(false);
   }
 }
 
 void GetBooleanv(GLenum pname, GLboolean* params) {
   assert(params);
@@ -2818,17 +2822,17 @@ void DestroyContext(Context* c) {
     return;
   }
   if (ctx == c) {
     MakeCurrent(nullptr);
   }
   delete c;
 }
 
-size_t ReportMemory(Context *ctx, size_t (*size_of_op)(void*)) {
+size_t ReportMemory(Context* ctx, size_t (*size_of_op)(void*)) {
   size_t size = 0;
   if (ctx) {
     for (auto& t : ctx->textures) {
       if (t && t->should_free()) {
         size += size_of_op(t->buf);
       }
     }
   }
--- a/gfx/wr/swgl/src/gl_defs.h
+++ b/gfx/wr/swgl/src/gl_defs.h
@@ -126,16 +126,18 @@ typedef intptr_t GLintptr;
 #define GL_TEXTURE12 0x84CC
 #define GL_TEXTURE13 0x84CD
 #define GL_TEXTURE14 0x84CE
 #define GL_TEXTURE15 0x84CF
 #define GL_MAX_TEXTURE_UNITS 0x84E2
 #define GL_MAX_TEXTURE_IMAGE_UNITS 0x8872
 #define GL_MAX_TEXTURE_SIZE 0x0D33
 #define GL_MAX_ARRAY_TEXTURE_LAYERS 0x88FF
+#define GL_MIN_PROGRAM_TEXEL_OFFSET 0x8904
+#define GL_MAX_PROGRAM_TEXEL_OFFSET 0x8905
 
 #define GL_VERTEX_SHADER 0x8B31
 #define GL_FRAGMENT_SHADER 0x8B30
 
 #define GL_BLEND 0x0BE2
 #define GL_ZERO 0
 #define GL_ONE 1
 #define GL_SRC_COLOR 0x0300
--- a/gfx/wr/swgl/src/texture.h
+++ b/gfx/wr/swgl/src/texture.h
@@ -155,21 +155,23 @@ SI Float fetchOffsetsR16(S sampler, I32 
 }
 
 template <typename S>
 vec4 texelFetchR16(S sampler, ivec2 P) {
   I32 offset = P.x + P.y * sampler->stride;
   return vec4(fetchOffsetsR16(sampler, offset), 0.0f, 0.0f, 1.0f);
 }
 
-template <typename S>
-SI vec4 fetchOffsetsFloat(S sampler, I32 offset) {
-  return pixel_float_to_vec4(
-      *(Float*)&sampler->buf[offset.x], *(Float*)&sampler->buf[offset.y],
-      *(Float*)&sampler->buf[offset.z], *(Float*)&sampler->buf[offset.w]);
+SI vec4 fetchOffsetsFloat(const uint32_t* buf, I32 offset) {
+  return pixel_float_to_vec4(*(Float*)&buf[offset.x], *(Float*)&buf[offset.y],
+                             *(Float*)&buf[offset.z], *(Float*)&buf[offset.w]);
+}
+
+SI vec4 fetchOffsetsFloat(samplerCommon* sampler, I32 offset) {
+  return fetchOffsetsFloat(sampler->buf, offset);
 }
 
 vec4 texelFetchFloat(sampler2D sampler, ivec2 P) {
   I32 offset = P.x * 4 + P.y * sampler->stride;
   return fetchOffsetsFloat(sampler, offset);
 }
 
 template <typename S>
@@ -302,21 +304,23 @@ vec4 texelFetch(sampler2DRect sampler, i
     case TextureFormat::YUV422:
       return texelFetchYUV422(sampler, P);
     default:
       assert(false);
       return vec4();
   }
 }
 
-template <typename S>
-SI ivec4 fetchOffsetsInt(S sampler, I32 offset) {
-  return pixel_int_to_ivec4(
-      *(I32*)&sampler->buf[offset.x], *(I32*)&sampler->buf[offset.y],
-      *(I32*)&sampler->buf[offset.z], *(I32*)&sampler->buf[offset.w]);
+SI ivec4 fetchOffsetsInt(const uint32_t* buf, I32 offset) {
+  return pixel_int_to_ivec4(*(I32*)&buf[offset.x], *(I32*)&buf[offset.y],
+                            *(I32*)&buf[offset.z], *(I32*)&buf[offset.w]);
+}
+
+SI ivec4 fetchOffsetsInt(samplerCommon* sampler, I32 offset) {
+  return fetchOffsetsInt(sampler->buf, offset);
 }
 
 ivec4 texelFetch(isampler2D sampler, ivec2 P, int lod) {
   assert(lod == 0);
   P = clamp2D(P, sampler);
   assert(sampler->format == TextureFormat::RGBA32I);
   I32 offset = P.x * 4 + P.y * sampler->stride;
   return fetchOffsetsInt(sampler, offset);
@@ -324,53 +328,80 @@ ivec4 texelFetch(isampler2D sampler, ive
 
 ivec4_scalar texelFetch(isampler2D sampler, ivec2_scalar P, int lod) {
   assert(lod == 0);
   P = clamp2D(P, sampler);
   assert(sampler->format == TextureFormat::RGBA32I);
   return *(ivec4_scalar*)&sampler->buf[P.x * 4 + P.y * sampler->stride];
 }
 
-SI vec4_scalar* texelFetchPtr(sampler2D sampler, ivec2_scalar P, int min_x,
-                              int max_x, int min_y, int max_y) {
-  P.x = min(max(P.x, -min_x), int(sampler->width) - 1 - max_x);
-  P.y = min(max(P.y, -min_y), int(sampler->height) - 1 - max_y);
-  assert(sampler->format == TextureFormat::RGBA32F);
-  return (vec4_scalar*)&sampler->buf[P.x * 4 + P.y * sampler->stride];
-}
+constexpr int MAX_TEXEL_OFFSET = 8;
 
-SI ivec4_scalar* texelFetchPtr(isampler2D sampler, ivec2_scalar P, int min_x,
-                               int max_x, int min_y, int max_y) {
-  P.x = min(max(P.x, -min_x), int(sampler->width) - 1 - max_x);
-  P.y = min(max(P.y, -min_y), int(sampler->height) - 1 - max_y);
-  assert(sampler->format == TextureFormat::RGBA32I);
-  return (ivec4_scalar*)&sampler->buf[P.x * 4 + P.y * sampler->stride];
-}
+// Fill texelFetchOffset outside the valid texture bounds with zeroes. The
+// stride will be set to 0 so that only one row of zeroes is needed.
+static const uint32_t
+    zeroFetchBuf[MAX_TEXEL_OFFSET * sizeof(Float) / sizeof(uint32_t)] = {0};
+
+struct FetchScalar {
+  const uint32_t* buf;
+  uint32_t stride;
+};
 
 template <typename S>
-SI I32 texelFetchPtr(S sampler, ivec2 P, int min_x, int max_x, int min_y,
-                     int max_y) {
-  P.x = clampCoord(P.x, int(sampler->width) - max_x, -min_x);
-  P.y = clampCoord(P.y, int(sampler->height) - max_y, -min_y);
-  return P.x * 4 + P.y * sampler->stride;
+SI FetchScalar texelFetchPtr(S sampler, ivec2_scalar P, int min_x, int max_x,
+                             int min_y, int max_y) {
+  assert(max_x < MAX_TEXEL_OFFSET);
+  if (P.x < -min_x || P.x >= int(sampler->width) - max_x || P.y < -min_y ||
+      P.y >= int(sampler->height) - max_y) {
+    return FetchScalar{zeroFetchBuf, 0};
+  }
+  return FetchScalar{&sampler->buf[P.x * 4 + P.y * sampler->stride],
+                     sampler->stride};
+}
+
+SI vec4_scalar texelFetchUnchecked(sampler2D sampler, FetchScalar ptr, int x,
+                                   int y = 0) {
+  assert(sampler->format == TextureFormat::RGBA32F);
+  return *(vec4_scalar*)&ptr.buf[x * 4 + y * ptr.stride];
+}
+
+SI ivec4_scalar texelFetchUnchecked(isampler2D sampler, FetchScalar ptr, int x,
+                                    int y = 0) {
+  assert(sampler->format == TextureFormat::RGBA32I);
+  return *(ivec4_scalar*)&ptr.buf[x * 4 + y * ptr.stride];
 }
 
-template <typename S, typename P>
-SI P texelFetchUnchecked(S sampler, P* ptr, int x, int y = 0) {
-  return ptr[x + y * (sampler->stride >> 2)];
+struct FetchVector {
+  const uint32_t* buf;
+  I32 offset;
+  uint32_t stride;
+};
+
+template <typename S>
+SI FetchVector texelFetchPtr(S sampler, ivec2 P, int min_x, int max_x,
+                             int min_y, int max_y) {
+  assert(max_x < MAX_TEXEL_OFFSET);
+  if (test_any(P.x < -min_x || P.x >= int(sampler->width) - max_x ||
+               P.y < -min_y || P.y >= int(sampler->height) - max_y)) {
+    return FetchVector{zeroFetchBuf, I32(0), 0};
+  }
+  return FetchVector{sampler->buf, P.x * 4 + P.y * sampler->stride,
+                     sampler->stride};
 }
 
-SI vec4 texelFetchUnchecked(sampler2D sampler, I32 offset, int x, int y = 0) {
+SI vec4 texelFetchUnchecked(sampler2D sampler, FetchVector ptr, int x,
+                            int y = 0) {
   assert(sampler->format == TextureFormat::RGBA32F);
-  return fetchOffsetsFloat(sampler, offset + (x * 4 + y * sampler->stride));
+  return fetchOffsetsFloat(&ptr.buf[x * 4 + y * ptr.stride], ptr.offset);
 }
 
-SI ivec4 texelFetchUnchecked(isampler2D sampler, I32 offset, int x, int y = 0) {
+SI ivec4 texelFetchUnchecked(isampler2D sampler, FetchVector ptr, int x,
+                             int y = 0) {
   assert(sampler->format == TextureFormat::RGBA32I);
-  return fetchOffsetsInt(sampler, offset + (x * 4 + y * sampler->stride));
+  return fetchOffsetsInt(&ptr.buf[x * 4 + y * ptr.stride], ptr.offset);
 }
 
 #define texelFetchOffset(sampler, P, lod, offset) \
   texelFetch(sampler, (P) + (offset), lod)
 
 // Scale texture coords for quantization, subtract offset for filtering
 // (assuming coords already offset to texel centers), and round to nearest
 // 1/scale increment