This was found by enabling the float-cast-overflow check in UBSan and running existing fuzzers. This type of issue can create inconsistencies across platforms, architectures and optimization levels.

Found with m-c 20220718-d39e3f5f3cbb

To enable this check add the following to your mozconfig:

ac_add_options --enable-undefined-sanitizer="float-cast-overflow"

src/swgl_ext.h:547:16: runtime error: -nan is outside the range of representable values of type 'int'
    #0 0x7f34c04b9722 in int spanNeedsScale<glsl::vec2>(int, glsl::vec2) src/gfx/wr/swgl/src/swgl_ext.h:547:16
    #1 0x7f34c04b9722 in LinearFilter needsTextureLinear<glsl::sampler2D_impl*, glsl::vec2>(glsl::sampler2D_impl*, glsl::vec2, int) src/gfx/wr/swgl/src/swgl_ext.h:565:19
    #2 0x7f34c04b9722 in int blendTextureLinearRepeat<true, glsl::sampler2D_impl*, NoColor, unsigned int>(glsl::sampler2D_impl*, glsl::vec2, int, glsl::vec2_scalar const&, glsl::vec4_scalar const&, glsl::vec4_scalar const&, NoColor, unsigned int*) src/gfx/wr/swgl/src/swgl_ext.h:716:7
    #3 0x7f34c0628c61 in brush_image_ALPHA_PASS_ANTIALIASING_REPETITION_TEXTURE_2D_frag::swgl_drawSpanRGBA8() src/objdir-ff-ubsan/x86_64-unknown-linux-gnu/release/build/swgl-e6f615b0ed997614/out/brush_image_ALPHA_PASS_ANTIALIASING_REPETITION_TEXTURE_2D.h:962:2
    #4 0x7f34c061d8e1 in brush_image_ALPHA_PASS_ANTIALIASING_REPETITION_TEXTURE_2D_frag::draw_span_RGBA8(glsl::FragmentShaderImpl*) src/objdir-ff-ubsan/x86_64-unknown-linux-gnu/release/build/swgl-e6f615b0ed997614/out/brush_image_ALPHA_PASS_ANTIALIASING_REPETITION_TEXTURE_2D.h:1010:28
    #5 0x7f34c096fce0 in glsl::FragmentShaderImpl::draw_span(unsigned int*, int) src/gfx/wr/swgl/src/program.h:168:12
    #6 0x7f34c096fce0 in void draw_depth_span<unsigned int>(unsigned int, unsigned int*, DepthCursor&) src/gfx/wr/swgl/src/rasterize.h:628:38
    #7 0x7f34c096fce0 in void draw_quad_spans<unsigned int>(int, glsl::vec2_scalar*, unsigned int, glsl::vec3*, Texture&, Texture&, ClipRect const&) src/gfx/wr/swgl/src/rasterize.h:1023:13
    #8 0x7f34c0458d9c in draw_quad(int, Texture&, Texture&) src/gfx/wr/swgl/src/rasterize.h:1621:5
    #9 0x7f34c0457861 in void draw_elements<unsigned short>(int, int, unsigned long, VertexArray&, Texture&, Texture&) src/gfx/wr/swgl/src/rasterize.h:1651:5
    #10 0x7f34c04574ac in DrawElementsInstanced src/gfx/wr/swgl/src/gl.cc:2744:7
    #11 0x7f34c0059691 in webrender::device::gl::Device::draw_indexed_triangles_instanced_u16::h1280b683ca83c461 src/gfx/wr/webrender/src/device/gl.rs:3633:9
    #12 0x7f34bf961950 in webrender::renderer::Renderer::draw_instanced_batch::hf5f12125a6dc5753 src/gfx/wr/webrender/src/renderer/mod.rs:2513:17
    #13 0x7f34bfdf70dc in webrender::renderer::Renderer::draw_alpha_batch_container::hd877b341db1f8a6e src/gfx/wr/webrender/src/renderer/mod.rs:3168:17
    #14 0x7f34bfe056d5 in webrender::renderer::Renderer::draw_picture_cache_target::hc874798bdc8a3658 src/gfx/wr/webrender/src/renderer/mod.rs:2955:17
    #15 0x7f34bfe056d5 in webrender::renderer::Renderer::draw_frame::h2bad2ceb21f936dc src/gfx/wr/webrender/src/renderer/mod.rs:4899:21
    #16 0x7f34bfde0f65 in webrender::renderer::Renderer::render_impl::h4dd846038a698e3f src/gfx/wr/webrender/src/renderer/mod.rs:2015:17
    #17 0x7f34bfddd75e in webrender::renderer::Renderer::render::heba9b507cc2c540c src/gfx/wr/webrender/src/renderer/mod.rs:1737:30
    #18 0x7f34bee65f7d in wr_renderer_render src/gfx/webrender_bindings/src/bindings.rs:620:11
    #19 0x7f34b03bbbde in mozilla::wr::RendererOGL::UpdateAndRender(mozilla::Maybe<mozilla::gfx::IntSizeTyped<mozilla::gfx::UnknownUnits> > const&, mozilla::Maybe<mozilla::wr::ImageFormat> const&, mozilla::Maybe<mozilla::Range<unsigned char> > const&, bool*, mozilla::wr::RendererStats*) src/gfx/webrender_bindings/RendererOGL.cpp:185:8
    #20 0x7f34b03ba3c2 in mozilla::wr::RenderThread::UpdateAndRender(mozilla::wr::WrWindowId, mozilla::layers::BaseTransactionId<mozilla::VsyncIdType> const&, mozilla::TimeStamp const&, bool, mozilla::Maybe<mozilla::gfx::IntSizeTyped<mozilla::gfx::UnknownUnits> > const&, mozilla::Maybe<mozilla::wr::ImageFormat> const&, mozilla::Maybe<mozilla::Range<unsigned char> > const&, bool*) src/gfx/webrender_bindings/RenderThread.cpp:560:31
    #21 0x7f34b03b961d in mozilla::wr::RenderThread::HandleFrameOneDoc(mozilla::wr::WrWindowId, bool) src/gfx/webrender_bindings/RenderThread.cpp:410:3
    #22 0x7f34b03db566 in decltype(*(fp).*fp0(Get<0ul>(fp1).PassAsParameter(), Get<1ul>(fp1).PassAsParameter())) mozilla::detail::RunnableMethodArguments<mozilla::wr::WrWindowId, bool>::applyImpl<mozilla::wr::RenderThread, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool), StoreCopyPassByConstLRef<mozilla::wr::WrWindowId>, StoreCopyPassByConstLRef<bool>, 0ul, 1ul>(mozilla::wr::RenderThread*, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool), mozilla::Tuple<StoreCopyPassByConstLRef<mozilla::wr::WrWindowId>, StoreCopyPassByConstLRef<bool> >&, std::integer_sequence<unsigned long, 0ul, 1ul>) src/objdir-ff-ubsan/dist/include/nsThreadUtils.h:1147:12
    #23 0x7f34b03db3ab in decltype(applyImpl(fp, fp0, *(this).mArguments, std::integer_sequence<unsigned long, 0ul, 1ul>{})) mozilla::detail::RunnableMethodArguments<mozilla::wr::WrWindowId, bool>::apply<mozilla::wr::RenderThread, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool)>(mozilla::wr::RenderThread*, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool)) src/objdir-ff-ubsan/dist/include/nsThreadUtils.h:1153:12
    #24 0x7f34b03db3ab in mozilla::detail::RunnableMethodImpl<mozilla::wr::RenderThread*, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool), true, (mozilla::RunnableKind)0, mozilla::wr::WrWindowId, bool>::Run() src/objdir-ff-ubsan/dist/include/nsThreadUtils.h:1200:13
    #25 0x7f34ad8a68ae in nsThread::ProcessNextEvent(bool, bool*) src/xpcom/threads/nsThread.cpp:1199:16
    #26 0x7f34ad8aed24 in NS_ProcessNextEvent(nsIThread*, bool) src/xpcom/threads/nsThreadUtils.cpp:465:10
    #27 0x7f34aefeba84 in mozilla::ipc::MessagePumpForNonMainThreads::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:330:5
    #28 0x7f34aee5dd61 in MessageLoop::RunInternal() src/ipc/chromium/src/base/message_loop.cc:380:10
    #29 0x7f34aee5dd61 in MessageLoop::RunHandler() src/ipc/chromium/src/base/message_loop.cc:373:3
    #30 0x7f34aee5dd61 in MessageLoop::Run() src/ipc/chromium/src/base/message_loop.cc:355:3
    #31 0x7f34ad89ea12 in nsThread::ThreadFunc(void*) src/xpcom/threads/nsThread.cpp:384:10
    #32 0x7f34d9918bbe in _pt_root src/nsprpub/pr/src/pthreads/ptthread.c:201:5
    #33 0x7f34d96386da in start_thread /build/glibc-CVJwZb/glibc-2.27/nptl/pthread_create.c:463
    #34 0x7f34d861661e in __clone /build/glibc-CVJwZb/glibc-2.27/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95

A Pernosco session is available here: https://pernos.co/debug/Kz5TbTs5QtiXU3IIL68Gvw/index.html