This was found by enabling the float-cast-overflow check in UBSan and fuzzing. This type of issue can create inconsistencies across platforms, architectures and optimization levels.

Found with m-c 20220304-ee4f4beb8186

To enable this check add the following to your mozconfig:

ac_add_options --enable-undefined-sanitizer="float-cast-overflow"

src/layout/generic/nsFloatManager.cpp:2807:10: runtime error: -nan is outside the range of representable values of type 'int'
    #0 0x7fa218ea75c5 in nsFloatManager::ShapeInfo::ComputeEllipseLineInterceptDiff(int, int, int, int, int, int, int, int) src/layout/generic/nsFloatManager.cpp
    #1 0x7fa218ea7264 in nsFloatManager::EllipseShapeInfo::LineEdge(int, int, bool) const src/layout/generic/nsFloatManager.cpp:929:24
    #2 0x7fa218ea778d in nsFloatManager::EllipseShapeInfo::LineLeft(int, int) const src/layout/generic/nsFloatManager.cpp:1004:10
    #3 0x7fa218ea4291 in nsFloatManager::FloatInfo::LineLeft(nsFloatManager::ShapeType, int, int) const src/layout/generic/nsFloatManager.cpp:2350:43
    #4 0x7fa218ea38a3 in nsFloatManager::GetFlowArea(mozilla::WritingMode, int, int, nsFloatManager::BandInfoType, nsFloatManager::ShapeType, mozilla::LogicalRect, nsFloatManager::SavedState*, nsSize const&) const src/layout/generic/nsFloatManager.cpp:224:16
    #5 0x7fa218dd2fb8 in mozilla::BlockReflowState::GetFloatAvailableSpaceWithState(int, nsFloatManager::ShapeType, nsFloatManager::SavedState*) const src/layout/generic/BlockReflowState.cpp:308:43
    #6 0x7fa218e3ccef in mozilla::BlockReflowState::GetFloatAvailableSpace(int) const src/layout/generic/BlockReflowState.h:119:12
    #7 0x7fa218e3ccef in mozilla::BlockReflowState::GetFloatAvailableSpace() const src/layout/generic/BlockReflowState.h:113:12
    #8 0x7fa218e3ccef in nsBlockFrame::DoReflowInlineFrames(mozilla::BlockReflowState&, nsLineLayout&, nsLineList_iterator, nsFlowAreaRect&, int&, nsFloatManager::SavedState*, bool*, LineReflowStatus*, bool) src/layout/generic/nsBlockFrame.cpp:4464:37
    #9 0x7fa218e379db in nsBlockFrame::ReflowInlineFrames(mozilla::BlockReflowState&, nsLineList_iterator, bool*) src/layout/generic/nsBlockFrame.cpp:4240:9
    #10 0x7fa218e31f38 in nsBlockFrame::ReflowLine(mozilla::BlockReflowState&, nsLineList_iterator, bool*) src/layout/generic/nsBlockFrame.cpp:3227:5
    #11 0x7fa218e2a99e in nsBlockFrame::ReflowDirtyLines(mozilla::BlockReflowState&) src/layout/generic/nsBlockFrame.cpp:2761:7
    #12 0x7fa218e2501d in nsBlockFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/nsBlockFrame.cpp:1394:3
    #13 0x7fa218e3ac01 in nsBlockReflowContext::ReflowBlock(mozilla::LogicalRect const&, bool, nsCollapsingMargin&, int, bool, nsLineBox*, mozilla::ReflowInput&, nsReflowStatus&, mozilla::BlockReflowState&) src/layout/generic/nsBlockReflowContext.cpp:288:11
    #14 0x7fa218e34bd9 in nsBlockFrame::ReflowBlockFrame(mozilla::BlockReflowState&, nsLineList_iterator, bool*) src/layout/generic/nsBlockFrame.cpp:3874:11
    #15 0x7fa218e32117 in nsBlockFrame::ReflowLine(mozilla::BlockReflowState&, nsLineList_iterator, bool*) src/layout/generic/nsBlockFrame.cpp:3224:5
    #16 0x7fa218e2a99e in nsBlockFrame::ReflowDirtyLines(mozilla::BlockReflowState&) src/layout/generic/nsBlockFrame.cpp:2761:7
    #17 0x7fa218e2501d in nsBlockFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/nsBlockFrame.cpp:1394:3
    #18 0x7fa218e53b9a in nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, mozilla::WritingMode const&, mozilla::LogicalPoint const&, nsSize const&, nsIFrame::ReflowChildFlags, nsReflowStatus&, nsOverflowContinuationTracker*) src/layout/generic/nsContainerFrame.cpp:1005:14
    #19 0x7fa218e526b7 in nsCanvasFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/nsCanvasFrame.cpp:791:7
    #20 0x7fa218e53b9a in nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, mozilla::WritingMode const&, mozilla::LogicalPoint const&, nsSize const&, nsIFrame::ReflowChildFlags, nsReflowStatus&, nsOverflowContinuationTracker*) src/layout/generic/nsContainerFrame.cpp:1005:14
    #21 0x7fa218edf6cd in nsHTMLScrollFrame::ReflowScrolledFrame(mozilla::ScrollReflowInput&, bool, bool, mozilla::ReflowOutput*) src/layout/generic/nsGfxScrollFrame.cpp:838:3
    #22 0x7fa218ee0999 in nsHTMLScrollFrame::ReflowContents(mozilla::ScrollReflowInput&, mozilla::ReflowOutput const&) src/layout/generic/nsGfxScrollFrame.cpp:973:3
    #23 0x7fa218ee5ac0 in nsHTMLScrollFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/nsGfxScrollFrame.cpp:1395:3
    #24 0x7fa218e179f2 in nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, int, int, nsIFrame::ReflowChildFlags, nsReflowStatus&, nsOverflowContinuationTracker*) src/layout/generic/nsContainerFrame.cpp:1045:14
    #25 0x7fa218e17234 in mozilla::ViewportFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) src/layout/generic/ViewportFrame.cpp:374:7
    #26 0x7fa218c46c1a in mozilla::PresShell::DoReflow(nsIFrame*, bool, mozilla::OverflowChangedTracker*) src/layout/base/PresShell.cpp:9553:11
    #27 0x7fa218c56e77 in mozilla::PresShell::ProcessReflowCommands(bool) src/layout/base/PresShell.cpp:9724:24
    #28 0x7fa218c55eab in mozilla::PresShell::DoFlushPendingNotifications(mozilla::ChangesToFlush) src/layout/base/PresShell.cpp:4302:11
    #29 0x7fa218c54d09 in mozilla::PresShell::FlushPendingNotifications(mozilla::ChangesToFlush) src/objdir-ff-ubsan/dist/include/mozilla/PresShell.h:1448:5
    #30 0x7fa218c54d09 in mozilla::PresShell::DoFlushPendingNotifications(mozilla::FlushType) src/layout/base/PresShell.cpp:4096:3
    #31 0x7fa215bf3527 in mozilla::PresShell::FlushPendingNotifications(mozilla::FlushType) src/objdir-ff-ubsan/dist/include/mozilla/PresShell.h:1439:5
    #32 0x7fa215bf3527 in mozilla::EventStateManager::FlushLayout(nsPresContext*) src/dom/events/EventStateManager.cpp:5911:16
    #33 0x7fa215bedbba in mozilla::EventStateManager::PreHandleEvent(nsPresContext*, mozilla::WidgetEvent*, nsIFrame*, nsIContent*, nsEventStatus*, nsIContent*) src/dom/events/EventStateManager.cpp:768:7
    #34 0x7fa218c71f02 in mozilla::PresShell::EventHandler::DispatchEvent(mozilla::EventStateManager*, mozilla::WidgetEvent*, bool, nsEventStatus*, nsIContent*) src/layout/base/PresShell.cpp:8163:39
    #35 0x7fa218c6cbc6 in mozilla::PresShell::EventHandler::HandleEventWithCurrentEventInfo(mozilla::WidgetEvent*, nsEventStatus*, bool, nsIContent*) src/layout/base/PresShell.cpp:8132:17
    #36 0x7fa218c6c12e in mozilla::PresShell::EventHandler::HandleEventUsingCoordinates(nsIFrame*, mozilla::WidgetGUIEvent*, nsEventStatus*, bool) src/layout/base/PresShell.cpp:7050:30
    #37 0x7fa218c6a65e in mozilla::PresShell::EventHandler::HandleEvent(nsIFrame*, mozilla::WidgetGUIEvent*, bool, nsEventStatus*) src/layout/base/PresShell.cpp:6853:12
    #38 0x7fa218c691e8 in mozilla::PresShell::HandleEvent(nsIFrame*, mozilla::WidgetGUIEvent*, bool, nsEventStatus*) src/layout/base/PresShell.cpp:6796:23
    #39 0x7fa2185c9f52 in nsViewManager::DispatchEvent(mozilla::WidgetGUIEvent*, nsView*, nsEventStatus*) src/view/nsViewManager.cpp:685:18
    #40 0x7fa2185c9b55 in nsView::HandleEvent(mozilla::WidgetGUIEvent*, bool) src/view/nsView.cpp:1129:9
    #41 0x7fa2186493e2 in mozilla::widget::PuppetWidget::DispatchEvent(mozilla::WidgetGUIEvent*, nsEventStatus&) src/widget/PuppetWidget.cpp:354:37
    #42 0x7fa212906951 in mozilla::layers::APZCCallbackHelper::DispatchWidgetEvent(mozilla::WidgetGUIEvent&) src/gfx/layers/apz/util/APZCCallbackHelper.cpp:502:21
    #43 0x7fa2179a3d88 in mozilla::dom::BrowserChild::DispatchWidgetEventViaAPZ(mozilla::WidgetGUIEvent&) src/dom/ipc/BrowserChild.cpp:1800:10
    #44 0x7fa2179a3d88 in mozilla::dom::BrowserChild::HandleRealMouseButtonEvent(mozilla::WidgetMouseEvent const&, mozilla::layers::ScrollableLayerGuid const&, unsigned long const&) src/dom/ipc/BrowserChild.cpp:1763:3
    #45 0x7fa2179a5e68 in mozilla::dom::BrowserChild::RecvRealMouseButtonEvent(mozilla::WidgetMouseEvent const&, mozilla::layers::ScrollableLayerGuid const&, unsigned long const&) src/dom/ipc/BrowserChild.cpp:1730:3
    #46 0x7fa2179a611c in mozilla::dom::BrowserChild::RecvSynthMouseMoveEvent(mozilla::WidgetMouseEvent const&, mozilla::layers::ScrollableLayerGuid const&, unsigned long const&) src/dom/ipc/BrowserChild.cpp:1695:8
    #47 0x7fa217b4f143 in mozilla::dom::PBrowserChild::OnMessageReceived(IPC::Message const&) src/objdir-ff-ubsan/ipc/ipdl/PBrowserChild.cpp:5236:56
    #48 0x7fa217bdd2d5 in mozilla::dom::PContentChild::OnMessageReceived(IPC::Message const&) src/objdir-ff-ubsan/ipc/ipdl/PContentChild.cpp:8320:32
    #49 0x7fa21794b797 in mozilla::dom::ContentChild::OnMessageReceived(IPC::Message const&) src/dom/ipc/ContentChild.cpp:3766:25
    #50 0x7fa211b6bdb8 in mozilla::ipc::MessageChannel::DispatchAsyncMessage(mozilla::ipc::ActorLifecycleProxy*, IPC::Message const&) src/ipc/glue/MessageChannel.cpp:1674:25
    #51 0x7fa211b697a6 in mozilla::ipc::MessageChannel::DispatchMessage(mozilla::ipc::ActorLifecycleProxy*, IPC::Message&&) src/ipc/glue/MessageChannel.cpp:1599:9
    #52 0x7fa211b6a1d6 in mozilla::ipc::MessageChannel::RunMessage(mozilla::ipc::ActorLifecycleProxy*, mozilla::ipc::MessageChannel::MessageTask&) src/ipc/glue/MessageChannel.cpp:1462:3
    #53 0x7fa211b6abf5 in mozilla::ipc::MessageChannel::MessageTask::Run() src/ipc/glue/MessageChannel.cpp:1496:14
    #54 0x7fa2104ce7ea in mozilla::RunnableTask::Run() src/xpcom/threads/TaskController.cpp:467:16
    #55 0x7fa21048ef4f in mozilla::TaskController::DoExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) src/xpcom/threads/TaskController.cpp:770:26
    #56 0x7fa21048c59e in mozilla::TaskController::ExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) src/xpcom/threads/TaskController.cpp:606:15
    #57 0x7fa21048ccf4 in mozilla::TaskController::ProcessPendingMTTask(bool) src/xpcom/threads/TaskController.cpp:390:36
    #58 0x7fa2104bffe1 in mozilla::TaskController::InitializeInternal()::$_0::operator()() const src/xpcom/threads/TaskController.cpp:124:37
    #59 0x7fa2104bffe1 in mozilla::detail::RunnableFunction<mozilla::TaskController::InitializeInternal()::$_0>::Run() src/xpcom/threads/nsThreadUtils.h:531:5
    #60 0x7fa2104ab123 in nsThread::ProcessNextEvent(bool, bool*) src/xpcom/threads/nsThread.cpp:1173:16
    #61 0x7fa2104b47e4 in NS_ProcessNextEvent(nsIThread*, bool) src/xpcom/threads/nsThreadUtils.cpp:467:10
    #62 0x7fa211b72c72 in mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:85:21
    #63 0x7fa211b74252 in mozilla::ipc::MessagePumpForChildProcess::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:268:30
    #64 0x7fa2119e3191 in MessageLoop::RunInternal() src/ipc/chromium/src/base/message_loop.cc:331:10
    #65 0x7fa2119e3191 in MessageLoop::RunHandler() src/ipc/chromium/src/base/message_loop.cc:324:3
    #66 0x7fa2119e3191 in MessageLoop::Run() src/ipc/chromium/src/base/message_loop.cc:306:3
    #67 0x7fa2186c3338 in nsBaseAppShell::Run() src/widget/nsBaseAppShell.cpp:137:27
    #68 0x7fa21d535af7 in XRE_RunAppShell() src/toolkit/xre/nsEmbedFunctions.cpp:870:20
    #69 0x7fa211b74231 in mozilla::ipc::MessagePumpForChildProcess::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:235:9
    #70 0x7fa2119e3191 in MessageLoop::RunInternal() src/ipc/chromium/src/base/message_loop.cc:331:10
    #71 0x7fa2119e3191 in MessageLoop::RunHandler() src/ipc/chromium/src/base/message_loop.cc:324:3
    #72 0x7fa2119e3191 in MessageLoop::Run() src/ipc/chromium/src/base/message_loop.cc:306:3
    #73 0x7fa21d534c4d in XRE_InitChildProcess(int, char**, XREChildData const*) src/toolkit/xre/nsEmbedFunctions.cpp:729:34
    #74 0x7fa21d549df0 in mozilla::BootstrapImpl::XRE_InitChildProcess(int, char**, XREChildData const*) src/toolkit/xre/Bootstrap.cpp:67:12
    #75 0x5616e6451495 in content_process_main(mozilla::Bootstrap*, int, char**) src/browser/app/../../ipc/contentproc/plugin-container.cpp:57:28
    #76 0x5616e64518a5 in main src/browser/app/nsBrowserApp.cpp:327:18
    #77 0x7fa2394abc86 in __libc_start_main /build/glibc-uZu3wS/glibc-2.27/csu/../csu/libc-start.c:310
    #78 0x5616e63a0578 in _start (src/objdir-ff-ubsan/dist/bin/firefox+0xf4578)

Please ni? me if a Pernosco session would be helpful.

The severity field is not set for this bug.
:emilio, could you have a look please?

For more information, please visit auto_nag documentation.

This issue is currently triggered while fuzzing with the 'float-cast-overflow' UBSan check enabled. This issue will need to be addressed before the check can be enabled by default.

If it requires too much effort to fix immediately please ni? me and let me know. If necessary it will be added to a suppression list. Thank you :)

aY * aY or aRadiusY * aRadiusY can lead to 32-bit integer overflow,
resulting a negative number. Then sqrt() will produce a nan on a negative
number.

We should compute the y/radiusY division, and then square the result.

https://hg.mozilla.org/mozilla-central/rev/297dc4b99fde