This was found by enabling the float-cast-overflow check in UBSan and running existing tests. This type of issue can create inconsistencies across platforms, architectures and optimization levels.

To enable this check add the following to your mozconfig:

ac_add_options --enable-undefined-sanitizer="float-cast-overflow"

This issue is found by running existing test: css/css-text/tab-size/tab-size-integer-003.html

INFO - TEST-START | /css/css-text/tab-size/tab-size-integer-003.html
INFO - PID 13753 | 1642044238941   Marionette      INFO    Testing http://web-platform.test:8000/css/css-text/tab-size/tab-size-integer-003.html == http://web-platform.test:8000/css/css-text/tab-size/tab-size-integer-001-ref.html
INFO - PID 13753 | /builds/worker/checkouts/gecko/layout/generic/nsTextFrame.cpp:8678:37: runtime error: -nan is outside the range of representable values of type 'int'
INFO - PID 13753 |     #0 0x7fa59073a28f in nsTextFrame::AddInlineMinISizeForFlow(gfxContext*, nsIFrame::InlineMinISizeData*, nsTextFrame::TextRunType) /builds/worker/checkouts/gecko/layout/generic/nsTextFrame.cpp:8678:37
INFO - PID 13753 |     #1 0x7fa59073ad76 in nsTextFrame::AddInlineMinISize(gfxContext*, nsIFrame::InlineMinISizeData*) /builds/worker/checkouts/gecko/layout/generic/nsTextFrame.cpp:8749:10
INFO - PID 13753 |     #2 0x7fa59048b07f in nsBlockFrame::GetMinISize(gfxContext*) /builds/worker/checkouts/gecko/layout/generic/nsBlockFrame.cpp:827:16
INFO - PID 13753 |     #3 0x7fa59060bffa in nsIFrame::ShrinkWidthToFit(gfxContext*, int, mozilla::EnumSet<mozilla::ComputeSizeFlag, unsigned char>) /builds/worker/checkouts/gecko/layout/generic/nsIFrame.cpp:6585:22
INFO - PID 13753 |     #4 0x7fa5904d428e in nsContainerFrame::ComputeAutoSize(gfxContext*, mozilla::WritingMode, mozilla::LogicalSize const&, int, mozilla::LogicalSize const&, mozilla::LogicalSize const&, mozilla::StyleSizeOverrides const&, mozilla::EnumSet<mozilla::ComputeSizeFlag, unsigned char>) /builds/worker/checkouts/gecko/layout/generic/nsContainerFrame.cpp:916:11
INFO - PID 13753 |     #5 0x7fa5904e6d9a in nsIFrame::ComputeSize(gfxContext*, mozilla::WritingMode, mozilla::LogicalSize const&, int, mozilla::LogicalSize const&, mozilla::LogicalSize const&, mozilla::StyleSizeOverrides const&, mozilla::EnumSet<mozilla::ComputeSizeFlag, unsigned char>) /builds/worker/checkouts/gecko/layout/generic/nsIFrame.cpp:6218:7
INFO - PID 13753 |     #6 0x7fa5904519e4 in mozilla::ReflowInput::InitAbsoluteConstraints(nsPresContext*, mozilla::ReflowInput const*, mozilla::LogicalSize const&, mozilla::LayoutFrameType) /builds/worker/checkouts/gecko/layout/generic/ReflowInput.cpp:1714:26
INFO - PID 13753 |     #7 0x7fa59044888b in mozilla::ReflowInput::InitConstraints(nsPresContext*, mozilla::Maybe<mozilla::LogicalSize> const&, mozilla::Maybe<mozilla::LogicalMargin> const&, mozilla::Maybe<mozilla::LogicalMargin> const&, mozilla::LayoutFrameType) /builds/worker/checkouts/gecko/layout/generic/ReflowInput.cpp:2302:7
INFO - PID 13753 |     #8 0x7fa5904427bd in mozilla::ReflowInput::Init(nsPresContext*, mozilla::Maybe<mozilla::LogicalSize> const&, mozilla::Maybe<mozilla::LogicalMargin> const&, mozilla::Maybe<mozilla::LogicalMargin> const&) /builds/worker/checkouts/gecko/layout/generic/ReflowInput.cpp:356:3
INFO - PID 13753 |     #9 0x7fa590443a95 in mozilla::ReflowInput::ReflowInput(nsPresContext*, mozilla::ReflowInput const&, nsIFrame*, mozilla::LogicalSize const&, mozilla::Maybe<mozilla::LogicalSize> const&, mozilla::EnumSet<mozilla::ReflowInput::InitFlag, unsigned char>, mozilla::StyleSizeOverrides const&, mozilla::EnumSet<mozilla::ComputeSizeFlag, unsigned char>) /builds/worker/checkouts/gecko/layout/generic/ReflowInput.cpp:216:5
INFO - PID 13753 |     #10 0x7fa590481265 in nsAbsoluteContainingBlock::ReflowAbsoluteFrame(nsIFrame*, nsPresContext*, mozilla::ReflowInput const&, nsRect const&, nsAbsoluteContainingBlock::AbsPosReflowFlags, nsIFrame*, nsReflowStatus&, mozilla::OverflowAreas*) /builds/worker/checkouts/gecko/layout/generic/nsAbsoluteContainingBlock.cpp:796:15
INFO - PID 13753 |     #11 0x7fa59047eded in nsAbsoluteContainingBlock::Reflow(nsContainerFrame*, nsPresContext*, mozilla::ReflowInput const&, nsReflowStatus&, nsRect const&, nsAbsoluteContainingBlock::AbsPosReflowFlags, mozilla::OverflowAreas*) /builds/worker/checkouts/gecko/layout/generic/nsAbsoluteContainingBlock.cpp:221:7
INFO - PID 13753 |     #12 0x7fa59048f874 in nsBlockFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) /builds/worker/checkouts/gecko/layout/generic/nsBlockFrame.cpp:1645:26
INFO - PID 13753 |     #13 0x7fa590481557 in nsAbsoluteContainingBlock::ReflowAbsoluteFrame(nsIFrame*, nsPresContext*, mozilla::ReflowInput const&, nsRect const&, nsAbsoluteContainingBlock::AbsPosReflowFlags, nsIFrame*, nsReflowStatus&, mozilla::OverflowAreas*) /builds/worker/checkouts/gecko/layout/generic/nsAbsoluteContainingBlock.cpp:813:14
INFO - PID 13753 |     #14 0x7fa59047eded in nsAbsoluteContainingBlock::Reflow(nsContainerFrame*, nsPresContext*, mozilla::ReflowInput const&, nsReflowStatus&, nsRect const&, nsAbsoluteContainingBlock::AbsPosReflowFlags, mozilla::OverflowAreas*) /builds/worker/checkouts/gecko/layout/generic/nsAbsoluteContainingBlock.cpp:221:7
INFO - PID 13753 |     #15 0x7fa59060d4b4 in nsIFrame::ReflowAbsoluteFrames(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&, bool) /builds/worker/checkouts/gecko/layout/generic/nsIFrame.cpp:6801:24
INFO - PID 13753 |     #16 0x7fa59050d2f7 in nsIFrame::FinishReflowWithAbsoluteFrames(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&, bool) /builds/worker/checkouts/gecko/layout/generic/nsIFrame.cpp:6768:3
INFO - PID 13753 |     #17 0x7fa5904c648d in nsCanvasFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) /builds/worker/checkouts/gecko/layout/generic/nsCanvasFrame.cpp:881:3
INFO - PID 13753 |     #18 0x7fa5904c6cbd in nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, mozilla::WritingMode const&, mozilla::LogicalPoint const&, nsSize const&, nsIFrame::ReflowChildFlags, nsReflowStatus&, nsOverflowContinuationTracker*) /builds/worker/checkouts/gecko/layout/generic/nsContainerFrame.cpp:1000:14
INFO - PID 13753 |     #19 0x7fa590551e56 in nsHTMLScrollFrame::ReflowScrolledFrame(mozilla::ScrollReflowInput&, bool, bool, mozilla::ReflowOutput*) /builds/worker/checkouts/gecko/layout/generic/nsGfxScrollFrame.cpp:839:3
INFO - PID 13753 |     #20 0x7fa590553459 in nsHTMLScrollFrame::ReflowContents(mozilla::ScrollReflowInput&, mozilla::ReflowOutput const&) /builds/worker/checkouts/gecko/layout/generic/nsGfxScrollFrame.cpp:974:3
INFO - PID 13753 |     #21 0x7fa590559e24 in nsHTMLScrollFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) /builds/worker/checkouts/gecko/layout/generic/nsGfxScrollFrame.cpp:1396:3
INFO - PID 13753 |     #22 0x7fa59047dc2b in nsContainerFrame::ReflowChild(nsIFrame*, nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, int, int, nsIFrame::ReflowChildFlags, nsReflowStatus&, nsOverflowContinuationTracker*) /builds/worker/checkouts/gecko/layout/generic/nsContainerFrame.cpp:1040:14
INFO - PID 13753 |     #23 0x7fa59047d259 in mozilla::ViewportFrame::Reflow(nsPresContext*, mozilla::ReflowOutput&, mozilla::ReflowInput const&, nsReflowStatus&) /builds/worker/checkouts/gecko/layout/generic/ViewportFrame.cpp:374:7
INFO - PID 13753 |     #24 0x7fa5902c0099 in mozilla::PresShell::DoReflow(nsIFrame*, bool, mozilla::OverflowChangedTracker*) /builds/worker/checkouts/gecko/layout/base/PresShell.cpp:9636:11
INFO - PID 13753 |     #25 0x7fa5902d1d97 in mozilla::PresShell::ProcessReflowCommands(bool) /builds/worker/checkouts/gecko/layout/base/PresShell.cpp:9807:24
INFO - PID 13753 |     #26 0x7fa5902d02a5 in mozilla::PresShell::DoFlushPendingNotifications(mozilla::ChangesToFlush) /builds/worker/checkouts/gecko/layout/base/PresShell.cpp:4299:11
INFO - PID 13753 |     #27 0x7fa58b8d6ede in FlushPendingNotifications /builds/worker/workspace/obj-build/dist/include/mozilla/PresShell.h:1452:5
INFO - PID 13753 |     #28 0x7fa58b8d6ede in mozilla::dom::Document::FlushPendingNotifications(mozilla::ChangesToFlush) /builds/worker/checkouts/gecko/dom/base/Document.cpp:10746:16
INFO - PID 13753 |     #29 0x7fa58a90c65c in nsDocLoader::DocLoaderIsEmpty(bool, mozilla::Maybe<nsresult> const&) /builds/worker/checkouts/gecko/uriloader/base/nsDocLoader.cpp:739:14
INFO - PID 13753 |     #30 0x7fa58a90f275 in nsDocLoader::OnStopRequest(nsIRequest*, nsresult) /builds/worker/checkouts/gecko/uriloader/base/nsDocLoader.cpp:677:5
INFO - PID 13753 |     #31 0x7fa5935aa60b in nsDocShell::OnStopRequest(nsIRequest*, nsresult) /builds/worker/checkouts/gecko/docshell/base/nsDocShell.cpp:13540:23
INFO - PID 13753 |     #32 0x7fa588e6da6e in mozilla::net::nsLoadGroup::NotifyRemovalObservers(nsIRequest*, nsresult) /builds/worker/checkouts/gecko/netwerk/base/nsLoadGroup.cpp:614:22
INFO - PID 13753 |     #33 0x7fa588e70253 in mozilla::net::nsLoadGroup::RemoveRequest(nsIRequest*, nsISupports*, nsresult) /builds/worker/checkouts/gecko/netwerk/base/nsLoadGroup.cpp:518:10
INFO - PID 13753 |     #34 0x7fa58b8dda80 in mozilla::dom::Document::DoUnblockOnload() /builds/worker/checkouts/gecko/dom/base/Document.cpp:11554:18
INFO - PID 13753 |     #35 0x7fa58b8986b6 in mozilla::dom::Document::UnblockOnload(bool) /builds/worker/checkouts/gecko/dom/base/Document.cpp:11484:9
INFO - PID 13753 |     #36 0x7fa58b8bbe17 in mozilla::dom::Document::DispatchContentLoadedEvents() /builds/worker/checkouts/gecko/dom/base/Document.cpp:7999:3
INFO - PID 13753 |     #37 0x7fa58b97eedf in applyImpl<mozilla::dom::Document, void (mozilla::dom::Document::*)()> /builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h:1147:12
INFO - PID 13753 |     #38 0x7fa58b97eedf in apply<mozilla::dom::Document, void (mozilla::dom::Document::*)()> /builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h:1153:12
INFO - PID 13753 |     #39 0x7fa58b97eedf in mozilla::detail::RunnableMethodImpl<mozilla::dom::Document*, void (mozilla::dom::Document::*)(), true, (mozilla::RunnableKind)0>::Run() /builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h:1200:13
INFO - PID 13753 |     #40 0x7fa588b0a37f in mozilla::SchedulerGroup::Runnable::Run() /builds/worker/checkouts/gecko/xpcom/threads/SchedulerGroup.cpp:144:20
INFO - PID 13753 |     #41 0x7fa588b55fa2 in mozilla::RunnableTask::Run() /builds/worker/checkouts/gecko/xpcom/threads/TaskController.cpp:468:16
INFO - PID 13753 |     #42 0x7fa588b1d01d in mozilla::TaskController::DoExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) /builds/worker/checkouts/gecko/xpcom/threads/TaskController.cpp:771:26
INFO - PID 13753 |     #43 0x7fa588b1a8d8 in mozilla::TaskController::ExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) /builds/worker/checkouts/gecko/xpcom/threads/TaskController.cpp:607:15
INFO - PID 13753 |     #44 0x7fa588b1afe9 in mozilla::TaskController::ProcessPendingMTTask(bool) /builds/worker/checkouts/gecko/xpcom/threads/TaskController.cpp:391:36
INFO - PID 13753 |     #45 0x7fa588b5e461 in operator() /builds/worker/checkouts/gecko/xpcom/threads/TaskController.cpp:124:37
INFO - PID 13753 |     #46 0x7fa588b5e461 in mozilla::detail::RunnableFunction<mozilla::TaskController::InitializeInternal()::$_0>::Run() /builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.h:531:5
INFO - PID 13753 |     #47 0x7fa588b3c9fb in nsThread::ProcessNextEvent(bool, bool*) /builds/worker/checkouts/gecko/xpcom/threads/nsThread.cpp:1195:16
INFO - PID 13753 |     #48 0x7fa588b47b5c in NS_ProcessNextEvent(nsIThread*, bool) /builds/worker/checkouts/gecko/xpcom/threads/nsThreadUtils.cpp:467:10
INFO - PID 13753 |     #49 0x7fa589ce9848 in mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate*) /builds/worker/checkouts/gecko/ipc/glue/MessagePump.cpp:85:21
INFO - PID 13753 |     #50 0x7fa589bf7171 in RunInternal /builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc:331:10
INFO - PID 13753 |     #51 0x7fa589bf7171 in RunHandler /builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc:324:3
INFO - PID 13753 |     #52 0x7fa589bf7171 in MessageLoop::Run() /builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc:306:3
INFO - PID 13753 |     #53 0x7fa58fd3f8e7 in nsBaseAppShell::Run() /builds/worker/checkouts/gecko/widget/nsBaseAppShell.cpp:137:27
INFO - PID 13753 |     #54 0x7fa59402d4cf in XRE_RunAppShell() /builds/worker/checkouts/gecko/toolkit/xre/nsEmbedFunctions.cpp:864:20
INFO - PID 13753 |     #55 0x7fa589bf7171 in RunInternal /builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc:331:10
INFO - PID 13753 |     #56 0x7fa589bf7171 in RunHandler /builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc:324:3
INFO - PID 13753 |     #57 0x7fa589bf7171 in MessageLoop::Run() /builds/worker/checkouts/gecko/ipc/chromium/src/base/message_loop.cc:306:3
INFO - PID 13753 |     #58 0x7fa59402cef6 in XRE_InitChildProcess(int, char**, XREChildData const*) /builds/worker/checkouts/gecko/toolkit/xre/nsEmbedFunctions.cpp:701:34
INFO - PID 13753 |     #59 0x55c4a320d6dd in content_process_main(mozilla::Bootstrap*, int, char**) /builds/worker/checkouts/gecko/browser/app/../../ipc/contentproc/plugin-container.cpp:57:28
INFO - PID 13753 |     #60 0x55c4a320dafd in main /builds/worker/checkouts/gecko/browser/app/nsBrowserApp.cpp:327:18
INFO - PID 13753 |     #61 0x7fa5ab3afb96 in __libc_start_main /tmp/glibc/csu/../csu/libc-start.c:310
INFO - PID 13753 |     #62 0x55c4a315c7cc in _start (/builds/worker/workspace/build/application/firefox/firefox+0x577cc)

https://hg.mozilla.org/mozilla-central/rev/ba7513117392