No test case is available.

Assertion failure: topWin->mModalStateDepth != 0, at src/dom/base/nsGlobalWindowOuter.cpp:6455

#0 0x7fa037bc7a60 in nsGlobalWindowOuter::LeaveModalState() src/dom/base/nsGlobalWindowOuter.cpp:6455:3
#1 0x7fa037b65c2b in nsDOMWindowUtils::LeaveModalState() src/dom/base/nsDOMWindowUtils.cpp:2149:11
#2 0x7fa036126fd5 in NS_InvokeByIndex src/xpcom/reflect/xptcall/md/unix/xptcinvoke_asm_x86_64_unix.S:101
#3 0x7fa0372039cd in Invoke src/js/xpconnect/src/XPCWrappedNative.cpp:1620:10
#4 0x7fa0372039cd in CallMethodHelper::Call() src/js/xpconnect/src/XPCWrappedNative.cpp:1176:19
#5 0x7fa037203717 in XPCWrappedNative::CallMethod(XPCCallContext&, XPCWrappedNative::CallMode) src/js/xpconnect/src/XPCWrappedNative.cpp:1142:23
#6 0x7fa0372052d1 in XPC_WN_CallMethod(JSContext*, unsigned int, JS::Value*) src/js/xpconnect/src/XPCWrappedNativeJSOps.cpp:925:10
#7 0x7fa03bf9bb11 in CallJSNative(JSContext*, bool (*)(JSContext*, unsigned int, JS::Value*), js::CallReason, JS::CallArgs const&) src/js/src/vm/Interpreter.cpp:507:13
#8 0x7fa03bf9b248 in js::InternalCallOrConstruct(JSContext*, JS::CallArgs const&, js::MaybeConstruct, js::CallReason) src/js/src/vm/Interpreter.cpp:599:12
#9 0x7fa03bf9cdf3 in InternalCall(JSContext*, js::AnyInvokeArgs const&, js::CallReason) src/js/src/vm/Interpreter.cpp:664:10
#10 0x7fa03bf90bc3 in CallFromStack src/js/src/vm/Interpreter.cpp:668:10
#11 0x7fa03bf90bc3 in Interpret(JSContext*, js::RunState&) src/js/src/vm/Interpreter.cpp:3337:16
#12 0x7fa03bf87d34 in js::RunScript(JSContext*, js::RunState&) src/js/src/vm/Interpreter.cpp:477:13
#13 0x7fa03bf9b219 in js::InternalCallOrConstruct(JSContext*, JS::CallArgs const&, js::MaybeConstruct, js::CallReason) src/js/src/vm/Interpreter.cpp:636:13
#14 0x7fa03bf9cdf3 in InternalCall(JSContext*, js::AnyInvokeArgs const&, js::CallReason) src/js/src/vm/Interpreter.cpp:664:10
#15 0x7fa03bf9d02f in js::Call(JSContext*, JS::Handle<JS::Value>, JS::Handle<JS::Value>, js::AnyInvokeArgs const&, JS::MutableHandle<JS::Value>, js::CallReason) src/js/src/vm/Interpreter.cpp:681:8
#16 0x7fa03c58180b in JS::Call(JSContext*, JS::Handle<JS::Value>, JS::Handle<JS::Value>, JS::HandleValueArray const&, JS::MutableHandle<JS::Value>) src/js/src/jsapi.cpp:2830:10
#17 0x7fa0381eb5f5 in mozilla::dom::MessageListener::ReceiveMessage(mozilla::dom::BindingCallContext&, JS::Handle<JS::Value>, mozilla::dom::ReceiveMessageArgument const&, JS::MutableHandle<JS::Value>, mozilla::ErrorResult&) /builds/worker/workspace/obj-build/dom/bindings/MessageManagerBinding.cpp:6297:8
#18 0x7fa03a214ade in mozilla::dom::MessageListener::ReceiveMessage(mozilla::dom::ReceiveMessageArgument const&, JS::MutableHandle<JS::Value>, mozilla::ErrorResult&, char const*, mozilla::dom::CallbackObject::ExceptionHandling, JS::Realm*) /builds/worker/workspace/obj-build/dist/include/mozilla/dom/MessageManagerBinding.h:784:12
#19 0x7fa03a2148e9 in mozilla::dom::JSActor::CallReceiveMessage(JSContext*, mozilla::dom::JSActorMessageMeta const&, JS::Handle<JS::Value>, JS::MutableHandle<JS::Value>, mozilla::ErrorResult&) src/dom/ipc/jsactor/JSActor.cpp:256:22
#20 0x7fa03a214c62 in mozilla::dom::JSActor::ReceiveMessage(JSContext*, mozilla::dom::JSActorMessageMeta const&, JS::Handle<JS::Value>, mozilla::ErrorResult&) src/dom/ipc/jsactor/JSActor.cpp:269:3
#21 0x7fa03a217e36 in mozilla::dom::JSActorManager::ReceiveRawMessage(mozilla::dom::JSActorMessageMeta const&, mozilla::Maybe<mozilla::dom::ipc::StructuredCloneData>&&, mozilla::Maybe<mozilla::dom::ipc::StructuredCloneData>&&) src/dom/ipc/jsactor/JSActorManager.cpp:181:14
#22 0x7fa03a20087f in mozilla::dom::WindowGlobalChild::RecvRawMessage(mozilla::dom::JSActorMessageMeta const&, mozilla::Maybe<mozilla::dom::ClonedMessageData> const&, mozilla::Maybe<mozilla::dom::ClonedMessageData> const&) src/dom/ipc/WindowGlobalChild.cpp:569:3
#23 0x7fa036fea6c2 in mozilla::dom::PWindowGlobalChild::OnMessageReceived(IPC::Message const&) /builds/worker/workspace/obj-build/ipc/ipdl/PWindowGlobalChild.cpp:1245:61
#24 0x7fa036b69c4d in mozilla::dom::PContentChild::OnMessageReceived(IPC::Message const&) /builds/worker/workspace/obj-build/ipc/ipdl/PContentChild.cpp:8518:32
#25 0x7fa0369e84ae in mozilla::ipc::MessageChannel::DispatchAsyncMessage(mozilla::ipc::ActorLifecycleProxy*, IPC::Message const&) src/ipc/glue/MessageChannel.cpp:2150:25
#26 0x7fa0369e4c6f in mozilla::ipc::MessageChannel::DispatchMessage(IPC::Message&&) src/ipc/glue/MessageChannel.cpp:2074:9
#27 0x7fa0369e6076 in mozilla::ipc::MessageChannel::RunMessage(mozilla::ipc::MessageChannel::MessageTask&) src/ipc/glue/MessageChannel.cpp:1922:3
#28 0x7fa0369e6c9b in mozilla::ipc::MessageChannel::MessageTask::Run() src/ipc/glue/MessageChannel.cpp:1953:13
#29 0x7fa0360e77af in mozilla::RunnableTask::Run() src/xpcom/threads/TaskController.cpp:450:16
#30 0x7fa0360e5e1a in mozilla::TaskController::DoExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) src/xpcom/threads/TaskController.cpp:720:26
#31 0x7fa0360e4ec4 in mozilla::TaskController::ExecuteNextTaskOnlyMainThreadInternal(mozilla::detail::BaseAutoLock<mozilla::Mutex&> const&) src/xpcom/threads/TaskController.cpp:579:15
#32 0x7fa0360e5077 in mozilla::TaskController::ProcessPendingMTTask(bool) src/xpcom/threads/TaskController.cpp:373:36
#33 0x7fa0360eb006 in operator() src/xpcom/threads/TaskController.cpp:120:37
#34 0x7fa0360eb006 in mozilla::detail::RunnableFunction<mozilla::TaskController::InitializeInternal()::$_3>::Run() /builds/worker/workspace/obj-build/dist/include/nsThreadUtils.h:577:5
#35 0x7fa0360fc587 in nsThread::ProcessNextEvent(bool, bool*) src/xpcom/threads/nsThread.cpp:1197:14
#36 0x7fa0361022ca in NS_ProcessNextEvent(nsIThread*, bool) src/xpcom/threads/nsThreadUtils.cpp:513:10
#37 0x7fa0369edd96 in mozilla::ipc::MessagePump::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:87:21
#38 0x7fa03695fc83 in MessageLoop::RunInternal() src/ipc/chromium/src/base/message_loop.cc:334:10
#39 0x7fa03695fb9d in RunHandler src/ipc/chromium/src/base/message_loop.cc:327:3
#40 0x7fa03695fb9d in MessageLoop::Run() src/ipc/chromium/src/base/message_loop.cc:309:3
#41 0x7fa03a657038 in nsBaseAppShell::Run() src/widget/nsBaseAppShell.cpp:137:27
#42 0x7fa03be61583 in XRE_RunAppShell() src/toolkit/xre/nsEmbedFunctions.cpp:913:20
#43 0x7fa0369eeb59 in mozilla::ipc::MessagePumpForChildProcess::Run(base::MessagePump::Delegate*) src/ipc/glue/MessagePump.cpp:237:9
#44 0x7fa03695fc83 in MessageLoop::RunInternal() src/ipc/chromium/src/base/message_loop.cc:334:10
#45 0x7fa03695fb9d in RunHandler src/ipc/chromium/src/base/message_loop.cc:327:3
#46 0x7fa03695fb9d in MessageLoop::Run() src/ipc/chromium/src/base/message_loop.cc:309:3
#47 0x7fa03be61168 in XRE_InitChildProcess(int, char**, XREChildData const*) src/toolkit/xre/nsEmbedFunctions.cpp:744:34
#48 0x558d487aa997 in content_process_main src/browser/app/../../ipc/contentproc/plugin-container.cpp:56:28
#49 0x558d487aa997 in main src/browser/app/nsBrowserApp.cpp:304:18

A Pernosco session is available here: https://pernos.co/debug/WpkxeEd2bRtCXt-V31ZOsA/index.html

JS Stack is:

#1  0x00007f7b8b0c04d2 in nsGlobalWindowOuter::LeaveModalState (this=0x55ac155b59f0) at /home/twsmith/code/mozilla-central/dom/base/nsGlobalWindowOuter.cpp:6455
(pernosco) p DumpJSStack() 
0 receiveMessage(message = "[object Object]") ["resource://gre/actors/BrowserElementChild.jsm":33:39]
    this = [object JSWindowActorChild]

This seems like it's managed by the front-end code in here.

So presumably we get a message with forceLeave but we're not in a modal state... Paul?

It seems the top window is basically fresh, EnterModalState hadn't been called yet.

Looking at the screenshot, the leaveModalState call comes from here: https://searchfox.org/mozilla-central/rev/50215d649d4854812837f1343e8f47bd998dacb5/browser/actors/PromptParent.jsm#295.
Since this is called async if we're showing a tab prompt, we should also use maybeLeaveModalState, like we do for content prompts: https://searchfox.org/mozilla-central/rev/50215d649d4854812837f1343e8f47bd998dacb5/browser/actors/PromptParent.jsm#192

I'm still not quite sure what causes the window to not have modal state. We are calling it here: https://searchfox.org/mozilla-central/rev/50215d649d4854812837f1343e8f47bd998dacb5/browser/actors/PromptParent.jsm#266
Perhaps there is overlap between the async enter/leave messages. Not something I've observed in normal usage though.

https://hg.mozilla.org/mozilla-central/rev/af2dbd15d9a0