The following testcase crashes on mozilla-central revision f3b757156f69 (build with --enable-gczeal --enable-optimize="-O2 -g" --enable-address-sanitizer --enable-posix-nspr-emulation --disable-jemalloc --disable-tests --disable-debug, run with --fuzzing-safe --thread-count=2 --baseline-eager): var lfcode = new Array(); lfcode.push = loadFile; gczeal(2); lfcode.push("5"); lfcode.push(` var whitespace = [ {s : '\\u0009', t : 'HORIZONTAL TAB'}, {s : '\\u000B', t : 'VERTICAL TAB'}, {s : '\\u000C', t : 'FORMFEED'}, {s : '\\u0020', t : 'SPACE'}, {s : '\\u00A0', t : 'NO-BREAK SPACE'}, {s : '\\u1680', t : 'OGHAM SPACE MARK'}, {s : '\\u180E', t : 'MONGOLIAN VOWEL SEPARATOR'}, {s : '\\u2000', t : 'EN QUAD'}, {s : '\\u2001', t : 'EM QUAD'}, {s : '\\u2002', t : 'EN SPACE'}, {s : '\\u2003', t : 'EM SPACE'}, {s : '\\u2004', t : 'THREE-PER-EM SPACE'}, {s : '\\u2005', t : 'FOUR-PER-EM SPACE'}, {s : '\\u2007', t : 'FIGURE SPACE'}, {s : '\\u2008', t : 'PUNCTUATION SPACE'}, {s : '\\u2009', t : 'THIN SPACE'}, {s : '\\u200A', t : 'HAIR SPACE'}, {s : '\\u2028', t : 'LINE SEPARATOR'}, {s : '\\u2029', t : 'PARAGRAPH SEPARATOR'}, {s : '\\u200B', t : 'ZERO WIDTH SPACE (category Cf)'} ]; { { } } `); function loadFile(lfVarx) { try { if (lfVarx.substr(-3) != ".js" && lfVarx.length != 1) { switch (lfRunTypeId) { default: var lfGlobal = newGlobal(); lfGlobal.offThreadCompileScript(lfVarx); lfGlobal.runOffThreadScript(); evaluate(lfVarx, { noScriptRval : true, compileAndGo : true }); break; } } else if (!isNaN(lfVarx)) { lfRunTypeId = parseInt(lfVarx); } } catch (lfVare) {} } Backtrace: ==29461==ERROR: AddressSanitizer: heap-use-after-free on address 0x617000027888 at pc 0xbfa7b7 bp 0x7fff450c8fb0 sp 0x7fff450c8fa8 WRITE of size 8 at 0x617000027888 thread T0 #0 0xbfa7b6 in mozilla::LinkedListElement<js::UnboxedLayout>::remove() js/src/opt64asan/js/src/../../dist/include/mozilla/LinkedList.h:208 #1 0xbfa7b6 in js::ObjectGroup::unboxedLayout() js/src/vm/UnboxedObject.cpp:278 #2 0xbfa7b6 in js::ObjectGroup::sweep(js::AutoClearTypeInferenceStateOnOOM*) js/src/vm/TypeInference.cpp:4119 #3 0x15d3c00 in js::ObjectGroup::maybeSweep(js::AutoClearTypeInferenceStateOnOOM*) js/src/vm/ObjectGroup-inl.h:26 #4 0x15d3c00 in SweepThing(js::ObjectGroup*, js::AutoClearTypeInferenceStateOnOOM*) js/src/jsgc.cpp:5184 #5 0x15d3c00 in _ZL14SweepArenaListIN2js11ObjectGroupEJPNS0_32AutoClearTypeInferenceStateOnOOMEEEbPPNS0_2gc11ArenaHeaderERNS0_11SliceBudgetEDpT0_ js/src/jsgc.cpp:5193 #6 0x15d3c00 in js::gc::GCRuntime::sweepPhase(js::SliceBudget&) js/src/jsgc.cpp:5237 #7 0x15db319 in js::gc::GCRuntime::incrementalCollectSlice(js::SliceBudget&, JS::gcreason::Reason) js/src/jsgc.cpp:5880 #8 0x15dcb98 in js::gc::GCRuntime::gcCycle(bool, js::SliceBudget&, JS::gcreason::Reason) js/src/jsgc.cpp:6063 #9 0x15ddef8 in js::gc::GCRuntime::collect(bool, js::SliceBudget, JS::gcreason::Reason) js/src/jsgc.cpp:6177 #10 0x15bfca9 in js::gc::GCRuntime::gc(JSGCInvocationKind, JS::gcreason::Reason) js/src/jsgc.cpp:6238 #11 0x14f3eac in js::DestroyContext(JSContext*, js::DestroyContextMode) js/src/jscntxt.cpp:186 #12 0x4d5fcc in DestroyContext(JSContext*, bool) js/src/shell/js.cpp:5596 #13 0x4d5fcc in main js/src/shell/js.cpp:6393 #14 0x7fd2f2812ec4 in __libc_start_main /build/buildd/eglibc-2.19/csu/libc-start.c:287 #15 0x4c740c in _start (/home/ubuntu/mozilla-central/js/src/opt64asan/dist/bin/js+0x4c740c) 0x617000027888 is located 520 bytes inside of 704-byte region [0x617000027680,0x617000027940) freed by thread T0 here: #0 0x4adacf in __interceptor_free /srv/repos/llvm/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:64 #1 0x15c3b38 in js_free(void*) js/src/opt64asan/js/src/../../dist/include/js/Utility.h:158 #2 0x15c3b38 in _ZL9js_deleteI13JSCompartmentEvPT_ js/src/opt64asan/js/src/../../dist/include/js/Utility.h:255 #3 0x15c3b38 in JS::Zone::sweepCompartments(js::FreeOp*, bool, bool) js/src/jsgc.cpp:3604 #4 0x15c3fda in js::gc::GCRuntime::sweepZones(js::FreeOp*, bool) js/src/jsgc.cpp:3645 #5 0x15d5bfa in js::gc::GCRuntime::endSweepPhase(bool) js/src/jsgc.cpp:5369 #6 0x15db33c in js::gc::GCRuntime::incrementalCollectSlice(js::SliceBudget&, JS::gcreason::Reason) js/src/jsgc.cpp:5883 #7 0x15dcb98 in js::gc::GCRuntime::gcCycle(bool, js::SliceBudget&, JS::gcreason::Reason) js/src/jsgc.cpp:6063 #8 0x15ddef8 in js::gc::GCRuntime::collect(bool, js::SliceBudget, JS::gcreason::Reason) js/src/jsgc.cpp:6177 #9 0x15e53dd in js::gc::GCRuntime::gc(JSGCInvocationKind, JS::gcreason::Reason) js/src/jsgc.cpp:6238 #10 0x15e53dd in js::gc::GCRuntime::runDebugGC() js/src/jsgc.cpp:6677 #11 0x8cf37f in js::gc::GCRuntime::gcIfNeededPerAllocation(JSContext*) js/src/gc/Allocator.cpp:28 #12 0x8ef7a6 in JSContext::runtime() const js/src/gc/Allocator.cpp:55 #13 0x8ef7a6 in js::ObjectGroup* js::Allocate<js::ObjectGroup, (js::AllowGC)1>(js::ExclusiveContext*) js/src/gc/Allocator.cpp:211 #14 0xac4826 in JSObject::getClass() const js/src/vm/ObjectGroup.cpp:1387 #15 0xac4826 in JSObject::makeLazyGroup(JSContext*, JS::Handle<JSObject*>) js/src/vm/ObjectGroup.cpp:328 #16 0xeace8d in JSObject::getGroup(JSContext*) js/src/jsobjinlines.h:134 #17 0xeace8d in js::jit::DoSetPropFallback(JSContext*, js::jit::BaselineFrame*, js::jit::ICSetProp_Fallback*, JS::Handle<JS::Value>, JS::Handle<JS::Value>, JS::MutableHandle<JS::Value>) js/src/jit/BaselineIC.cpp:8662 #18 0xee78e6 in EnterBaseline(JSContext*, js::jit::EnterJitData&) js/src/jit/BaselineJIT.cpp:124 [...] #31 0x9ae23e in js::RunScript(JSContext*, js::RunState&) js/src/vm/Interpreter.cpp:742 previously allocated by thread T0 here: #0 0x4adce7 in __interceptor_malloc /srv/repos/llvm/projects/compiler-rt/lib/asan/asan_malloc_linux.cc:74 #1 0x502dbb in js_malloc(unsigned long) js/src/opt64asan/js/src/../../dist/include/js/Utility.h:135 #2 0x502dbb in _ZL13js_pod_mallocIhEPT_m js/src/opt64asan/js/src/../../dist/include/js/Utility.h:290 #3 0x502dbb in unsigned char* js::MallocProvider<js::ExclusiveContext>::pod_malloc<unsigned char>(unsigned long) js/src/vm/MallocProvider.h:63 #4 0x15e21b1 in JSCompartment* js::MallocProvider<js::ExclusiveContext>::new_<JSCompartment, JS::Zone*&, JS::CompartmentOptions const&>(JS::Zone*&, JS::CompartmentOptions const&) js/src/vm/MallocProvider.h:178 #5 0x15e21b1 in js::NewCompartment(JSContext*, JS::Zone*, JSPrincipals*, JS::CompartmentOptions const&) js/src/jsgc.cpp:6528 #6 0x99b3df in js::GlobalObject::new_(JSContext*, js::Class const*, JSPrincipals*, JS::OnNewGlobalHookOption, JS::CompartmentOptions const&) js/src/vm/GlobalObject.cpp:287 #7 0x9a347c in js::StartOffThreadParseScript(JSContext*, JS::ReadOnlyCompileOptions const&, char16_t const*, unsigned long, void (*)(void*, void*), void*) js/src/vm/HelperThreads.cpp:334 #8 0x4f13ff in OffThreadCompileScript(JSContext*, unsigned int, JS::Value*) js/src/shell/js.cpp:3322 #9 0x98e673 in js::CallJSNative(JSContext*, bool (*)(JSContext*, unsigned int, JS::Value*), JS::CallArgs const&) js/src/jscntxtinlines.h:235 #10 0x98e673 in js::Invoke(JSContext*, JS::CallArgs, js::MaybeConstruct) js/src/vm/Interpreter.cpp:811 #11 0x937f63 in js::Invoke(JSContext*, JS::Value const&, JS::Value const&, unsigned int, JS::Value const*, JS::MutableHandle<JS::Value>) js/src/vm/Interpreter.cpp:866 #12 0x175ebf5 in js::DirectProxyHandler::call(JSContext*, JS::Handle<JSObject*>, JS::CallArgs const&) const js/src/proxy/DirectProxyHandler.cpp:77 #13 0x175ebf5 in js::CrossCompartmentWrapper::call(JSContext*, JS::Handle<JSObject*>, JS::CallArgs const&) const js/src/proxy/CrossCompartmentWrapper.cpp:289 #14 0x17705ee in js::Proxy::call(JSContext*, JS::Handle<JSObject*>, JS::CallArgs const&) js/src/proxy/Proxy.cpp:391 #15 0x1773832 in js::proxy_Call(JSContext*, unsigned int, JS::Value*) js/src/proxy/Proxy.cpp:697 #16 0x98ea32 in js::CallJSNative(JSContext*, bool (*)(JSContext*, unsigned int, JS::Value*), JS::CallArgs const&) js/src/jscntxtinlines.h:235 [...] #29 0x9e2414 in js::ExecuteKernel(JSContext*, JS::Handle<JSScript*>, JSObject&, JS::Value const&, JS::Value const&, js::ExecuteType, js::AbstractFramePtr, JS::Value*) js/src/vm/Interpreter.cpp:993 SUMMARY: AddressSanitizer: heap-use-after-free js/src/opt64asan/js/src/../../dist/include/mozilla/LinkedList.h:208 mozilla::LinkedListElement<js::UnboxedLayout>::remove() Shadow bytes around the buggy address: 0x0c2e7fffcec0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x0c2e7fffced0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c2e7fffcee0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c2e7fffcef0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c2e7fffcf00: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd =>0x0c2e7fffcf10: fd[fd]fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c2e7fffcf20: fd fd fd fd fd fd fd fd fa fa fa fa fa fa fa fa 0x0c2e7fffcf30: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x0c2e7fffcf40: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c2e7fffcf50: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd 0x0c2e7fffcf60: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd Shadow byte legend (one shadow byte represents 8 application bytes): Addressable: 00 Partially addressable: 01 02 03 04 05 06 07 Heap left redzone: fa Freed heap region: fd ==29461==ABORTING The test for this bug doesn't reproduce for me on any of my triage machines, but it reproduces on the original EC2 instance. I suspect this is a timing problem because an off-thread script is involved. On the EC2 instance, the issue reproduced intermittently with about 40-50% chance for a crash. Marking s-s because this is a use-after-free.

Nathan, if you want to poke around with a TSan JS shell, here's a good candidate for a test case.

This looks a lot like some of the crashes in bug 1191465, so I'm calling it a regression from bug 1181908.

This should be fixed by a backout, though it would still be good to understand what is going wrong.

Well, maybe this should be left open until somebody confirms that this isn't some underlying issue that is just exposed by bz's patch. I don't know how his patch could cause a crash in sweeping unboxed objects.

(In reply to Andrew McCreight [:mccr8] from comment #1) > Nathan, if you want to poke around with a TSan JS shell, here's a good > candidate for a test case. I will put it on my plate.

I investigated an identical looking crash, before we knew about the latent crashtastrophe. The root cause was heap corruption from the CompileOptions patch. The watchpoint I put on the list node to detect the error wasn't even firing because the corrupting write wasn't aligned. I think we can safely assume this is more CompileOptions fallout.

(In reply to Nathan Froyd [:froydnj][:nfroyd] from comment #5) > (In reply to Andrew McCreight [:mccr8] from comment #1) > > Nathan, if you want to poke around with a TSan JS shell, here's a good > > candidate for a test case. > > I will put it on my plate. Several hundred TSan runs didn't turn up anything interesting. It's possible there are/were timing issues involved?

Since GC runs on a different thread and typically runs based on some metrics (memory usage, timing), it might be the main reason for non-determinism here. It might be GC vs. the off-thread script. Not sure how we can test this more reliably.

The testcase calls gczeal(), so those GC heuristics shouldn't be your source of non-determinism. Try changing it to gczeal(2, 1) so it collects on every allocation instead of every 100 allocations. You could also try some of the other gczeal modes: https://dxr.mozilla.org/mozilla-central/rev/f61c3cc0eb8b7533818e7379ccc063b611015d9d/js/src/jsgc.cpp#1189 I'm surprised to see js::gc::GCRuntime::incrementalCollectSlice on the stack with gczeal(2).

I typically try all of this before filing test cases. I have encountered lots of test cases where gczeal(2,X) reproduced a test for some X, but gczeal(2,1) would not. I guess it has to do with the order in which things happen in different threads (GC vs. Off-thread script for example).

> I don't know how his patch could cause a crash in sweeping unboxed objects. Because the code that was supposed to clear this list didn't, but the entire relevant codepath wasn't getting exercised because the optimization it's involved was getting turned off incorrectly in some cases until I fixed the CompileOptions stuff... See bug 1181908 comment 25 and bug 1181908 comment 26 for the analysis.

In any case, I expect that this is in fact fixed now.