From 363383cd7a71095cb55cb2deeaeed7272c803d56 Mon Sep 17 00:00:00 2001 From: Brian Hackett Date: Wed, 22 Oct 2014 07:57:33 -0700 Subject: [PATCH] Bug 1072564 - Incrementalize sweeping of type information, r=billm. --- js/src/ds/LifoAlloc.h | 1 + js/src/gc/GCRuntime.h | 1 + js/src/gc/Heap.h | 11 + js/src/gc/Marking.cpp | 12 +- js/src/gc/Statistics.cpp | 7 +- js/src/gc/Statistics.h | 5 +- js/src/gc/Zone.cpp | 18 +- js/src/gc/Zone.h | 2 +- .../tests/parallel/alloc-many-objs.js | 2 +- js/src/jit/Ion.cpp | 34 +- js/src/jit/Ion.h | 4 +- js/src/jit/IonAnalysis.cpp | 2 +- js/src/jit/IonBuilder.cpp | 16 +- js/src/jscompartment.cpp | 2 - js/src/jscompartment.h | 2 - js/src/jsgc.cpp | 262 +++++++++---- js/src/jsgc.h | 81 ++-- js/src/jsinfer.cpp | 360 ++++++++++-------- js/src/jsinfer.h | 134 +++++-- js/src/jsinferinlines.h | 118 ++++-- js/src/jsobjinlines.h | 2 +- js/src/jsopcode.cpp | 4 +- js/src/jsscript.cpp | 6 +- js/src/jsscript.h | 27 +- js/src/vm/ForkJoin.cpp | 10 +- js/src/vm/Interpreter.cpp | 4 +- 26 files changed, 728 insertions(+), 399 deletions(-) diff --git a/js/src/ds/LifoAlloc.h b/js/src/ds/LifoAlloc.h index efcfeeae4ba..0f06dcd9c2a 100644 --- a/js/src/ds/LifoAlloc.h +++ b/js/src/ds/LifoAlloc.h @@ -227,6 +227,7 @@ class LifoAlloc // Steal allocated chunks from |other|. void steal(LifoAlloc *other) { MOZ_ASSERT(!other->markCount); + MOZ_ASSERT(!latest); // Copy everything from |other| to |this| except for |peakSize_|, which // requires some care. diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index 051c2dde759..7de9ef8ea80 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -672,6 +672,7 @@ class GCRuntime */ JS::Zone *zoneGroups; JS::Zone *currentZoneGroup; + bool sweepingTypes; unsigned finalizePhase; JS::Zone *sweepZone; unsigned sweepKindIndex; diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h index 9654836f9fe..bad1d6aa282 100644 --- a/js/src/gc/Heap.h +++ b/js/src/gc/Heap.h @@ -1139,6 +1139,17 @@ ArenaHeader::unsetAllocDuringSweep() auxNextLink = 0; } +inline void +ReleaseArenaList(ArenaHeader *aheader) +{ + ArenaHeader *next; + for (; aheader; aheader = next) { + // Copy aheader->next before releasing. + next = aheader->next; + aheader->chunk()->releaseArena(aheader); + } +} + static void AssertValidColor(const TenuredCell *thing, uint32_t color) { diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp index e417cdfaf2a..de433fdc43b 100644 --- a/js/src/gc/Marking.cpp +++ b/js/src/gc/Marking.cpp @@ -511,16 +511,8 @@ IsAboutToBeFinalizedFromAnyThread(T **thingp) Zone *zone = thing->asTenured().zoneFromAnyThread(); if (zone->isGCSweeping()) { - /* - * We should return false for things that have been allocated during - * incremental sweeping, but this possibility doesn't occur at the moment - * because this function is only called at the very start of the sweeping a - * compartment group and during minor gc. Rather than do the extra check, - * we just assert that it's not necessary. - */ - MOZ_ASSERT_IF(!rt->isHeapMinorCollecting(), - !thing->asTenured().arenaHeader()->allocatedDuringIncremental); - + if (thing->asTenured().arenaHeader()->allocatedDuringIncremental) + return false; return !thing->asTenured().isMarked(); } #ifdef JSGC_COMPACTING diff --git a/js/src/gc/Statistics.cpp b/js/src/gc/Statistics.cpp index e79ce77e99f..f6a9b571b52 100644 --- a/js/src/gc/Statistics.cpp +++ b/js/src/gc/Statistics.cpp @@ -301,10 +301,9 @@ static const PhaseInfo phases[] = { { PHASE_SWEEP_BREAKPOINT, "Sweep Breakpoints", PHASE_SWEEP_COMPARTMENTS }, { PHASE_SWEEP_REGEXP, "Sweep Regexps", PHASE_SWEEP_COMPARTMENTS }, { PHASE_SWEEP_MISC, "Sweep Miscellaneous", PHASE_SWEEP_COMPARTMENTS }, - { PHASE_DISCARD_ANALYSIS, "Discard Analysis", PHASE_SWEEP_COMPARTMENTS }, - { PHASE_DISCARD_TI, "Discard TI", PHASE_DISCARD_ANALYSIS }, - { PHASE_FREE_TI_ARENA, "Free TI Arena", PHASE_DISCARD_ANALYSIS }, - { PHASE_SWEEP_TYPES, "Sweep Types", PHASE_DISCARD_ANALYSIS }, + { PHASE_SWEEP_TYPES, "Sweep type information", PHASE_SWEEP_COMPARTMENTS }, + { PHASE_SWEEP_TYPES_BEGIN, "Sweep type tables and compilations", PHASE_SWEEP_TYPES }, + { PHASE_SWEEP_TYPES_END, "Free type arena", PHASE_SWEEP_TYPES }, { PHASE_SWEEP_OBJECT, "Sweep Object", PHASE_SWEEP }, { PHASE_SWEEP_STRING, "Sweep String", PHASE_SWEEP }, { PHASE_SWEEP_SCRIPT, "Sweep Script", PHASE_SWEEP }, diff --git a/js/src/gc/Statistics.h b/js/src/gc/Statistics.h index 5ab10cac2c5..36c4405d9c5 100644 --- a/js/src/gc/Statistics.h +++ b/js/src/gc/Statistics.h @@ -54,10 +54,9 @@ enum Phase { PHASE_SWEEP_BREAKPOINT, PHASE_SWEEP_REGEXP, PHASE_SWEEP_MISC, - PHASE_DISCARD_ANALYSIS, - PHASE_DISCARD_TI, - PHASE_FREE_TI_ARENA, PHASE_SWEEP_TYPES, + PHASE_SWEEP_TYPES_BEGIN, + PHASE_SWEEP_TYPES_END, PHASE_SWEEP_OBJECT, PHASE_SWEEP_STRING, PHASE_SWEEP_SCRIPT, diff --git a/js/src/gc/Zone.cpp b/js/src/gc/Zone.cpp index f88bfb1d793..dc9b3c223c4 100644 --- a/js/src/gc/Zone.cpp +++ b/js/src/gc/Zone.cpp @@ -105,27 +105,15 @@ Zone::onTooMuchMalloc() } void -Zone::sweepAnalysis(FreeOp *fop, bool releaseTypes) +Zone::beginSweepTypes(FreeOp *fop, bool releaseTypes) { // Periodically release observed types for all scripts. This is safe to // do when there are no frames for the zone on the stack. if (active) releaseTypes = false; - bool oom = false; - types.sweep(fop, releaseTypes, &oom); - - // If there was an OOM while sweeping types, the type information needs to - // be deoptimized so that it will still correct (i.e. overapproximates the - // possible types in the zone), but the constraints might not have been - // triggered on the deoptimization or even copied over completely. In this - // case, destroy all JIT code and new script information in the zone, the - // only things whose correctness depends on the type constraints. - if (oom) { - setPreservingCode(false); - discardJitCode(fop); - types.clearAllNewScriptsOnOOM(); - } + types::AutoClearTypeInferenceStateOnOOM oom(this); + types.beginSweep(fop, releaseTypes, oom); } void diff --git a/js/src/gc/Zone.h b/js/src/gc/Zone.h index f72f539b9d3..fd0808fee7c 100644 --- a/js/src/gc/Zone.h +++ b/js/src/gc/Zone.h @@ -157,7 +157,7 @@ struct Zone : public JS::shadow::Zone, } void reportAllocationOverflow() { js_ReportAllocationOverflow(nullptr); } - void sweepAnalysis(js::FreeOp *fop, bool releaseTypes); + void beginSweepTypes(js::FreeOp *fop, bool releaseTypes); bool hasMarkedCompartments(); diff --git a/js/src/jit-test/tests/parallel/alloc-many-objs.js b/js/src/jit-test/tests/parallel/alloc-many-objs.js index 97893a9dfbe..3a8bc31dcab 100644 --- a/js/src/jit-test/tests/parallel/alloc-many-objs.js +++ b/js/src/jit-test/tests/parallel/alloc-many-objs.js @@ -11,7 +11,7 @@ function testMap() { print(m.mode+" "+m.expect); nums.mapPar(function (v) { var x = []; - for (var i = 0; i < 45000; i++) { + for (var i = 0; i < 20000; i++) { x[i] = {from: v}; } return x; diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp index b68dc535138..5ecd91d4f72 100644 --- a/js/src/jit/Ion.cpp +++ b/js/src/jit/Ion.cpp @@ -2846,7 +2846,7 @@ jit::InvalidateAll(FreeOp *fop, Zone *zone) void jit::Invalidate(types::TypeZone &types, FreeOp *fop, - const Vector &invalid, bool resetUses, + const types::RecompileInfoVector &invalid, bool resetUses, bool cancelOffThread) { JitSpew(JitSpew_IonInvalidate, "Start invalidation."); @@ -2855,23 +2855,24 @@ jit::Invalidate(types::TypeZone &types, FreeOp *fop, // to the traversal which frames have been invalidated. size_t numInvalidations = 0; for (size_t i = 0; i < invalid.length(); i++) { - const types::CompilerOutput &co = *invalid[i].compilerOutput(types); - if (!co.isValid()) + const types::CompilerOutput *co = invalid[i].compilerOutput(types); + if (!co) continue; + MOZ_ASSERT(co->isValid()); if (cancelOffThread) - CancelOffThreadIonCompile(co.script()->compartment(), co.script()); + CancelOffThreadIonCompile(co->script()->compartment(), co->script()); - if (!co.ion()) + if (!co->ion()) continue; JitSpew(JitSpew_IonInvalidate, " Invalidate %s:%u, IonScript %p", - co.script()->filename(), co.script()->lineno(), co.ion()); + co->script()->filename(), co->script()->lineno(), co->ion()); // Keep the ion script alive during the invalidation and flag this // ionScript as being invalidated. This increment is removed by the // loop after the calls to InvalidateActivation. - co.ion()->incref(); + co->ion()->incref(); numInvalidations++; } @@ -2887,19 +2888,20 @@ jit::Invalidate(types::TypeZone &types, FreeOp *fop, // IonScript will be immediately destroyed. Otherwise, it will be held live // until its last invalidated frame is destroyed. for (size_t i = 0; i < invalid.length(); i++) { - types::CompilerOutput &co = *invalid[i].compilerOutput(types); - if (!co.isValid()) + types::CompilerOutput *co = invalid[i].compilerOutput(types); + if (!co) continue; + MOZ_ASSERT(co->isValid()); - ExecutionMode executionMode = co.mode(); - JSScript *script = co.script(); - IonScript *ionScript = co.ion(); + ExecutionMode executionMode = co->mode(); + JSScript *script = co->script(); + IonScript *ionScript = co->ion(); if (!ionScript) continue; SetIonScript(nullptr, script, executionMode, nullptr); ionScript->decref(fop); - co.invalidate(); + co->invalidate(); numInvalidations--; // Wait for the scripts to get warm again before doing another @@ -2923,7 +2925,7 @@ jit::Invalidate(types::TypeZone &types, FreeOp *fop, } void -jit::Invalidate(JSContext *cx, const Vector &invalid, bool resetUses, +jit::Invalidate(JSContext *cx, const types::RecompileInfoVector &invalid, bool resetUses, bool cancelOffThread) { jit::Invalidate(cx->zone()->types, cx->runtime()->defaultFreeOp(), invalid, resetUses, @@ -2934,7 +2936,7 @@ bool jit::IonScript::invalidate(JSContext *cx, bool resetUses, const char *reason) { JitSpew(JitSpew_IonInvalidate, " Invalidate IonScript %p: %s", this, reason); - Vector list(cx); + types::RecompileInfoVector list; if (!list.append(recompileInfo())) return false; Invalidate(cx, list, resetUses, true); @@ -2968,7 +2970,7 @@ jit::Invalidate(JSContext *cx, JSScript *script, ExecutionMode mode, bool resetU js_free(buf); } - Vector scripts(cx); + types::RecompileInfoVector scripts; switch (mode) { case SequentialExecution: diff --git a/js/src/jit/Ion.h b/js/src/jit/Ion.h index a2c2fc66ea6..0569349eba5 100644 --- a/js/src/jit/Ion.h +++ b/js/src/jit/Ion.h @@ -126,9 +126,9 @@ IonExecStatus FastInvoke(JSContext *cx, HandleFunction fun, CallArgs &args); // Walk the stack and invalidate active Ion frames for the invalid scripts. void Invalidate(types::TypeZone &types, FreeOp *fop, - const Vector &invalid, bool resetUses = true, + const types::RecompileInfoVector &invalid, bool resetUses = true, bool cancelOffThread = true); -void Invalidate(JSContext *cx, const Vector &invalid, bool resetUses = true, +void Invalidate(JSContext *cx, const types::RecompileInfoVector &invalid, bool resetUses = true, bool cancelOffThread = true); bool Invalidate(JSContext *cx, JSScript *script, ExecutionMode mode, bool resetUses = true, bool cancelOffThread = true); diff --git a/js/src/jit/IonAnalysis.cpp b/js/src/jit/IonAnalysis.cpp index 847d0c4402b..1a8b19f079b 100644 --- a/js/src/jit/IonAnalysis.cpp +++ b/js/src/jit/IonAnalysis.cpp @@ -2850,7 +2850,7 @@ jit::AnalyzeNewScriptDefiniteProperties(JSContext *cx, JSFunction *fun, types::TypeObject *type, HandleNativeObject baseobj, Vector *initializerList) { - MOZ_ASSERT(cx->compartment()->activeAnalysis); + MOZ_ASSERT(cx->zone()->types.activeAnalysis); // When invoking 'new' on the specified script, try to find some properties // which will definitely be added to the created object before it has a diff --git a/js/src/jit/IonBuilder.cpp b/js/src/jit/IonBuilder.cpp index 6dca4ba3f7b..ff714b32594 100644 --- a/js/src/jit/IonBuilder.cpp +++ b/js/src/jit/IonBuilder.cpp @@ -4271,11 +4271,10 @@ IonBuilder::inlineScriptedCall(CallInfo &callInfo, JSFunction *target) // Improve type information of |this| when not set. if (callInfo.constructing() && - !callInfo.thisArg()->resultTypeSet() && - calleeScript->types) + !callInfo.thisArg()->resultTypeSet()) { types::StackTypeSet *types = types::TypeScript::ThisTypes(calleeScript); - if (!types->unknown()) { + if (types && !types->unknown()) { types::TemporaryTypeSet *clonedTypes = types->clone(alloc_->lifoAlloc()); if (!clonedTypes) return oom(); @@ -5207,9 +5206,8 @@ IonBuilder::createThisScriptedSingleton(JSFunction *target, MDefinition *callee) if (!templateObject->hasTenuredProto() || templateObject->getProto() != proto) return nullptr; - if (!target->nonLazyScript()->types) - return nullptr; - if (!types::TypeScript::ThisTypes(target->nonLazyScript())->hasType(types::Type::ObjectType(templateObject))) + types::StackTypeSet *thisTypes = types::TypeScript::ThisTypes(target->nonLazyScript()); + if (!thisTypes || !thisTypes->hasType(types::Type::ObjectType(templateObject))) return nullptr; // Generate an inline path to create a new |this| object with @@ -5580,6 +5578,9 @@ IonBuilder::testShouldDOMCall(types::TypeSet *inTypes, static bool ArgumentTypesMatch(MDefinition *def, types::StackTypeSet *calleeTypes) { + if (!calleeTypes) + return false; + if (def->resultTypeSet()) { MOZ_ASSERT(def->type() == MIRType_Value || def->mightBeType(def->type())); return def->resultTypeSet()->isSubset(calleeTypes); @@ -5605,9 +5606,6 @@ IonBuilder::testNeedsArgumentCheck(JSFunction *target, CallInfo &callInfo) JSScript *targetScript = target->nonLazyScript(); - if (!targetScript->types) - return true; - if (!ArgumentTypesMatch(callInfo.thisArg(), types::TypeScript::ThisTypes(targetScript))) return true; uint32_t expected_args = Min(callInfo.argc(), target->nargs()); diff --git a/js/src/jscompartment.cpp b/js/src/jscompartment.cpp index 57db299da36..b071929d36d 100644 --- a/js/src/jscompartment.cpp +++ b/js/src/jscompartment.cpp @@ -100,8 +100,6 @@ JSCompartment::init(JSContext *cx) if (cx) cx->runtime()->dateTimeInfo.updateTimeZoneAdjustment(); - activeAnalysis = false; - if (!crossCompartmentWrappers.init(0)) return false; diff --git a/js/src/jscompartment.h b/js/src/jscompartment.h index c74f1e3258e..dd010dbd38f 100644 --- a/js/src/jscompartment.h +++ b/js/src/jscompartment.h @@ -196,8 +196,6 @@ struct JSCompartment */ void adoptWorkerAllocator(js::Allocator *workerAllocator); - bool activeAnalysis; - /* Type information about the scripts and objects in this compartment. */ js::types::TypeCompartment types; diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index 50592d7b421..3ba9f854098 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -352,22 +352,6 @@ struct js::gc::FinalizePhase #define PHASE(x, p) { ArrayLength(x), x, p } -/* - * Finalization order for foreground non-incrementally swept things. - */ -static const AllocKind ImmediatePhaseObjects[] = { - FINALIZE_OBJECT0, - FINALIZE_OBJECT2, - FINALIZE_OBJECT4, - FINALIZE_OBJECT8, - FINALIZE_OBJECT12, - FINALIZE_OBJECT16 -}; - -static const FinalizePhase ImmediateFinalizePhases[] = { - PHASE(ImmediatePhaseObjects, gcstats::PHASE_SWEEP_OBJECT) -}; - /* * Finalization order for incrementally swept things. */ @@ -566,11 +550,13 @@ FinalizeTypedArenas(FreeOp *fop, ArenaHeader **src, SortedArenaList &dest, AllocKind thingKind, - SliceBudget &budget) + SliceBudget &budget, + ArenaLists::KeepArenasEnum keepArenas) { /* - * Finalize arenas from src list, releasing empty arenas and inserting the - * others into the appropriate destination size bins. + * Finalize arenas from src list, releasing empty arenas if keepArenas + * wasn't specified and inserting the others into the appropriate + * destination size bins. */ /* @@ -578,7 +564,7 @@ FinalizeTypedArenas(FreeOp *fop, * but in that case, we want to hold on to the memory in our arena * lists, not offer it up for reuse. */ - bool releaseArenas = !InParallelSection(); + MOZ_ASSERT_IF(InParallelSection(), keepArenas); size_t thingSize = Arena::thingSize(thingKind); size_t thingsPerArena = Arena::thingsPerArena(thingSize); @@ -590,10 +576,10 @@ FinalizeTypedArenas(FreeOp *fop, if (nmarked) dest.insertAt(aheader, nfree); - else if (releaseArenas) - aheader->chunk()->releaseArena(aheader); - else + else if (keepArenas) aheader->chunk()->recycleArena(aheader, dest, thingKind, thingsPerArena); + else + aheader->chunk()->releaseArena(aheader); budget.step(thingsPerArena); if (budget.isOverBudget()) @@ -612,7 +598,8 @@ FinalizeArenas(FreeOp *fop, ArenaHeader **src, SortedArenaList &dest, AllocKind thingKind, - SliceBudget &budget) + SliceBudget &budget, + ArenaLists::KeepArenasEnum keepArenas) { switch (thingKind) { case FINALIZE_OBJECT0: @@ -627,33 +614,33 @@ FinalizeArenas(FreeOp *fop, case FINALIZE_OBJECT12_BACKGROUND: case FINALIZE_OBJECT16: case FINALIZE_OBJECT16_BACKGROUND: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_SCRIPT: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_LAZY_SCRIPT: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_SHAPE: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_ACCESSOR_SHAPE: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_BASE_SHAPE: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_TYPE_OBJECT: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_STRING: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_FAT_INLINE_STRING: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_EXTERNAL_STRING: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_SYMBOL: - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); case FINALIZE_JITCODE: { // JitCode finalization may release references on an executable // allocator that is accessed when requesting interrupts. JSRuntime::AutoLockForInterrupt lock(fop->runtime()); - return FinalizeTypedArenas(fop, src, dest, thingKind, budget); + return FinalizeTypedArenas(fop, src, dest, thingKind, budget, keepArenas); } default: MOZ_CRASH("Invalid alloc kind"); @@ -2054,7 +2041,7 @@ ArenaLists::wipeDuringParallelExecution(JSRuntime *rt) if (!arenaLists[i].isEmpty()) { purge(thingKind); - forceFinalizeNow(&fop, thingKind); + forceFinalizeNow(&fop, thingKind, KEEP_ARENAS); } } } @@ -2506,18 +2493,18 @@ ArenaLists::finalizeNow(FreeOp *fop, const FinalizePhase& phase) { gcstats::AutoPhase ap(fop->runtime()->gc.stats, phase.statsPhase); for (unsigned i = 0; i < phase.length; ++i) - finalizeNow(fop, phase.kinds[i]); + finalizeNow(fop, phase.kinds[i], RELEASE_ARENAS, nullptr); } void -ArenaLists::finalizeNow(FreeOp *fop, AllocKind thingKind) +ArenaLists::finalizeNow(FreeOp *fop, AllocKind thingKind, KeepArenasEnum keepArenas, ArenaHeader **empty) { MOZ_ASSERT(!IsBackgroundFinalized(thingKind)); - forceFinalizeNow(fop, thingKind); + forceFinalizeNow(fop, thingKind, keepArenas, empty); } void -ArenaLists::forceFinalizeNow(FreeOp *fop, AllocKind thingKind) +ArenaLists::forceFinalizeNow(FreeOp *fop, AllocKind thingKind, KeepArenasEnum keepArenas, ArenaHeader **empty) { MOZ_ASSERT(backgroundFinalizeState[thingKind] == BFS_DONE); @@ -2530,9 +2517,14 @@ ArenaLists::forceFinalizeNow(FreeOp *fop, AllocKind thingKind) SortedArenaList finalizedSorted(thingsPerArena); SliceBudget budget; - FinalizeArenas(fop, &arenas, finalizedSorted, thingKind, budget); + FinalizeArenas(fop, &arenas, finalizedSorted, thingKind, budget, keepArenas); MOZ_ASSERT(!arenas); + if (empty) { + MOZ_ASSERT(keepArenas == KEEP_ARENAS); + finalizedSorted.extractEmpty(empty); + } + arenaLists[thingKind] = finalizedSorted.toArenaList(); } @@ -2586,6 +2578,8 @@ ArenaLists::queueForBackgroundSweep(FreeOp *fop, AllocKind thingKind) ArenaLists::backgroundFinalize(FreeOp *fop, ArenaHeader *listHead) { MOZ_ASSERT(listHead); + MOZ_ASSERT(!InParallelSection()); + AllocKind thingKind = listHead->getAllocKind(); Zone *zone = listHead->zone; @@ -2593,7 +2587,7 @@ ArenaLists::backgroundFinalize(FreeOp *fop, ArenaHeader *listHead) SortedArenaList finalizedSorted(thingsPerArena); SliceBudget budget; - FinalizeArenas(fop, &listHead, finalizedSorted, thingKind, budget); + FinalizeArenas(fop, &listHead, finalizedSorted, thingKind, budget, RELEASE_ARENAS); MOZ_ASSERT(!listHead); // When arenas are queued for background finalization, all arenas are moved @@ -2624,6 +2618,72 @@ ArenaLists::backgroundFinalize(FreeOp *fop, ArenaHeader *listHead) lists->backgroundFinalizeState[thingKind] = BFS_DONE; } +void +ArenaLists::queueForegroundObjectsForSweep(FreeOp *fop) +{ + gcstats::AutoPhase ap(fop->runtime()->gc.stats, gcstats::PHASE_SWEEP_OBJECT); + +#ifdef DEBUG + for (size_t i = 0; i < FINALIZE_OBJECT_LIMIT; i++) + MOZ_ASSERT(savedObjectArenas[i].isEmpty()); + MOZ_ASSERT(savedEmptyObjectArenas == nullptr); +#endif + + // Foreground finalized objects must be finalized at the beginning of the + // sweep phase, before control can return to the mutator. Otherwise, + // mutator behavior can resurrect certain objects whose references would + // otherwise have been erased by the finalizer. + finalizeNow(fop, FINALIZE_OBJECT0, KEEP_ARENAS, &savedEmptyObjectArenas); + finalizeNow(fop, FINALIZE_OBJECT2, KEEP_ARENAS, &savedEmptyObjectArenas); + finalizeNow(fop, FINALIZE_OBJECT4, KEEP_ARENAS, &savedEmptyObjectArenas); + finalizeNow(fop, FINALIZE_OBJECT8, KEEP_ARENAS, &savedEmptyObjectArenas); + finalizeNow(fop, FINALIZE_OBJECT12, KEEP_ARENAS, &savedEmptyObjectArenas); + finalizeNow(fop, FINALIZE_OBJECT16, KEEP_ARENAS, &savedEmptyObjectArenas); + + // Prevent the arenas from having new objects allocated into them. We need + // to know which objects are marked while we incrementally sweep dead + // references from type information. + savedObjectArenas[FINALIZE_OBJECT0] = arenaLists[FINALIZE_OBJECT0].copyAndClear(); + savedObjectArenas[FINALIZE_OBJECT2] = arenaLists[FINALIZE_OBJECT2].copyAndClear(); + savedObjectArenas[FINALIZE_OBJECT4] = arenaLists[FINALIZE_OBJECT4].copyAndClear(); + savedObjectArenas[FINALIZE_OBJECT8] = arenaLists[FINALIZE_OBJECT8].copyAndClear(); + savedObjectArenas[FINALIZE_OBJECT12] = arenaLists[FINALIZE_OBJECT12].copyAndClear(); + savedObjectArenas[FINALIZE_OBJECT16] = arenaLists[FINALIZE_OBJECT16].copyAndClear(); +} + +void +ArenaLists::mergeForegroundSweptObjectArenas() +{ + ReleaseArenaList(savedEmptyObjectArenas); + savedEmptyObjectArenas = nullptr; + + mergeSweptArenas(FINALIZE_OBJECT0); + mergeSweptArenas(FINALIZE_OBJECT2); + mergeSweptArenas(FINALIZE_OBJECT4); + mergeSweptArenas(FINALIZE_OBJECT8); + mergeSweptArenas(FINALIZE_OBJECT12); + mergeSweptArenas(FINALIZE_OBJECT16); +} + +inline void +ArenaLists::mergeSweptArenas(AllocKind thingKind) +{ + ArenaList *al = &arenaLists[thingKind]; + ArenaList *saved = &savedObjectArenas[thingKind]; + + *al = saved->insertListWithCursorAtEnd(*al); + saved->clear(); +} + +void +ArenaLists::queueForegroundThingsForSweep(FreeOp *fop) +{ + gcShapeArenasToUpdate = arenaListsToSweep[FINALIZE_SHAPE]; + gcAccessorShapeArenasToUpdate = arenaListsToSweep[FINALIZE_ACCESSOR_SHAPE]; + gcTypeObjectArenasToUpdate = arenaListsToSweep[FINALIZE_TYPE_OBJECT]; + gcScriptArenasToUpdate = arenaListsToSweep[FINALIZE_SCRIPT]; +} + static void * RunLastDitchGC(JSContext *cx, JS::Zone *zone, AllocKind thingKind) { @@ -4749,9 +4809,10 @@ GCRuntime::beginSweepingZoneGroup() } { - gcstats::AutoPhase ap(stats, gcstats::PHASE_DISCARD_ANALYSIS); + gcstats::AutoPhase ap1(stats, gcstats::PHASE_SWEEP_TYPES); + gcstats::AutoPhase ap2(stats, gcstats::PHASE_SWEEP_TYPES_BEGIN); for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { - zone->sweepAnalysis(&fop, releaseObservedTypes && !zone->isPreservingCode()); + zone->beginSweepTypes(&fop, releaseObservedTypes && !zone->isPreservingCode()); } } @@ -4799,8 +4860,7 @@ GCRuntime::beginSweepingZoneGroup() for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { gcstats::AutoSCC scc(stats, zoneGroupIndex); - for (unsigned i = 0; i < ArrayLength(ImmediateFinalizePhases); ++i) - zone->allocator.arenas.finalizeNow(&fop, ImmediateFinalizePhases[i]); + zone->allocator.arenas.queueForegroundObjectsForSweep(&fop); } for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { gcstats::AutoSCC scc(stats, zoneGroupIndex); @@ -4814,12 +4874,11 @@ GCRuntime::beginSweepingZoneGroup() } for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) { gcstats::AutoSCC scc(stats, zoneGroupIndex); - zone->allocator.arenas.gcShapeArenasToSweep = - zone->allocator.arenas.arenaListsToSweep[FINALIZE_SHAPE]; - zone->allocator.arenas.gcAccessorShapeArenasToSweep = - zone->allocator.arenas.arenaListsToSweep[FINALIZE_ACCESSOR_SHAPE]; + zone->allocator.arenas.queueForegroundThingsForSweep(&fop); } + sweepingTypes = true; + finalizePhase = 0; sweepZone = currentZoneGroup; sweepKindIndex = 0; @@ -4888,10 +4947,14 @@ bool ArenaLists::foregroundFinalize(FreeOp *fop, AllocKind thingKind, SliceBudget &sliceBudget, SortedArenaList &sweepList) { + MOZ_ASSERT(!InParallelSection()); + if (!arenaListsToSweep[thingKind] && incrementalSweptArenas.isEmpty()) return true; - if (!FinalizeArenas(fop, &arenaListsToSweep[thingKind], sweepList, thingKind, sliceBudget)) { + if (!FinalizeArenas(fop, &arenaListsToSweep[thingKind], sweepList, + thingKind, sliceBudget, RELEASE_ARENAS)) + { incrementalSweptArenaKind = thingKind; incrementalSweptArenas = sweepList.toArenaList(); return false; @@ -4915,8 +4978,18 @@ GCRuntime::drainMarkStack(SliceBudget &sliceBudget, gcstats::Phase phase) return marker.drainMarkStack(sliceBudget); } +// Advance to the next entry in a list of arenas, returning false if the +// mutator should resume running. static bool -SweepShapes(ArenaHeader **arenasToSweep, size_t thingsPerArena, SliceBudget &sliceBudget) +AdvanceArenaList(ArenaHeader **list, AllocKind kind, SliceBudget &sliceBudget) +{ + *list = (*list)->next; + sliceBudget.step(Arena::thingsPerArena(Arena::thingSize(kind))); + return !sliceBudget.isOverBudget(); +} + +static bool +SweepShapes(ArenaHeader **arenasToSweep, AllocKind kind, SliceBudget &sliceBudget) { while (ArenaHeader *arena = *arenasToSweep) { for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) { @@ -4925,10 +4998,8 @@ SweepShapes(ArenaHeader **arenasToSweep, size_t thingsPerArena, SliceBudget &sli shape->sweep(); } - *arenasToSweep = arena->next; - sliceBudget.step(thingsPerArena); - if (sliceBudget.isOverBudget()) - return false; /* Yield to the mutator. */ + if (!AdvanceArenaList(arenasToSweep, kind, sliceBudget)) + return false; } return true; @@ -4945,6 +5016,64 @@ GCRuntime::sweepPhase(SliceBudget &sliceBudget) return false; for (;;) { + // Sweep dead type information stored in scripts and type objects, but + // don't finalize them yet. We have to sweep dead information from both + // live and dead scripts and type objects, so that no dead references + // remain in them. Type inference can end up crawling these zones + // again, such as for TypeCompartment::markSetsUnknown, and if this + // happens after sweeping for the zone group finishes we won't be able + // to determine which things in the zone are live. + if (sweepingTypes) { + gcstats::AutoPhase ap1(stats, gcstats::PHASE_SWEEP_COMPARTMENTS); + gcstats::AutoPhase ap2(stats, gcstats::PHASE_SWEEP_TYPES); + + for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) { + ArenaLists &al = sweepZone->allocator.arenas; + + types::AutoClearTypeInferenceStateOnOOM oom(sweepZone); + + while (ArenaHeader *arena = al.gcScriptArenasToUpdate) { + for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) { + JSScript *script = i.get(); + script->maybeSweepTypes(&oom); + } + + if (!AdvanceArenaList(&al.gcScriptArenasToUpdate, + FINALIZE_SCRIPT, sliceBudget)) + { + return false; + } + } + + while (ArenaHeader *arena = al.gcTypeObjectArenasToUpdate) { + for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) { + types::TypeObject *object = i.get(); + object->maybeSweep(&oom); + } + + if (!AdvanceArenaList(&al.gcTypeObjectArenasToUpdate, + FINALIZE_TYPE_OBJECT, sliceBudget)) + { + return false; + } + } + + // Finish sweeping type information in the zone. + { + gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_TYPES_END); + sweepZone->types.endSweep(rt); + } + + // Foreground finalized objects have already been finalized, + // and now their arenas can be reclaimed by freeing empty ones + // and making non-empty ones available for allocation. + al.mergeForegroundSweptObjectArenas(); + } + + sweepZone = currentZoneGroup; + sweepingTypes = false; + } + /* Finalize foreground finalized things. */ for (; finalizePhase < ArrayLength(IncrementalFinalizePhases) ; ++finalizePhase) { gcstats::AutoPhase ap(stats, IncrementalFinalizePhases[finalizePhase].statsPhase); @@ -4979,15 +5108,13 @@ GCRuntime::sweepPhase(SliceBudget &sliceBudget) for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) { Zone *zone = sweepZone; - if (!SweepShapes(&zone->allocator.arenas.gcShapeArenasToSweep, - Arena::thingsPerArena(Arena::thingSize(FINALIZE_SHAPE)), - sliceBudget)) + if (!SweepShapes(&zone->allocator.arenas.gcShapeArenasToUpdate, + FINALIZE_SHAPE, sliceBudget)) { return false; /* Yield to the mutator. */ } - if (!SweepShapes(&zone->allocator.arenas.gcAccessorShapeArenasToSweep, - Arena::thingsPerArena(Arena::thingSize(FINALIZE_ACCESSOR_SHAPE)), - sliceBudget)) + if (!SweepShapes(&zone->allocator.arenas.gcAccessorShapeArenasToUpdate, + FINALIZE_ACCESSOR_SHAPE, sliceBudget)) { return false; /* Yield to the mutator. */ } @@ -6107,12 +6234,14 @@ gc::MergeCompartments(JSCompartment *source, JSCompartment *target) source->clearTables(); - // Fixup compartment pointers in source to refer to target. + // Fixup compartment pointers in source to refer to target, and make sure + // type information generations are in sync. for (ZoneCellIter iter(source->zone(), FINALIZE_SCRIPT); !iter.done(); iter.next()) { JSScript *script = iter.get(); MOZ_ASSERT(script->compartment() == source); script->compartment_ = target; + script->setTypesGeneration(target->zone()->types.generation); } for (ZoneCellIter iter(source->zone(), FINALIZE_BASE_SHAPE); !iter.done(); iter.next()) { @@ -6121,6 +6250,11 @@ gc::MergeCompartments(JSCompartment *source, JSCompartment *target) base->compartment_ = target; } + for (ZoneCellIter iter(source->zone(), FINALIZE_TYPE_OBJECT); !iter.done(); iter.next()) { + types::TypeObject *type = iter.get(); + type->setGeneration(target->zone()->types.generation); + } + // Fixup zone pointers in source's zone to refer to target's zone. for (size_t thingKind = 0; thingKind != FINALIZE_LIMIT; thingKind++) { diff --git a/js/src/jsgc.h b/js/src/jsgc.h index ce51940bd29..1b5660730b8 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -421,6 +421,12 @@ class ArenaList { check(); } + ArenaList copyAndClear() { + ArenaList result = *this; + clear(); + return result; + } + bool isEmpty() const { check(); return !head_; @@ -548,6 +554,16 @@ class SortedArenaList segments[nfree].append(aheader); } + // Remove all empty arenas, inserting them as a linked list. + void extractEmpty(ArenaHeader **empty) { + SortedArenaListSegment &segment = segments[thingsPerArena_]; + if (segment.head) { + *segment.tailp = *empty; + *empty = segment.head; + segment.clear(); + } + } + // Links up the tail of each non-empty segment to the head of the next // non-empty segment, creating a contiguous list that is returned as an // ArenaList. This is not a destructive operation: neither the head nor tail @@ -604,9 +620,19 @@ class ArenaLists unsigned incrementalSweptArenaKind; ArenaList incrementalSweptArenas; - /* Shape arenas to be swept in the foreground. */ - ArenaHeader *gcShapeArenasToSweep; - ArenaHeader *gcAccessorShapeArenasToSweep; + // Arena lists which have yet to be swept, but need additional foreground + // processing before they are swept. + ArenaHeader *gcShapeArenasToUpdate; + ArenaHeader *gcAccessorShapeArenasToUpdate; + ArenaHeader *gcScriptArenasToUpdate; + ArenaHeader *gcTypeObjectArenasToUpdate; + + // While sweeping type information, these lists save the arenas for the + // objects which have already been finalized in the foreground (which must + // happen at the beginning of the GC), so that type sweeping can determine + // which of the object pointers are marked. + ArenaList savedObjectArenas[FINALIZE_OBJECT_LIMIT]; + ArenaHeader *savedEmptyObjectArenas; public: ArenaLists() { @@ -617,8 +643,11 @@ class ArenaLists for (size_t i = 0; i != FINALIZE_LIMIT; ++i) arenaListsToSweep[i] = nullptr; incrementalSweptArenaKind = FINALIZE_LIMIT; - gcShapeArenasToSweep = nullptr; - gcAccessorShapeArenasToSweep = nullptr; + gcShapeArenasToUpdate = nullptr; + gcAccessorShapeArenasToUpdate = nullptr; + gcScriptArenasToUpdate = nullptr; + gcTypeObjectArenasToUpdate = nullptr; + savedEmptyObjectArenas = nullptr; } ~ArenaLists() { @@ -628,19 +657,13 @@ class ArenaLists * the background finalization is disabled. */ MOZ_ASSERT(backgroundFinalizeState[i] == BFS_DONE); - ArenaHeader *next; - for (ArenaHeader *aheader = arenaLists[i].head(); aheader; aheader = next) { - // Copy aheader->next before releasing. - next = aheader->next; - aheader->chunk()->releaseArena(aheader); - } - } - ArenaHeader *next; - for (ArenaHeader *aheader = incrementalSweptArenas.head(); aheader; aheader = next) { - // Copy aheader->next before releasing. - next = aheader->next; - aheader->chunk()->releaseArena(aheader); + ReleaseArenaList(arenaLists[i].head()); } + ReleaseArenaList(incrementalSweptArenas.head()); + + for (size_t i = 0; i < FINALIZE_OBJECT_LIMIT; i++) + ReleaseArenaList(savedObjectArenas[i].head()); + ReleaseArenaList(savedEmptyObjectArenas); } static uintptr_t getFreeListOffset(AllocKind thingKind) { @@ -823,11 +846,10 @@ class ArenaLists ArenaHeader *relocateArenas(ArenaHeader *relocatedList); #endif - void queueObjectsForSweep(FreeOp *fop); - void queueStringsAndSymbolsForSweep(FreeOp *fop); - void queueShapesForSweep(FreeOp *fop); - void queueScriptsForSweep(FreeOp *fop); - void queueJitCodeForSweep(FreeOp *fop); + void queueForegroundObjectsForSweep(FreeOp *fop); + void queueForegroundThingsForSweep(FreeOp *fop); + + void mergeForegroundSweptObjectArenas(); bool foregroundFinalize(FreeOp *fop, AllocKind thingKind, SliceBudget &sliceBudget, SortedArenaList &sweepList); @@ -835,14 +857,25 @@ class ArenaLists void wipeDuringParallelExecution(JSRuntime *rt); + // When finalizing arenas, whether to keep empty arenas on the list or + // release them immediately. + enum KeepArenasEnum { + RELEASE_ARENAS, + KEEP_ARENAS + }; + private: inline void finalizeNow(FreeOp *fop, const FinalizePhase& phase); inline void queueForForegroundSweep(FreeOp *fop, const FinalizePhase& phase); inline void queueForBackgroundSweep(FreeOp *fop, const FinalizePhase& phase); - inline void finalizeNow(FreeOp *fop, AllocKind thingKind); - inline void forceFinalizeNow(FreeOp *fop, AllocKind thingKind); + + inline void finalizeNow(FreeOp *fop, AllocKind thingKind, + KeepArenasEnum keepArenas, ArenaHeader **empty = nullptr); + inline void forceFinalizeNow(FreeOp *fop, AllocKind thingKind, + KeepArenasEnum keepArenas, ArenaHeader **empty = nullptr); inline void queueForForegroundSweep(FreeOp *fop, AllocKind thingKind); inline void queueForBackgroundSweep(FreeOp *fop, AllocKind thingKind); + inline void mergeSweptArenas(AllocKind thingKind); TenuredCell *allocateFromArena(JS::Zone *zone, AllocKind thingKind, AutoMaybeStartBackgroundAllocation &maybeStartBGAlloc); diff --git a/js/src/jsinfer.cpp b/js/src/jsinfer.cpp index f64dc2f5a87..39ed1d04503 100644 --- a/js/src/jsinfer.cpp +++ b/js/src/jsinfer.cpp @@ -461,7 +461,7 @@ ConstraintTypeSet::addConstraint(JSContext *cx, TypeConstraint *constraint, bool return false; } - MOZ_ASSERT(cx->compartment()->activeAnalysis); + MOZ_ASSERT(cx->zone()->types.activeAnalysis); InferSpew(ISpewOps, "addConstraint: %sT%p%s %sC%p%s %s", InferSpewColor(this), this, InferSpewColorReset(), @@ -555,7 +555,7 @@ TypeSet::addType(Type type, LifoAlloc *alloc) void ConstraintTypeSet::addType(ExclusiveContext *cxArg, Type type) { - MOZ_ASSERT(cxArg->compartment()->activeAnalysis); + MOZ_ASSERT(cxArg->zone()->types.activeAnalysis); if (hasType(type)) return; @@ -924,7 +924,7 @@ TypeScript::FreezeTypeSets(CompilerConstraintList *constraints, JSScript *script TemporaryTypeSet **pBytecodeTypes) { LifoAlloc *alloc = constraints->alloc(); - StackTypeSet *existing = script->types->typeArray(); + StackTypeSet *existing = script->types()->typeArray(); size_t count = NumTypeSets(script); TemporaryTypeSet *types = alloc->newArrayUninitialized(count); @@ -1192,7 +1192,7 @@ types::FinishCompilation(JSContext *cx, HandleScript script, ExecutionMode execu TypeZone &types = cx->zone()->types; if (!types.compilerOutputs) { - types.compilerOutputs = cx->new_< Vector >(cx); + types.compilerOutputs = cx->new_(); if (!types.compilerOutputs) return false; } @@ -1205,10 +1205,12 @@ types::FinishCompilation(JSContext *cx, HandleScript script, ExecutionMode execu #endif uint32_t index = types.compilerOutputs->length(); - if (!types.compilerOutputs->append(co)) + if (!types.compilerOutputs->append(co)) { + js_ReportOutOfMemory(cx); return false; + } - *precompileInfo = RecompileInfo(index); + *precompileInfo = RecompileInfo(index, types.generation); bool succeeded = true; @@ -1220,7 +1222,10 @@ types::FinishCompilation(JSContext *cx, HandleScript script, ExecutionMode execu for (size_t i = 0; i < constraints->numFrozenScripts(); i++) { const CompilerConstraintList::FrozenScript &entry = constraints->frozenScript(i); - MOZ_ASSERT(entry.script->types); + if (!entry.script->types()) { + succeeded = false; + break; + } if (!CheckFrozenTypeSet(cx, entry.thisTypes, types::TypeScript::ThisTypes(entry.script))) succeeded = false; @@ -1232,7 +1237,7 @@ types::FinishCompilation(JSContext *cx, HandleScript script, ExecutionMode execu succeeded = false; } for (size_t i = 0; i < entry.script->nTypeSets(); i++) { - if (!CheckFrozenTypeSet(cx, &entry.bytecodeTypes[i], &entry.script->types->typeArray()[i])) + if (!CheckFrozenTypeSet(cx, &entry.bytecodeTypes[i], &entry.script->types()->typeArray()[i])) succeeded = false; } @@ -1244,7 +1249,7 @@ types::FinishCompilation(JSContext *cx, HandleScript script, ExecutionMode execu size_t count = TypeScript::NumTypeSets(entry.script); - StackTypeSet *array = entry.script->types->typeArray(); + StackTypeSet *array = entry.script->types()->typeArray(); for (size_t i = 0; i < count; i++) { if (!array[i].addConstraint(cx, cx->typeLifoAlloc().new_(entry.script), false)) succeeded = false; @@ -1288,7 +1293,7 @@ types::FinishDefinitePropertiesAnalysis(JSContext *cx, CompilerConstraintList *c for (size_t i = 0; i < constraints->numFrozenScripts(); i++) { const CompilerConstraintList::FrozenScript &entry = constraints->frozenScript(i); JSScript *script = entry.script; - MOZ_ASSERT(script->types); + MOZ_ASSERT(script->types()); MOZ_ASSERT(TypeScript::ThisTypes(script)->isSubset(entry.thisTypes)); @@ -1299,16 +1304,14 @@ types::FinishDefinitePropertiesAnalysis(JSContext *cx, CompilerConstraintList *c MOZ_ASSERT(TypeScript::ArgTypes(script, j)->isSubset(&entry.argTypes[j])); for (size_t j = 0; j < script->nTypeSets(); j++) - MOZ_ASSERT(script->types->typeArray()[j].isSubset(&entry.bytecodeTypes[j])); + MOZ_ASSERT(script->types()->typeArray()[j].isSubset(&entry.bytecodeTypes[j])); } #endif for (size_t i = 0; i < constraints->numFrozenScripts(); i++) { const CompilerConstraintList::FrozenScript &entry = constraints->frozenScript(i); JSScript *script = entry.script; - MOZ_ASSERT(script->types); - - if (!script->types) + if (!script->types()) MOZ_CRASH(); CheckDefinitePropertiesTypeSet(cx, entry.thisTypes, TypeScript::ThisTypes(script)); @@ -1320,7 +1323,7 @@ types::FinishDefinitePropertiesAnalysis(JSContext *cx, CompilerConstraintList *c CheckDefinitePropertiesTypeSet(cx, &entry.argTypes[j], TypeScript::ArgTypes(script, j)); for (size_t j = 0; j < script->nTypeSets(); j++) - CheckDefinitePropertiesTypeSet(cx, &entry.bytecodeTypes[j], &script->types->typeArray()[j]); + CheckDefinitePropertiesTypeSet(cx, &entry.bytecodeTypes[j], &script->types()->typeArray()[j]); } } @@ -2390,20 +2393,24 @@ types::TypeCanHaveExtraIndexedProperties(CompilerConstraintList *constraints, } void -TypeZone::processPendingRecompiles(FreeOp *fop) +TypeZone::processPendingRecompiles(FreeOp *fop, RecompileInfoVector &recompiles) { - if (!pendingRecompiles) - return; + MOZ_ASSERT(!recompiles.empty()); - /* Steal the list of scripts to recompile, else we will try to recursively recompile them. */ - Vector *pending = pendingRecompiles; - pendingRecompiles = nullptr; + /* + * Steal the list of scripts to recompile, to make sure we don't try to + * recursively recompile them. + */ + RecompileInfoVector pending; + for (size_t i = 0; i < recompiles.length(); i++) { + if (!pending.append(recompiles[i])) + CrashAtUnhandlableOOM("processPendingRecompiles"); + } + recompiles.clear(); - MOZ_ASSERT(!pending->empty()); + jit::Invalidate(*this, fop, pending); - jit::Invalidate(*this, fop, *pending); - - fop->delete_(pending); + MOZ_ASSERT(recompiles.empty()); } void @@ -2418,13 +2425,7 @@ TypeZone::addPendingRecompile(JSContext *cx, const RecompileInfo &info) co->setPendingInvalidation(); - if (!pendingRecompiles) { - pendingRecompiles = cx->new_< Vector >(cx); - if (!pendingRecompiles) - CrashAtUnhandlableOOM("Could not update pendingRecompiles"); - } - - if (!pendingRecompiles->append(info)) + if (!cx->zone()->types.activeAnalysis->pendingRecompiles.append(info)) CrashAtUnhandlableOOM("Could not update pendingRecompiles"); } @@ -2476,9 +2477,9 @@ TypeCompartment::markSetsUnknown(JSContext *cx, TypeObject *target) for (gc::ZoneCellIter i(cx->zone(), gc::FINALIZE_SCRIPT); !i.done(); i.next()) { JSScript *script = i.get(); - if (script->types) { + if (script->types()) { unsigned count = TypeScript::NumTypeSets(script); - StackTypeSet *typeArray = script->types->typeArray(); + StackTypeSet *typeArray = script->types()->typeArray(); for (unsigned i = 0; i < count; i++) { if (typeArray[i].hasType(Type::ObjectType(target))) typeArray[i].addType(cx, Type::AnyObjectType()); @@ -2496,21 +2497,21 @@ TypeCompartment::print(JSContext *cx, bool force) gc::AutoSuppressGC suppressGC(cx); JSAutoRequest request(cx); - JSCompartment *compartment = this->compartment(); - AutoEnterAnalysis enter(nullptr, compartment); + Zone *zone = compartment()->zone(); + AutoEnterAnalysis enter(nullptr, zone); if (!force && !InferSpewActive(ISpewResult)) return; - for (gc::ZoneCellIter i(compartment->zone(), gc::FINALIZE_SCRIPT); !i.done(); i.next()) { + for (gc::ZoneCellIter i(zone, gc::FINALIZE_SCRIPT); !i.done(); i.next()) { // Note: use cx->runtime() instead of cx to work around IsInRequest(cx) // assertion failures when we're called from DestroyContext. RootedScript script(cx->runtime(), i.get()); - if (script->types) - script->types->printTypes(cx, script); + if (script->types()) + script->types()->printTypes(cx, script); } - for (gc::ZoneCellIter i(compartment->zone(), gc::FINALIZE_TYPE_OBJECT); !i.done(); i.next()) { + for (gc::ZoneCellIter i(zone, gc::FINALIZE_TYPE_OBJECT); !i.done(); i.next()) { TypeObject *object = i.get(); object->print(); } @@ -2587,7 +2588,7 @@ void TypeCompartment::setTypeToHomogenousArray(ExclusiveContext *cx, JSObject *obj, Type elementType) { - MOZ_ASSERT(cx->compartment()->activeAnalysis); + MOZ_ASSERT(cx->zone()->types.activeAnalysis); if (!arrayTypeTable) { arrayTypeTable = cx->new_(); @@ -3204,7 +3205,7 @@ TypeObject::markUnknown(ExclusiveContext *cx) { AutoEnterAnalysis enter(cx); - MOZ_ASSERT(cx->compartment()->activeAnalysis); + MOZ_ASSERT(cx->zone()->types.activeAnalysis); MOZ_ASSERT(!unknownProperties()); InferSpew(ISpewOps, "UnknownProperties: %s", TypeObjectString(this)); @@ -3234,7 +3235,7 @@ TypeObject::markUnknown(ExclusiveContext *cx) void TypeObject::maybeClearNewScriptOnOOM() { - MOZ_ASSERT(zone()->isGCSweeping()); + MOZ_ASSERT(zone()->isGCSweepingOrCompacting()); if (!isMarked()) return; @@ -3457,7 +3458,7 @@ types::AddClearDefiniteFunctionUsesInScript(JSContext *cx, TypeObject *type, TypeObjectKey *calleeKey = Type::ObjectType(calleeScript->functionNonDelazifying()).objectKey(); unsigned count = TypeScript::NumTypeSets(script); - StackTypeSet *typeArray = script->types->typeArray(); + StackTypeSet *typeArray = script->types()->typeArray(); for (unsigned i = 0; i < count; i++) { StackTypeSet *types = &typeArray[i]; @@ -3667,7 +3668,7 @@ types::UseNewTypeForClone(JSFunction *fun) bool JSScript::makeTypes(JSContext *cx) { - MOZ_ASSERT(!types); + MOZ_ASSERT(!types_); AutoEnterAnalysis enter(cx); @@ -3678,7 +3679,8 @@ JSScript::makeTypes(JSContext *cx) if (!typeScript) return false; - types = typeScript; + types_ = typeScript; + setTypesGeneration(cx->zone()->types.generation); #ifdef DEBUG StackTypeSet *typeArray = typeScript->typeArray(); @@ -3734,7 +3736,7 @@ JSFunction::setTypeForScriptedFunction(ExclusiveContext *cx, HandleFunction fun, /* static */ void TypeNewScript::make(JSContext *cx, TypeObject *type, JSFunction *fun) { - MOZ_ASSERT(cx->compartment()->activeAnalysis); + MOZ_ASSERT(cx->zone()->types.activeAnalysis); MOZ_ASSERT(!type->newScript()); if (type->unknownProperties()) @@ -3882,6 +3884,12 @@ TypeNewScript::maybeAnalyze(JSContext *cx, TypeObject *type, bool *regenerate, b // whether the new type table was updated and type needs to be refreshed. MOZ_ASSERT(this == type->newScript()); + // Make sure there aren't dead references in preliminaryObjects. This can + // clear out the new script information on OOM. + type->maybeSweep(nullptr); + if (!type->newScript()) + return true; + if (regenerate) *regenerate = false; @@ -4177,7 +4185,7 @@ TypeNewScript::trace(JSTracer *trc) } void -TypeNewScript::sweep(FreeOp *fop) +TypeNewScript::sweep() { // preliminaryObjects only holds weak pointers, so clear any objects that // are about to be destroyed. @@ -4531,8 +4539,15 @@ ExclusiveContext::getSingletonType(const Class *clasp, TaggedProto proto) ///////////////////////////////////////////////////////////////////// void -ConstraintTypeSet::sweep(Zone *zone, bool *oom) +ConstraintTypeSet::sweep(Zone *zone, AutoClearTypeInferenceStateOnOOM &oom) { + MOZ_ASSERT(zone->isGCSweepingOrCompacting()); + + // IsAboutToBeFinalized doesn't work right on tenured objects when called + // during a minor collection. + MOZ_ASSERT(!zone->runtimeFromMainThread()->isHeapMinorCollecting()); + MOZ_ASSERT(!zone->runtimeFromMainThread()->isFJMinorCollecting()); + /* * Purge references to type objects that are no longer live. Type sets hold * only weak references. For type sets containing more than one object, @@ -4555,7 +4570,7 @@ ConstraintTypeSet::sweep(Zone *zone, bool *oom) if (pentry) { *pentry = object; } else { - *oom = true; + oom.setOOM(); flags |= TYPE_FLAG_ANYOBJECT; clearObjects(); objectCount = 0; @@ -4587,7 +4602,7 @@ ConstraintTypeSet::sweep(Zone *zone, bool *oom) copy->next = constraintList; constraintList = copy; } else { - *oom = true; + oom.setOOM(); } } constraint = constraint->next; @@ -4601,27 +4616,53 @@ TypeObject::clearProperties() propertySet = nullptr; } +#ifdef DEBUG +bool +TypeObject::needsSweep() +{ + return generation() != zone()->types.generation; +} +#endif + +static void +EnsureHasAutoClearTypeInferenceStateOnOOM(AutoClearTypeInferenceStateOnOOM *&oom, Zone *zone, + Maybe &fallback) +{ + if (!oom) { + if (zone->types.activeAnalysis) { + oom = &zone->types.activeAnalysis->oom; + } else { + fallback.emplace(zone); + oom = &fallback.ref(); + } + } +} + /* * Before sweeping the arenas themselves, scan all type objects in a * compartment to fixup weak references: property type sets referencing dead * JS and type objects, and singleton JS objects whose type is not referenced - * elsewhere. This also releases memory associated with dead type objects, - * so that type objects do not need later finalization. + * elsewhere. This is done either incrementally as part of the sweep, or on + * demand as type objects are accessed before their contents have been swept. */ -inline void -TypeObject::sweep(FreeOp *fop, bool *oom) +void +TypeObject::maybeSweep(AutoClearTypeInferenceStateOnOOM *oom) { - MOZ_ASSERT(zone()->isGCSweepingOrCompacting()); - - if (zone()->isGCSweeping() && !isMarked()) { - // Take care of any finalization required for this object. - if (newScript()) - fop->delete_(newScript()); + if (generation() == zone()->types.generation) { + // No sweeping required. return; } + setGeneration(zone()->types.generation); + + MOZ_ASSERT(zone()->isGCSweepingOrCompacting()); + MOZ_ASSERT(!zone()->runtimeFromMainThread()->isHeapMinorCollecting()); + + Maybe fallbackOOM; + EnsureHasAutoClearTypeInferenceStateOnOOM(oom, zone(), fallbackOOM); + if (newScript()) - newScript()->sweep(fop); + newScript()->sweep(); LifoAlloc &typeLifoAlloc = zone()->types.typeLifoAlloc; @@ -4656,12 +4697,12 @@ TypeObject::sweep(FreeOp *fop, bool *oom) (typeLifoAlloc, propertySet, propertyCount, prop->id); if (pentry) { *pentry = newProp; - newProp->types.sweep(zone(), oom); + newProp->types.sweep(zone(), *oom); continue; } } - *oom = true; + oom->setOOM(); addFlags(OBJECT_FLAG_DYNAMIC_MASK | OBJECT_FLAG_UNKNOWN_PROPERTIES); clearProperties(); return; @@ -4677,9 +4718,9 @@ TypeObject::sweep(FreeOp *fop, bool *oom) Property *newProp = typeLifoAlloc.new_(*prop); if (newProp) { propertySet = (Property **) newProp; - newProp->types.sweep(zone(), oom); + newProp->types.sweep(zone(), *oom); } else { - *oom = true; + oom->setOOM(); addFlags(OBJECT_FLAG_DYNAMIC_MASK | OBJECT_FLAG_UNKNOWN_PROPERTIES); clearProperties(); return; @@ -4917,17 +4958,51 @@ TypeCompartment::~TypeCompartment() } /* static */ void -TypeScript::Sweep(FreeOp *fop, JSScript *script, bool *oom) +JSScript::maybeSweepTypes(AutoClearTypeInferenceStateOnOOM *oom) { - JSCompartment *compartment = script->compartment(); - MOZ_ASSERT(compartment->zone()->isGCSweepingOrCompacting()); + if (!types_ || typesGeneration() == zone()->types.generation) + return; - unsigned num = NumTypeSets(script); - StackTypeSet *typeArray = script->types->typeArray(); + setTypesGeneration(zone()->types.generation); - /* Remove constraints and references to dead objects from the persistent type sets. */ + MOZ_ASSERT(zone()->isGCSweepingOrCompacting()); + MOZ_ASSERT(!zone()->runtimeFromMainThread()->isHeapMinorCollecting()); + + Maybe fallbackOOM; + EnsureHasAutoClearTypeInferenceStateOnOOM(oom, zone(), fallbackOOM); + + TypeZone &types = zone()->types; + + // Destroy all type information attached to the script if desired. We can + // only do this if nothing has been compiled for the script, which will be + // the case unless the script has been compiled since we started sweeping. + if (types.sweepReleaseTypes && + !hasBaselineScript() && + !hasIonScript() && + !hasParallelIonScript()) + { + types_->destroy(); + types_ = nullptr; + + // Freeze constraints on stack type sets need to be regenerated the + // next time the script is analyzed. + hasFreezeConstraints_ = false; + + return; + } + + unsigned num = TypeScript::NumTypeSets(this); + StackTypeSet *typeArray = types_->typeArray(); + + // Remove constraints and references to dead objects from stack type sets. for (unsigned i = 0; i < num; i++) - typeArray[i].sweep(compartment->zone(), oom); + typeArray[i].sweep(zone(), *oom); + + // Update the recompile indexes in any IonScripts still on the script. + if (hasIonScript()) + ionScript()->recompileInfoRef().shouldSweep(types); + if (hasParallelIonScript()) + parallelIonScript()->recompileInfoRef().shouldSweep(types); } void @@ -4985,115 +5060,83 @@ TypeObject::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const TypeZone::TypeZone(Zone *zone) : zone_(zone), typeLifoAlloc(TYPE_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), + generation(0), compilerOutputs(nullptr), - pendingRecompiles(nullptr) + sweepTypeLifoAlloc(TYPE_LIFO_ALLOC_PRIMARY_CHUNK_SIZE), + sweepCompilerOutputs(nullptr), + sweepReleaseTypes(false), + activeAnalysis(nullptr) { } TypeZone::~TypeZone() { js_delete(compilerOutputs); - js_delete(pendingRecompiles); + js_delete(sweepCompilerOutputs); } void -TypeZone::sweep(FreeOp *fop, bool releaseTypes, bool *oom) +TypeZone::beginSweep(FreeOp *fop, bool releaseTypes, AutoClearTypeInferenceStateOnOOM &oom) { MOZ_ASSERT(zone()->isGCSweepingOrCompacting()); + MOZ_ASSERT(!sweepCompilerOutputs); + MOZ_ASSERT(!sweepReleaseTypes); - JSRuntime *rt = fop->runtime(); + sweepReleaseTypes = releaseTypes; - /* - * Clear the analysis pool, but don't release its data yet. While - * sweeping types any live data will be allocated into the pool. - */ - LifoAlloc oldAlloc(typeLifoAlloc.defaultChunkSize()); - oldAlloc.steal(&typeLifoAlloc); - - /* Sweep and find compressed indexes for each compiler output. */ - size_t newCompilerOutputCount = 0; + // Clear the analysis pool, but don't release its data yet. While sweeping + // types any live data will be allocated into the pool. + sweepTypeLifoAlloc.steal(&typeLifoAlloc); + // Sweep any invalid or dead compiler outputs, and keep track of the new + // index for remaining live outputs. if (compilerOutputs) { + CompilerOutputVector *newCompilerOutputs = nullptr; for (size_t i = 0; i < compilerOutputs->length(); i++) { CompilerOutput &output = (*compilerOutputs)[i]; if (output.isValid()) { JSScript *script = output.script(); + ExecutionMode mode = output.mode(); if (IsScriptAboutToBeFinalized(&script)) { - jit::GetIonScript(script, output.mode())->recompileInfoRef() = RecompileInfo(uint32_t(-1)); + jit::GetIonScript(script, mode)->recompileInfoRef() = RecompileInfo(); output.invalidate(); } else { - output.setSweepIndex(newCompilerOutputCount++); + CompilerOutput newOutput(script, output.mode()); + + if (!newCompilerOutputs) + newCompilerOutputs = js_new(); + if (newCompilerOutputs && newCompilerOutputs->append(newOutput)) { + output.setSweepIndex(newCompilerOutputs->length() - 1); + } else { + oom.setOOM(); + jit::GetIonScript(script, mode)->recompileInfoRef() = RecompileInfo(); + output.invalidate(); + } } } } + sweepCompilerOutputs = compilerOutputs; + compilerOutputs = newCompilerOutputs; } - { - gcstats::AutoPhase ap2(rt->gc.stats, !rt->isHeapCompacting(), - gcstats::PHASE_DISCARD_TI); + // All existing RecompileInfos are stale and will be updated to the new + // compiler outputs list later during the sweep. Don't worry about overflow + // here, since stale indexes will persist only until the sweep finishes. + generation++; - for (ZoneCellIterUnderGC i(zone(), FINALIZE_SCRIPT); !i.done(); i.next()) { - JSScript *script = i.get(); - if (script->types) { - types::TypeScript::Sweep(fop, script, oom); + for (CompartmentsInZoneIter comp(zone()); !comp.done(); comp.next()) + comp->types.sweep(fop); +} - if (releaseTypes) { - script->types->destroy(); - script->types = nullptr; +void +TypeZone::endSweep(JSRuntime *rt) +{ + js_delete(sweepCompilerOutputs); + sweepCompilerOutputs = nullptr; - /* - * Freeze constraints on stack type sets need to be - * regenerated the next time the script is analyzed. - */ - script->clearHasFreezeConstraints(); + sweepReleaseTypes = false; - MOZ_ASSERT(!script->hasIonScript()); - MOZ_ASSERT(!script->hasParallelIonScript()); - } else { - /* Update the recompile indexes in any IonScripts still on the script. */ - if (script->hasIonScript()) - script->ionScript()->recompileInfoRef().shouldSweep(*this); - if (script->hasParallelIonScript()) - script->parallelIonScript()->recompileInfoRef().shouldSweep(*this); - } - } - } - } - - { - gcstats::AutoPhase ap2(rt->gc.stats, !rt->isHeapCompacting(), - gcstats::PHASE_SWEEP_TYPES); - - for (gc::ZoneCellIterUnderGC iter(zone(), gc::FINALIZE_TYPE_OBJECT); - !iter.done(); iter.next()) - { - TypeObject *object = iter.get(); - object->sweep(fop, oom); - } - - for (CompartmentsInZoneIter comp(zone()); !comp.done(); comp.next()) - comp->types.sweep(fop); - } - - if (compilerOutputs) { - size_t sweepIndex = 0; - for (size_t i = 0; i < compilerOutputs->length(); i++) { - CompilerOutput output = (*compilerOutputs)[i]; - if (output.isValid()) { - MOZ_ASSERT(sweepIndex == output.sweepIndex()); - output.invalidateSweepIndex(); - (*compilerOutputs)[sweepIndex++] = output; - } - } - MOZ_ASSERT(sweepIndex == newCompilerOutputCount); - JS_ALWAYS_TRUE(compilerOutputs->resize(newCompilerOutputCount)); - } - - { - gcstats::AutoPhase ap2(rt->gc.stats, !rt->isHeapCompacting(), - gcstats::PHASE_FREE_TI_ARENA); - rt->freeLifoAlloc.transferFrom(&oldAlloc); - } + rt->freeLifoAlloc.transferFrom(&sweepTypeLifoAlloc); } void @@ -5103,7 +5146,17 @@ TypeZone::clearAllNewScriptsOnOOM() !iter.done(); iter.next()) { TypeObject *object = iter.get(); - object->maybeClearNewScriptOnOOM(); + if (!IsTypeObjectAboutToBeFinalized(&object)) + object->maybeClearNewScriptOnOOM(); + } +} + +AutoClearTypeInferenceStateOnOOM::~AutoClearTypeInferenceStateOnOOM() +{ + if (oom) { + zone->setPreservingCode(false); + zone->discardJitCode(zone->runtimeFromMainThread()->defaultFreeOp()); + zone->types.clearAllNewScriptsOnOOM(); } } @@ -5111,12 +5164,12 @@ TypeZone::clearAllNewScriptsOnOOM() void TypeScript::printTypes(JSContext *cx, HandleScript script) const { - MOZ_ASSERT(script->types == this); + MOZ_ASSERT(script->types() == this); if (!script->hasBaselineScript()) return; - AutoEnterAnalysis enter(nullptr, script->compartment()); + AutoEnterAnalysis enter(nullptr, script->zone()); if (script->functionNonDelazifying()) fprintf(stderr, "Function"); @@ -5168,5 +5221,6 @@ TypeScript::printTypes(JSContext *cx, HandleScript script) const void TypeObject::setNewScript(TypeNewScript *newScript) { + MOZ_ASSERT(!needsSweep()); newScript_ = newScript; } diff --git a/js/src/jsinfer.h b/js/src/jsinfer.h index 2cf97b394e9..5f37a566d3b 100644 --- a/js/src/jsinfer.h +++ b/js/src/jsinfer.h @@ -503,7 +503,12 @@ enum MOZ_ENUM_TYPE(uint32_t) { /* Mask for objects created with unknown properties. */ OBJECT_FLAG_UNKNOWN_MASK = OBJECT_FLAG_DYNAMIC_MASK - | OBJECT_FLAG_SETS_MARKED_UNKNOWN + | OBJECT_FLAG_SETS_MARKED_UNKNOWN, + + // Mask/shift for this type object's generation. If out of sync with the + // TypeZone's generation, this TypeObject hasn't been swept yet. + OBJECT_FLAG_GENERATION_MASK = 0x02000000, + OBJECT_FLAG_GENERATION_SHIFT = 25, }; typedef uint32_t TypeObjectFlags; @@ -648,6 +653,29 @@ class TypeSet void clearObjects(); }; +// If there is an OOM while sweeping types, the type information is deoptimized +// so that it stays correct (i.e. overapproximates the possible types in the +// zone), but constraints might not have been triggered on the deoptimization +// or even copied over completely. In this case, destroy all JIT code and new +// script information in the zone, the only things whose correctness depends on +// the type constraints. +class AutoClearTypeInferenceStateOnOOM +{ + Zone *zone; + bool oom; + + public: + AutoClearTypeInferenceStateOnOOM(Zone *zone) + : zone(zone), oom(false) + {} + + ~AutoClearTypeInferenceStateOnOOM(); + + void setOOM() { + oom = true; + } +}; + /* Superclass common to stack and heap type sets. */ class ConstraintTypeSet : public TypeSet { @@ -666,7 +694,7 @@ class ConstraintTypeSet : public TypeSet /* Add a new constraint to this set. */ bool addConstraint(JSContext *cx, TypeConstraint *constraint, bool callExisting = true); - inline void sweep(JS::Zone *zone, bool *oom); + inline void sweep(JS::Zone *zone, AutoClearTypeInferenceStateOnOOM &oom); }; class StackTypeSet : public ConstraintTypeSet @@ -968,7 +996,7 @@ class TypeNewScript } void trace(JSTracer *trc); - void sweep(FreeOp *fop); + void sweep(); #ifdef JSGC_COMPACTING void fixupAfterMovingGC(); @@ -1072,19 +1100,23 @@ struct TypeObject : public gc::TenuredCell public: - TypeObjectFlags flags() const { + TypeObjectFlags flags() { + maybeSweep(nullptr); return flags_; } void addFlags(TypeObjectFlags flags) { + MOZ_ASSERT(!needsSweep()); flags_ |= flags; } void clearFlags(TypeObjectFlags flags) { + MOZ_ASSERT(!needsSweep()); flags_ &= ~flags; } TypeNewScript *newScript() { + maybeSweep(nullptr); return newScript_; } @@ -1155,7 +1187,7 @@ struct TypeObject : public gc::TenuredCell return hasAnyFlags(OBJECT_FLAG_PRE_TENURE) && !unknownProperties(); } - bool hasTenuredProto() const { + bool hasTenuredProto() { return !(flags() & OBJECT_FLAG_NURSERY_PROTO); } @@ -1211,7 +1243,23 @@ struct TypeObject : public gc::TenuredCell void print(); inline void clearProperties(); - inline void sweep(FreeOp *fop, bool *oom); + void maybeSweep(AutoClearTypeInferenceStateOnOOM *oom); + + private: +#ifdef DEBUG + bool needsSweep(); +#endif + + uint32_t generation() { + return (flags_ & OBJECT_FLAG_GENERATION_MASK) >> OBJECT_FLAG_GENERATION_SHIFT; + } + + public: + void setGeneration(uint32_t generation) { + MOZ_ASSERT(generation <= (OBJECT_FLAG_GENERATION_MASK >> OBJECT_FLAG_GENERATION_SHIFT)); + flags_ &= ~OBJECT_FLAG_GENERATION_MASK; + flags_ |= generation << OBJECT_FLAG_GENERATION_SHIFT; + } #ifdef JSGC_COMPACTING void fixupAfterMovingGC(); @@ -1219,12 +1267,7 @@ struct TypeObject : public gc::TenuredCell size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; - /* - * Type objects don't have explicit finalizers. Memory owned by a type - * object pending deletion is released when weak references are sweeped - * from all the compartment's type objects. - */ - void finalize(FreeOp *fop) {} + inline void finalize(FreeOp *fop); static inline ThingRootKind rootKind() { return THING_ROOT_TYPE_OBJECT; } @@ -1241,7 +1284,7 @@ struct TypeObject : public gc::TenuredCell } private: - inline uint32_t basePropertyCount() const; + inline uint32_t basePropertyCount(); inline void setBasePropertyCount(uint32_t count); static void staticAsserts() { @@ -1323,11 +1366,11 @@ class TypeScript StackTypeSet typeArray_[1]; public: - /* Array of type type sets for variables and JOF_TYPESET ops. */ + /* Array of type sets for variables and JOF_TYPESET ops. */ StackTypeSet *typeArray() const { // Ensure typeArray_ is the last data member of TypeScript. JS_STATIC_ASSERT(sizeof(TypeScript) == - sizeof(typeArray_) + offsetof(TypeScript, typeArray_)); + sizeof(typeArray_) + offsetof(TypeScript, typeArray_)); return const_cast(typeArray_); } @@ -1387,7 +1430,6 @@ class TypeScript static void Purge(JSContext *cx, HandleScript script); - static void Sweep(FreeOp *fop, JSScript *script, bool *oom); void destroy(); size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const { @@ -1583,9 +1625,6 @@ class CompilerOutput MOZ_CRASH(); sweepIndex_ = index; } - void invalidateSweepIndex() { - sweepIndex_ = INVALID_SWEEP_INDEX; - } uint32_t sweepIndex() { MOZ_ASSERT(sweepIndex_ != INVALID_SWEEP_INDEX); return sweepIndex_; @@ -1594,26 +1633,33 @@ class CompilerOutput class RecompileInfo { - uint32_t outputIndex; + // Index in the TypeZone's compilerOutputs or sweepCompilerOutputs arrays, + // depending on the generation value. + uint32_t outputIndex : 31; + + // If out of sync with the TypeZone's generation, this index is for the + // zone's sweepCompilerOutputs rather than compilerOutputs. + uint32_t generation : 1; public: - explicit RecompileInfo(uint32_t outputIndex = uint32_t(-1)) - : outputIndex(outputIndex) + RecompileInfo(uint32_t outputIndex, uint32_t generation) + : outputIndex(outputIndex), generation(generation) + {} + + RecompileInfo() + : outputIndex(JS_BITMASK(31)), generation(0) {} - bool operator == (const RecompileInfo &o) const { - return outputIndex == o.outputIndex; - } CompilerOutput *compilerOutput(TypeZone &types) const; CompilerOutput *compilerOutput(JSContext *cx) const; bool shouldSweep(TypeZone &types); }; +typedef Vector RecompileInfoVector; + /* Type information for a compartment. */ struct TypeCompartment { - /* Constraint solving worklist structures. */ - /* Number of scripts in this compartment. */ unsigned scriptCount; @@ -1621,7 +1667,6 @@ struct TypeCompartment AllocationSiteTable *allocationSiteTable; /* Tables for determining types of singleton/JSON objects. */ - ArrayTypeTable *arrayTypeTable; ObjectTypeTable *objectTypeTable; @@ -1670,37 +1715,56 @@ struct TypeCompartment void FixRestArgumentsType(ExclusiveContext *cxArg, ArrayObject *obj); +struct AutoEnterAnalysis; + struct TypeZone { - JS::Zone *zone_; + JS::Zone *zone_; /* Pool for type information in this zone. */ static const size_t TYPE_LIFO_ALLOC_PRIMARY_CHUNK_SIZE = 8 * 1024; - js::LifoAlloc typeLifoAlloc; + LifoAlloc typeLifoAlloc; + + // Current generation for sweeping. + uint32_t generation : 1; /* * All Ion compilations that have occured in this zone, for indexing via * RecompileInfo. This includes both valid and invalid compilations, though * invalidated compilations are swept on GC. */ - Vector *compilerOutputs; + typedef Vector CompilerOutputVector; + CompilerOutputVector *compilerOutputs; - /* Pending recompilations to perform before execution of JIT code can resume. */ - Vector *pendingRecompiles; + // During incremental sweeping, allocator holding the old type information + // for the zone. + LifoAlloc sweepTypeLifoAlloc; + + // During incremental sweeping, the old compiler outputs for use by + // recompile indexes with a stale generation. + CompilerOutputVector *sweepCompilerOutputs; + + // During incremental sweeping, whether to try to destroy all type + // information attached to scripts. + bool sweepReleaseTypes; + + // The topmost AutoEnterAnalysis on the stack, if there is one. + AutoEnterAnalysis *activeAnalysis; explicit TypeZone(JS::Zone *zone); ~TypeZone(); JS::Zone *zone() const { return zone_; } - void sweep(FreeOp *fop, bool releaseTypes, bool *oom); + void beginSweep(FreeOp *fop, bool releaseTypes, AutoClearTypeInferenceStateOnOOM &oom); + void endSweep(JSRuntime *rt); void clearAllNewScriptsOnOOM(); /* Mark a script as needing recompilation once inference has finished. */ void addPendingRecompile(JSContext *cx, const RecompileInfo &info); void addPendingRecompile(JSContext *cx, JSScript *script); - void processPendingRecompiles(FreeOp *fop); + void processPendingRecompiles(FreeOp *fop, RecompileInfoVector &recompiles); }; enum SpewChannel { diff --git a/js/src/jsinferinlines.h b/js/src/jsinferinlines.h index c04fc4dc45e..875e81647c4 100644 --- a/js/src/jsinferinlines.h +++ b/js/src/jsinferinlines.h @@ -48,9 +48,20 @@ CompilerOutput::ion() const inline CompilerOutput* RecompileInfo::compilerOutput(TypeZone &types) const { + if (generation != types.generation) { + if (!types.sweepCompilerOutputs || outputIndex >= types.sweepCompilerOutputs->length()) + return nullptr; + CompilerOutput *output = &(*types.sweepCompilerOutputs)[outputIndex]; + if (!output->isValid()) + return nullptr; + output = &(*types.compilerOutputs)[output->sweepIndex()]; + return output->isValid() ? output : nullptr; + } + if (!types.compilerOutputs || outputIndex >= types.compilerOutputs->length()) return nullptr; - return &(*types.compilerOutputs)[outputIndex]; + CompilerOutput *output = &(*types.compilerOutputs)[outputIndex]; + return output->isValid() ? output : nullptr; } inline CompilerOutput* @@ -66,8 +77,14 @@ RecompileInfo::shouldSweep(TypeZone &types) if (!output || !output->isValid()) return true; - // Update this info for the output's new index in the zone's compiler outputs. - outputIndex = output->sweepIndex(); + // If this info is for a compilation that occurred after sweeping started, + // the index is already correct. + MOZ_ASSERT_IF(generation == types.generation, + outputIndex == output - types.compilerOutputs->begin()); + + // Update this info for the output's index in the zone's compiler outputs. + outputIndex = output - types.compilerOutputs->begin(); + generation = types.generation; return false; } @@ -241,44 +258,45 @@ struct AutoEnterAnalysis /* Prevent GC activity in the middle of analysis. */ gc::AutoSuppressGC suppressGC; + // Allow clearing inference info on OOM during incremental sweeping. + AutoClearTypeInferenceStateOnOOM oom; + + // Pending recompilations to perform before execution of JIT code can resume. + RecompileInfoVector pendingRecompiles; + FreeOp *freeOp; - JSCompartment *compartment; - bool oldActiveAnalysis; + Zone *zone; explicit AutoEnterAnalysis(ExclusiveContext *cx) - : suppressGC(cx) + : suppressGC(cx), oom(cx->zone()) { - init(cx->defaultFreeOp(), cx->compartment()); + init(cx->defaultFreeOp(), cx->zone()); } - AutoEnterAnalysis(FreeOp *fop, JSCompartment *comp) - : suppressGC(comp) + AutoEnterAnalysis(FreeOp *fop, Zone *zone) + : suppressGC(zone->runtimeFromMainThread()), oom(zone) { - init(fop, comp); + init(fop, zone); } ~AutoEnterAnalysis() { - compartment->activeAnalysis = oldActiveAnalysis; + if (this != zone->types.activeAnalysis) + return; - /* - * If there are no more type inference activations on the stack, - * process any triggered recompilations. Note that we should not be - * invoking any scripted code while type inference is running. - */ - if (!compartment->activeAnalysis) { - TypeZone &types = compartment->zone()->types; - if (types.pendingRecompiles) - types.processPendingRecompiles(freeOp); - } + zone->types.activeAnalysis = nullptr; + + if (!pendingRecompiles.empty()) + zone->types.processPendingRecompiles(freeOp, pendingRecompiles); } private: - void init(FreeOp *fop, JSCompartment *comp) { - freeOp = fop; - compartment = comp; - oldActiveAnalysis = compartment->activeAnalysis; - compartment->activeAnalysis = true; + void init(FreeOp *fop, Zone *zone) { + this->freeOp = fop; + this->zone = zone; + + if (!zone->types.activeAnalysis) + zone->types.activeAnalysis = this; } }; @@ -380,7 +398,7 @@ TypeMonitorCall(JSContext *cx, const js::CallArgs &args, bool constructing) { if (args.callee().is()) { JSFunction *fun = &args.callee().as(); - if (fun->isInterpreted() && fun->nonLazyScript()->types) + if (fun->isInterpreted() && fun->nonLazyScript()->types()) TypeMonitorCallSlow(cx, &args.callee(), args, constructing); } } @@ -580,7 +598,8 @@ TypeScript::NumTypeSets(JSScript *script) /* static */ inline StackTypeSet * TypeScript::ThisTypes(JSScript *script) { - return script->types->typeArray() + script->nTypeSets(); + TypeScript *types = script->types(); + return types ? types->typeArray() + script->nTypeSets() : nullptr; } /* @@ -593,7 +612,8 @@ TypeScript::ThisTypes(JSScript *script) TypeScript::ArgTypes(JSScript *script, unsigned i) { MOZ_ASSERT(i < script->functionNonDelazifying()->nargs()); - return script->types->typeArray() + script->nTypeSets() + 1 + i; + TypeScript *types = script->types(); + return types ? types->typeArray() + script->nTypeSets() + 1 + i : nullptr; } template @@ -641,9 +661,12 @@ TypeScript::BytecodeTypes(JSScript *script, jsbytecode *pc, uint32_t *bytecodeMa TypeScript::BytecodeTypes(JSScript *script, jsbytecode *pc) { MOZ_ASSERT(CurrentThreadCanAccessRuntime(script->runtimeFromMainThread())); + TypeScript *types = script->types(); + if (!types) + return nullptr; uint32_t *hint = script->baselineScript()->bytecodeTypeMap() + script->nTypeSets(); return BytecodeTypes(script, pc, script->baselineScript()->bytecodeTypeMap(), - hint, script->types->typeArray()); + hint, types->typeArray()); } struct AllocationSiteKey : public DefaultHasher { @@ -767,15 +790,16 @@ TypeScript::MonitorAssign(JSContext *cx, HandleObject obj, jsid id) /* static */ inline void TypeScript::SetThis(JSContext *cx, JSScript *script, Type type) { - if (!script->types) + StackTypeSet *types = ThisTypes(script); + if (!types) return; - if (!ThisTypes(script)->hasType(type)) { + if (!types->hasType(type)) { AutoEnterAnalysis enter(cx); InferSpew(ISpewOps, "externalType: setThis #%u: %s", script->id(), TypeString(type)); - ThisTypes(script)->addType(cx, type); + types->addType(cx, type); } } @@ -788,15 +812,16 @@ TypeScript::SetThis(JSContext *cx, JSScript *script, const js::Value &value) /* static */ inline void TypeScript::SetArgument(JSContext *cx, JSScript *script, unsigned arg, Type type) { - if (!script->types) + StackTypeSet *types = ArgTypes(script, arg); + if (!types) return; - if (!ArgTypes(script, arg)->hasType(type)) { + if (!types->hasType(type)) { AutoEnterAnalysis enter(cx); InferSpew(ISpewOps, "externalType: setArg #%u %u: %s", script->id(), arg, TypeString(type)); - ArgTypes(script, arg)->addType(cx, type); + types->addType(cx, type); } } @@ -1173,11 +1198,19 @@ inline TypeObject::TypeObject(const Class *clasp, TaggedProto proto, TypeObjectF this->proto_ = proto.raw(); this->flags_ = initialFlags; + setGeneration(zone()->types.generation); + InferSpew(ISpewOps, "newObject: %s", TypeObjectString(this)); } +inline void +TypeObject::finalize(FreeOp *fop) +{ + fop->delete_(newScript_.get()); +} + inline uint32_t -TypeObject::basePropertyCount() const +TypeObject::basePropertyCount() { return (flags() & OBJECT_FLAG_PROPERTY_COUNT_MASK) >> OBJECT_FLAG_PROPERTY_COUNT_SHIFT; } @@ -1194,8 +1227,6 @@ TypeObject::setBasePropertyCount(uint32_t count) inline HeapTypeSet * TypeObject::getProperty(ExclusiveContext *cx, jsid id) { - MOZ_ASSERT(cx->compartment()->activeAnalysis); - MOZ_ASSERT(JSID_IS_VOID(id) || JSID_IS_EMPTY(id) || JSID_IS_STRING(id) || JSID_IS_SYMBOL(id)); MOZ_ASSERT_IF(!JSID_IS_EMPTY(id), id == IdToTypeId(id)); MOZ_ASSERT(!unknownProperties()); @@ -1282,10 +1313,17 @@ TypeNewScript::writeBarrierPre(TypeNewScript *newScript) } } /* namespace js::types */ +inline js::types::TypeScript * +JSScript::types() +{ + maybeSweepTypes(nullptr); + return types_; +} + inline bool JSScript::ensureHasTypes(JSContext *cx) { - return types || makeTypes(cx); + return types() || makeTypes(cx); } namespace js { diff --git a/js/src/jsobjinlines.h b/js/src/jsobjinlines.h index 35ac360c6f2..285f904641e 100644 --- a/js/src/jsobjinlines.h +++ b/js/src/jsobjinlines.h @@ -804,7 +804,7 @@ NewObjectMetadata(ExclusiveContext *cxArg, JSObject **pmetadata) MOZ_ASSERT(!*pmetadata); if (JSContext *cx = cxArg->maybeJSContext()) { if (MOZ_UNLIKELY((size_t)cx->compartment()->hasObjectMetadataCallback()) && - !cx->compartment()->activeAnalysis) + !cx->zone()->types.activeAnalysis) { // Use AutoEnterAnalysis to prohibit both any GC activity under the // callback, and any reentering of JS via Invoke() etc. diff --git a/js/src/jsopcode.cpp b/js/src/jsopcode.cpp index 6170e1e993c..e5c2c11b887 100644 --- a/js/src/jsopcode.cpp +++ b/js/src/jsopcode.cpp @@ -1001,7 +1001,7 @@ js_Disassemble1(JSContext *cx, HandleScript script, jsbytecode *pc, case JOF_OBJECT: { /* Don't call obj.toSource if analysis/inference is active. */ - if (script->compartment()->activeAnalysis) { + if (script->zone()->types.activeAnalysis) { Sprint(sp, " object"); break; } @@ -2051,7 +2051,7 @@ js::StopPCCountProfiling(JSContext *cx) for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) { for (ZoneCellIter i(zone, FINALIZE_SCRIPT); !i.done(); i.next()) { JSScript *script = i.get(); - if (script->hasScriptCounts() && script->types) { + if (script->hasScriptCounts() && script->types()) { ScriptAndCounts sac; sac.script = script; sac.scriptCounts.set(script->releaseScriptCounts()); diff --git a/js/src/jsscript.cpp b/js/src/jsscript.cpp index 0651fc6315d..2d4c12037ea 100644 --- a/js/src/jsscript.cpp +++ b/js/src/jsscript.cpp @@ -2642,7 +2642,7 @@ JSScript::sizeOfData(mozilla::MallocSizeOf mallocSizeOf) const size_t JSScript::sizeOfTypeScript(mozilla::MallocSizeOf mallocSizeOf) const { - return types->sizeOfIncludingThis(mallocSizeOf); + return types_->sizeOfIncludingThis(mallocSizeOf); } /* @@ -2675,8 +2675,8 @@ JSScript::finalize(FreeOp *fop) fop->runtime()->spsProfiler.onScriptFinalized(this); - if (types) - types->destroy(); + if (types_) + types_->destroy(); jit::DestroyIonScripts(fop, this); diff --git a/js/src/jsscript.h b/js/src/jsscript.h index 20591ecf34b..aec12c94de6 100644 --- a/js/src/jsscript.h +++ b/js/src/jsscript.h @@ -784,10 +784,10 @@ class JSScript : public js::gc::TenuredCell JSCompartment *compartment_; - /* Persistent type information retained across GCs. */ - js::types::TypeScript *types; - private: + /* Persistent type information retained across GCs. */ + js::types::TypeScript *types_; + // This script's ScriptSourceObject, or a CCW thereof. // // (When we clone a JSScript into a new compartment, we don't clone its @@ -970,6 +970,13 @@ class JSScript : public js::gc::TenuredCell bool needsArgsAnalysis_:1; bool needsArgsObj_:1; + // Generation for this script's TypeScript. If out of sync with the + // TypeZone's generation, the TypeScript needs to be swept. + // + // This should be a uint32 but is instead a bool so that MSVC packs it + // correctly. + bool typesGeneration_:1; + // // End of fields. Start methods. // @@ -1199,7 +1206,6 @@ class JSScript : public js::gc::TenuredCell bool hasFreezeConstraints() const { return hasFreezeConstraints_; } void setHasFreezeConstraints() { hasFreezeConstraints_ = true; } - void clearHasFreezeConstraints() { hasFreezeConstraints_ = false; } bool warnedAboutUndefinedProp() const { return warnedAboutUndefinedProp_; } void setWarnedAboutUndefinedProp() { warnedAboutUndefinedProp_ = true; } @@ -1259,6 +1265,15 @@ class JSScript : public js::gc::TenuredCell return needsArgsObj() && !strict(); } + uint32_t typesGeneration() const { + return (uint32_t) typesGeneration_; + } + + void setTypesGeneration(uint32_t generation) { + MOZ_ASSERT(generation <= 1); + typesGeneration_ = (bool) generation; + } + bool hasAnyIonScript() const { return hasIonScript() || hasParallelIonScript(); } @@ -1429,6 +1444,10 @@ class JSScript : public js::gc::TenuredCell /* Ensure the script has a TypeScript. */ inline bool ensureHasTypes(JSContext *cx); + inline js::types::TypeScript *types(); + + void maybeSweepTypes(js::types::AutoClearTypeInferenceStateOnOOM *oom); + inline js::GlobalObject &global() const; js::GlobalObject &uninlinedGlobal() const; diff --git a/js/src/vm/ForkJoin.cpp b/js/src/vm/ForkJoin.cpp index 7a3719e9f2a..e057b6b3450 100644 --- a/js/src/vm/ForkJoin.cpp +++ b/js/src/vm/ForkJoin.cpp @@ -210,7 +210,7 @@ class ForkJoinOperation TrafficLight appendCallTargetsToWorklist(uint32_t index, ExecutionStatus *status); TrafficLight appendCallTargetToWorklist(HandleScript script, ExecutionStatus *status); bool addToWorklist(HandleScript script); - inline bool hasScript(Vector &scripts, JSScript *script); + inline bool hasScript(const types::RecompileInfoVector &scripts, JSScript *script); }; // class ForkJoinOperation class ForkJoinShared : public ParallelJob, public Monitor @@ -1155,7 +1155,7 @@ ForkJoinOperation::reportBailoutWarnings() bool ForkJoinOperation::invalidateBailedOutScripts() { - Vector invalid(cx_); + types::RecompileInfoVector invalid; for (uint32_t i = 0; i < bailoutRecords_.length(); i++) { switch (bailoutRecords_[i].cause) { // No bailout. @@ -1312,10 +1312,10 @@ ForkJoinOperation::recoverFromBailout(ExecutionStatus *status) } bool -ForkJoinOperation::hasScript(Vector &scripts, JSScript *script) +ForkJoinOperation::hasScript(const types::RecompileInfoVector &invalid, JSScript *script) { - for (uint32_t i = 0; i < scripts.length(); i++) { - if (scripts[i] == script->parallelIonScript()->recompileInfo()) + for (uint32_t i = 0; i < invalid.length(); i++) { + if (invalid[i].compilerOutput(cx_)->script() == script) return true; } return false; diff --git a/js/src/vm/Interpreter.cpp b/js/src/vm/Interpreter.cpp index b93281feef1..3dfb367dedc 100644 --- a/js/src/vm/Interpreter.cpp +++ b/js/src/vm/Interpreter.cpp @@ -449,7 +449,7 @@ bool js::Invoke(JSContext *cx, CallArgs args, MaybeConstruct construct) { MOZ_ASSERT(args.length() <= ARGS_LENGTH_MAX); - MOZ_ASSERT(!cx->compartment()->activeAnalysis); + MOZ_ASSERT(!cx->zone()->types.activeAnalysis); /* Perform GC if necessary on exit from the function. */ AutoGCIfNeeded gcIfNeeded(cx); @@ -1441,7 +1441,7 @@ Interpret(JSContext *cx, RunState &state) JS_END_MACRO gc::MaybeVerifyBarriers(cx, true); - MOZ_ASSERT(!cx->compartment()->activeAnalysis); + MOZ_ASSERT(!cx->zone()->types.activeAnalysis); InterpreterFrame *entryFrame = state.pushInterpreterFrame(cx); if (!entryFrame)