Bug 988789. r=luke

This commit is contained in:
Douglas Crosher 2014-05-15 13:43:04 +10:00
parent ce8a5d0538
commit 3f5f841207
24 changed files with 231 additions and 308 deletions

View File

@ -204,7 +204,7 @@ Zone::discardJitCode(FreeOp *fop)
// well to preserve them.
if (script->hasParallelIonScript()) {
if (jit::ShouldPreserveParallelJITCode(runtimeFromMainThread(), script)) {
script->parallelIonScript()->purgeCaches(this);
script->parallelIonScript()->purgeCaches();
script->baselineScript()->setActive();
} else {
jit::FinishInvalidation<ParallelExecution>(fop, script);

View File

@ -7022,6 +7022,9 @@ CheckModule(ExclusiveContext *cx, AsmJSParser &parser, ParseNode *stmtList,
if (tk != TOK_EOF && tk != TOK_RC)
return m.fail(nullptr, "top-level export (return) must be the last statement");
// The instruction cache is flushed when dynamically linking, so can inhibit now.
AutoFlushICache afc("CheckModule", /* inhibit= */ true);
ScopedJSDeletePtr<AsmJSModule> module;
if (!FinishModule(m, &module))
return false;

View File

@ -769,12 +769,24 @@ LinkAsmJS(JSContext *cx, unsigned argc, JS::Value *vp)
RootedFunction fun(cx, &args.callee().as<JSFunction>());
Rooted<AsmJSModuleObject*> moduleObj(cx, &ModuleFunctionToModuleObject(fun));
// All ICache flushing of the module being linked has been inhibited under the
// assumption that the module is flushed after dynamic linking (when the last code
// mutation occurs). Thus, enter an AutoFlushICache context for the entire module
// now. The module range is set below.
AutoFlushICache afc("LinkAsmJS");
// When a module is linked, it is dynamically specialized to the given
// arguments (buffer, ffis). Thus, if the module is linked again (it is just
// a function so it can be called multiple times), we need to clone a new
// module.
if (moduleObj->module().isDynamicallyLinked() && !CloneModule(cx, &moduleObj))
return false;
if (moduleObj->module().isDynamicallyLinked()) {
if (!CloneModule(cx, &moduleObj))
return false;
} else {
// CloneModule already calls setAutoFlushICacheRange internally before patching
// the cloned module, so avoid calling twice.
moduleObj->module().setAutoFlushICacheRange();
}
AsmJSModule &module = moduleObj->module();

View File

@ -64,9 +64,6 @@ AsmJSModule::initHeap(Handle<ArrayBufferObject*> heap, JSContext *cx)
jit::Assembler::updateBoundsCheck(heapLength,
(jit::Instruction*)(heapAccesses_[i].offset() + code_));
}
// We already know the exact extent of areas that need to be patched, just make sure we
// flush all of them at once.
jit::AutoFlushCache::updateTop(uintptr_t(code_), pod.codeBytes_);
#endif
}
@ -301,6 +298,12 @@ AsmJSModule::restoreToInitialState(ArrayBufferObject *maybePrevBuffer, Exclusive
}
}
void
AsmJSModule::setAutoFlushICacheRange()
{
AutoFlushICache::setRange(uintptr_t(code_), pod.codeBytes_);
}
void
AsmJSModule::staticallyLink(ExclusiveContext *cx)
{
@ -870,6 +873,7 @@ AsmJSModule::deserialize(ExclusiveContext *cx, const uint8_t *cursor)
(cursor = staticLinkData_.deserialize(cx, cursor));
loadedFromCache_ = true;
return cursor;
}
@ -940,6 +944,10 @@ AsmJSModule::clone(JSContext *cx, ScopedJSDeletePtr<AsmJSModule> *moduleOut) con
out.loadedFromCache_ = loadedFromCache_;
// We already know the exact extent of areas that need to be patched, just make sure we
// flush all of them at once.
out.setAutoFlushICacheRange();
out.restoreToInitialState(maybeHeap_, cx);
return true;
}
@ -1352,6 +1360,13 @@ js::LookupAsmJSModuleInCache(ExclusiveContext *cx,
if (!module)
return false;
cursor = module->deserialize(cx, cursor);
// No need to flush the instruction cache now, it will be flushed when dynamically linking.
AutoFlushICache afc("LookupAsmJSModuleInCache", /* inhibit= */ true);
// We already know the exact extent of areas that need to be patched, just make sure we
// flush all of them at once.
module->setAutoFlushICacheRange();
if (!cursor)
return false;

View File

@ -823,6 +823,7 @@ class AsmJSModule
}
void restoreToInitialState(ArrayBufferObject *maybePrevBuffer, ExclusiveContext *cx);
void setAutoFlushICacheRange();
void staticallyLink(ExclusiveContext *cx);
uint8_t *codeBase() const {

View File

@ -107,6 +107,7 @@ BaselineCompiler::compile()
return Method_Error;
Linker linker(masm);
AutoFlushICache afc("Baseline");
JitCode *code = linker.newCode<CanGC>(cx, JSC::BASELINE_CODE);
if (!code)
return Method_Error;

View File

@ -790,6 +790,7 @@ JitRuntime::generateBaselineDebugModeOSRHandler(JSContext *cx, uint32_t *noFrame
masm.jump(target);
Linker linker(masm);
AutoFlushICache afc("BaselineDebugModeOSRHandler");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
if (!code)
return nullptr;

View File

@ -591,10 +591,10 @@ ICStubCompiler::getStubCode()
masm.setSecondScratchReg(BaselineSecondScratchReg);
#endif
AutoFlushCache afc("ICStubCompiler::getStubCode", cx->runtime()->jitRuntime());
if (!generateStubCode(masm))
return nullptr;
Linker linker(masm);
AutoFlushICache afc("getStubCode");
Rooted<JitCode *> newStubCode(cx, linker.newCode<CanGC>(cx, JSC::BASELINE_CODE));
if (!newStubCode)
return nullptr;

View File

@ -113,7 +113,6 @@ EnterBaseline(JSContext *cx, EnterJitData &data)
{
AssertCompartmentUnchanged pcc(cx);
JitActivation activation(cx, data.constructing);
AutoFlushInhibitor afi(cx->runtime()->jitRuntime());
if (data.osrFrame)
data.osrFrame->setRunningInJit();
@ -235,7 +234,6 @@ jit::BaselineCompile(JSContext *cx, JSScript *script)
if (!compiler.init())
return Method_Error;
AutoFlushCache afc("BaselineJIT", cx->runtime()->jitRuntime());
MethodStatus status = compiler.compile();
JS_ASSERT_IF(status == Method_Compiled, script->hasBaselineScript());
@ -764,12 +762,6 @@ BaselineScript::toggleDebugTraps(JSScript *script, jsbytecode *pc)
SrcNoteLineScanner scanner(script->notes(), script->lineno());
JSRuntime *rt = script->runtimeFromMainThread();
IonContext ictx(CompileRuntime::get(rt),
CompileCompartment::get(script->compartment()),
nullptr);
AutoFlushCache afc("DebugTraps", rt->jitRuntime());
for (uint32_t i = 0; i < numPCMappingIndexEntries(); i++) {
PCMappingIndexEntry &entry = pcMappingIndexEntry(i);

View File

@ -5239,6 +5239,7 @@ JitCompartment::generateStringConcatStub(JSContext *cx, ExecutionMode mode)
masm.ret();
Linker linker(masm);
AutoFlushICache afc("StringConcatStub");
JitCode *code = linker.newCode<CanGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -5273,6 +5274,7 @@ JitRuntime::generateMallocStub(JSContext *cx)
masm.ret();
Linker linker(masm);
AutoFlushICache afc("MallocStub");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -5303,6 +5305,7 @@ JitRuntime::generateFreeStub(JSContext *cx)
masm.ret();
Linker linker(masm);
AutoFlushICache afc("FreeStub");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -6572,6 +6575,7 @@ CodeGenerator::link(JSContext *cx, types::CompilerConstraintList *constraints)
// use the normal executable allocator so that we cannot segv during
// execution off the main thread.
Linker linker(masm);
AutoFlushICache afc("IonLink");
JitCode *code = (executionMode == SequentialExecution)
? linker.newCodeForIonScript(cx)
: linker.newCode<CanGC>(cx, JSC::ION_CODE);

View File

@ -162,7 +162,6 @@ JitRuntime::JitRuntime()
baselineDebugModeOSRHandler_(nullptr),
functionWrappers_(nullptr),
osrTempData_(nullptr),
flusher_(nullptr),
ionCodeProtected_(false),
ionReturnOverride_(MagicValue(JS_ARG_POISON))
{
@ -187,7 +186,6 @@ JitRuntime::initialize(JSContext *cx)
AutoCompartment ac(cx, cx->atomsCompartment());
IonContext ictx(cx, nullptr);
AutoFlushCache afc("JitRuntime::initialize", this);
execAlloc_ = cx->runtime()->getExecAlloc(cx);
if (!execAlloc_)
@ -1176,7 +1174,7 @@ IonScript::toggleBarriers(bool enabled)
}
void
IonScript::purgeCaches(Zone *zone)
IonScript::purgeCaches()
{
// Don't reset any ICs if we're invalidated, otherwise, repointing the
// inline jump could overwrite an invalidation marker. These ICs can
@ -1186,9 +1184,6 @@ IonScript::purgeCaches(Zone *zone)
if (invalidated())
return;
JSRuntime *rt = zone->runtimeFromMainThread();
IonContext ictx(CompileRuntime::get(rt));
AutoFlushCache afc("purgeCaches", rt->jitRuntime());
for (size_t i = 0; i < numCaches(); i++)
getCacheFromIndex(i).reset();
}
@ -1248,8 +1243,6 @@ jit::ToggleBarriers(JS::Zone *zone, bool needs)
if (!rt->hasJitRuntime())
return;
IonContext ictx(CompileRuntime::get(rt));
AutoFlushCache afc("ToggleBarriers", rt->jitRuntime());
for (gc::ZoneCellIterUnderGC i(zone, gc::FINALIZE_SCRIPT); !i.done(); i.next()) {
JSScript *script = i.get<JSScript>();
if (script->hasIonScript())
@ -1734,7 +1727,6 @@ AttachFinishedCompilations(JSContext *cx)
// Release the worker thread lock and root the compiler for GC.
AutoTempAllocatorRooter root(cx, &builder->alloc());
AutoUnlockWorkerThreadState unlock;
AutoFlushCache afc("AttachFinishedCompilations", cx->runtime()->jitRuntime());
success = codegen->link(cx, builder->constraints());
}
@ -1880,8 +1872,6 @@ IonCompile(JSContext *cx, JSScript *script,
return AbortReason_Alloc;
}
AutoFlushCache afc("IonCompile", cx->runtime()->jitRuntime());
AutoTempAllocatorRooter root(cx, temp);
types::CompilerConstraintList *constraints = types::NewCompilerConstraintList(*temp);
if (!constraints)
@ -2417,7 +2407,6 @@ EnterIon(JSContext *cx, EnterJitData &data)
{
AssertCompartmentUnchanged pcc(cx);
JitActivation activation(cx, data.constructing);
AutoFlushInhibitor afi(cx->runtime()->jitRuntime());
CALL_GENERATED_CODE(enter, data.jitcode, data.maxArgc, data.maxArgv, /* osrFrame = */nullptr, data.calleeToken,
/* scopeChain = */ nullptr, 0, data.result.address());
@ -2601,7 +2590,7 @@ InvalidateActivation(FreeOp *fop, uint8_t *jitTop, bool invalidateAll)
// Purge ICs before we mark this script as invalidated. This will
// prevent lastJump_ from appearing to be a bogus pointer, just
// in case anyone tries to read it.
ionScript->purgeCaches(script->zone());
ionScript->purgeCaches();
// Clean up any pointers from elsewhere in the runtime to this IonScript
// which is about to become disconnected from its JSScript.
@ -2682,8 +2671,6 @@ jit::InvalidateAll(FreeOp *fop, Zone *zone)
for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) {
if (iter->compartment()->zone() == zone) {
IonContext ictx(CompileRuntime::get(fop->runtime()));
AutoFlushCache afc("InvalidateAll", fop->runtime()->jitRuntime());
IonSpew(IonSpew_Invalidate, "Invalidating all frames for GC");
InvalidateActivation(fop, iter.jitTop(), true);
}
@ -2697,7 +2684,6 @@ jit::Invalidate(types::TypeZone &types, FreeOp *fop,
bool cancelOffThread)
{
IonSpew(IonSpew_Invalidate, "Start invalidation.");
AutoFlushCache afc ("Invalidate", fop->runtime()->jitRuntime());
// Add an invalidation reference to all invalidated IonScripts to indicate
// to the traversal which frames have been invalidated.
@ -2939,66 +2925,149 @@ jit::ForbidCompilation(JSContext *cx, JSScript *script, ExecutionMode mode)
MOZ_ASSUME_UNREACHABLE("No such execution mode");
}
void
AutoFlushCache::updateTop(uintptr_t p, size_t len)
AutoFlushICache *
PerThreadData::autoFlushICache() const
{
IonContext *ictx = MaybeGetIonContext();
JitRuntime *jrt = (ictx != nullptr) ? const_cast<JitRuntime *>(ictx->runtime->jitRuntime()) : nullptr;
if (!jrt || !jrt->flusher())
JSC::ExecutableAllocator::cacheFlush((void*)p, len);
else
jrt->flusher()->update(p, len);
return autoFlushICache_;
}
AutoFlushCache::AutoFlushCache(const char *nonce, JitRuntime *rt)
void
PerThreadData::setAutoFlushICache(AutoFlushICache *afc)
{
autoFlushICache_ = afc;
}
// Set the range for the merging of flushes. The flushing is deferred until the end of
// the AutoFlushICache context. Subsequent flushing within this range will is also
// deferred. This is only expected to be defined once for each AutoFlushICache
// context. It assumes the range will be flushed is required to be within an
// AutoFlushICache context.
void
AutoFlushICache::setRange(uintptr_t start, size_t len)
{
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS)
AutoFlushICache *afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache();
JS_ASSERT(afc);
JS_ASSERT(!afc->start_);
IonSpewCont(IonSpew_CacheFlush, "(%x %x):", start, len);
uintptr_t stop = start + len;
afc->start_ = start;
afc->stop_ = stop;
#endif
}
// Flush the instruction cache.
//
// If called within a dynamic AutoFlushICache context and if the range is already pending
// flushing for this AutoFlushICache context then the request is ignored with the
// understanding that it will be flushed on exit from the AutoFlushICache context.
// Otherwise the range is flushed immediately.
//
// Updates outside the current code object are typically the exception so they are flushed
// immediately rather than attempting to merge them.
//
// For efficiency it is expected that all large ranges will be flushed within an
// AutoFlushICache, so check. If this assertion is hit then it does not necessarily
// indicate a progam fault but it might indicate a lost opportunity to merge cache
// flushing. It can be corrected by wrapping the call in an AutoFlushICache to context.
void
AutoFlushICache::flush(uintptr_t start, size_t len)
{
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS)
AutoFlushICache *afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache();
if (!afc) {
IonSpewCont(IonSpew_CacheFlush, "#");
JSC::ExecutableAllocator::cacheFlush((void*)start, len);
JS_ASSERT(len <= 16);
return;
}
uintptr_t stop = start + len;
if (start >= afc->start_ && stop <= afc->stop_) {
// Update is within the pending flush range, so defer to the end of the context.
IonSpewCont(IonSpew_CacheFlush, afc->inhibit_ ? "-" : "=");
return;
}
IonSpewCont(IonSpew_CacheFlush, afc->inhibit_ ? "x" : "*");
JSC::ExecutableAllocator::cacheFlush((void *)start, len);
#endif
}
// Flag the current dynamic AutoFlushICache as inhibiting flushing. Useful in error paths
// where the changes are being abandoned.
void
AutoFlushICache::setInhibit()
{
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS)
AutoFlushICache *afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache();
JS_ASSERT(afc);
JS_ASSERT(afc->start_);
IonSpewCont(IonSpew_CacheFlush, "I");
afc->inhibit_ = true;
#endif
}
// The common use case is merging cache flushes when preparing a code object. In this
// case the entire range of the code object is being flushed and as the code is patched
// smaller redundant flushes could occur. The design allows an AutoFlushICache dynamic
// thread local context to be declared in which the range of the code object can be set
// which defers flushing until the end of this dynamic context. The redundant flushing
// within this code range is also deferred avoiding redundant flushing. Flushing outside
// this code range is not affected and proceeds immediately.
//
// In some cases flushing is not necessary, such as when compiling an asm.js module which
// is flushed again when dynamically linked, and also in error paths that abandon the
// code. Flushing within the set code range can be inhibited within the AutoFlushICache
// dynamic context by setting an inhibit flag.
//
// The JS compiler can be re-entered while within an AutoFlushICache dynamic context and
// it is assumed that code being assembled or patched is not executed before the exit of
// the respective AutoFlushICache dynamic context.
//
AutoFlushICache::AutoFlushICache(const char *nonce, bool inhibit)
: start_(0),
stop_(0),
name_(nonce),
runtime_(rt),
used_(false)
inhibit_(inhibit)
{
if (rt->flusher())
IonSpew(IonSpew_CacheFlush, "<%s ", nonce);
else
IonSpewCont(IonSpew_CacheFlush, "<%s ", nonce);
rt->setFlusher(this);
}
AutoFlushInhibitor::AutoFlushInhibitor(JitRuntime *rt)
: runtime_(rt),
afc(nullptr)
{
afc = rt->flusher();
// Ensure that called functions get a fresh flusher.
rt->setFlusher(nullptr);
// Ensure the current flusher has been flushed.
if (afc) {
afc->flushAnyway();
IonSpewCont(IonSpew_CacheFlush, "}");
}
}
AutoFlushInhibitor::~AutoFlushInhibitor()
{
JS_ASSERT(runtime_->flusher() == nullptr);
// Ensure any future modifications are recorded.
runtime_->setFlusher(afc);
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS)
PerThreadData *pt = TlsPerThreadData.get();
AutoFlushICache *afc = pt->PerThreadData::autoFlushICache();
if (afc)
IonSpewCont(IonSpew_CacheFlush, "{");
IonSpew(IonSpew_CacheFlush, "<%s,%s%s ", nonce, afc->name_, inhibit ? " I" : "");
else
IonSpewCont(IonSpew_CacheFlush, "<%s%s ", nonce, inhibit ? " I" : "");
prev_ = afc;
pt->PerThreadData::setAutoFlushICache(this);
#endif
}
AutoFlushICache::~AutoFlushICache()
{
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS)
PerThreadData *pt = TlsPerThreadData.get();
JS_ASSERT(pt->PerThreadData::autoFlushICache() == this);
if (!inhibit_ && start_)
JSC::ExecutableAllocator::cacheFlush((void *)start_, size_t(stop_ - start_));
IonSpewCont(IonSpew_CacheFlush, "%s%s>", name_, start_ ? "" : " U");
IonSpewFin(IonSpew_CacheFlush);
pt->PerThreadData::setAutoFlushICache(prev_);
#endif
}
void
jit::PurgeCaches(JSScript *script, Zone *zone)
jit::PurgeCaches(JSScript *script)
{
if (script->hasIonScript())
script->ionScript()->purgeCaches(zone);
script->ionScript()->purgeCaches();
if (script->hasParallelIonScript())
script->parallelIonScript()->purgeCaches(zone);
script->parallelIonScript()->purgeCaches();
}
size_t
@ -3105,7 +3174,6 @@ AutoDebugModeInvalidation::~AutoDebugModeInvalidation()
JSCompartment *comp = iter->compartment();
if (comp_ == comp || zone_ == comp->zone()) {
IonContext ictx(CompileRuntime::get(rt));
AutoFlushCache afc("AutoDebugModeInvalidation", rt->jitRuntime());
IonSpew(IonSpew_Invalidate, "Invalidating frames for debug mode toggle");
InvalidateActivation(fop, iter.jitTop(), true);
}

View File

@ -190,7 +190,7 @@ NumLocalsAndArgs(JSScript *script)
void ForbidCompilation(JSContext *cx, JSScript *script);
void ForbidCompilation(JSContext *cx, JSScript *script, ExecutionMode mode);
void PurgeCaches(JSScript *script, JS::Zone *zone);
void PurgeCaches(JSScript *script);
size_t SizeOfIonData(JSScript *script, mozilla::MallocSizeOf mallocSizeOf);
void DestroyIonScripts(FreeOp *fop, JSScript *script);
void TraceIonScripts(JSTracer* trc, JSScript *script);

View File

@ -403,6 +403,7 @@ IonCache::linkAndAttachStub(JSContext *cx, MacroAssembler &masm, StubAttacher &a
IonScript *ion, const char *attachKind)
{
Rooted<JitCode *> code(cx);
AutoFlushICache afc("IonCache");
LinkStatus status = linkCode(cx, masm, ion, code.address());
if (status != LINK_GOOD)
return status != LINK_ERROR;
@ -1692,8 +1693,6 @@ GetPropertyIC::update(JSContext *cx, size_t cacheIndex,
GetPropertyIC &cache = ion->getCache(cacheIndex).toGetProperty();
RootedPropertyName name(cx, cache.name());
AutoFlushCache afc ("GetPropertyCache", cx->runtime()->jitRuntime());
// Override the return value if we are invalidated (bug 728188).
AutoDetectInvalidation adi(cx, vp.address(), ion);
@ -1868,9 +1867,6 @@ GetPropertyParIC::update(ForkJoinContext *cx, size_t cacheIndex,
// new jitcode uses a global ExecutableAllocator tied to the runtime.
LockedJSContext ncx(cx);
// The flusher needs to be under lock.
AutoFlushCache afc("GetPropertyParCache", cx->runtime()->jitRuntime());
if (cache.canAttachStub()) {
bool alreadyStubbed;
if (!cache.hasOrAddStubbedShape(ncx, obj->lastProperty(), &alreadyStubbed))
@ -2748,8 +2744,6 @@ bool
SetPropertyIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj,
HandleValue value)
{
AutoFlushCache afc ("SetPropertyCache", cx->runtime()->jitRuntime());
void *returnAddr;
RootedScript script(cx, GetTopIonJSScript(cx, &returnAddr));
IonScript *ion = script->ionScript();
@ -2865,7 +2859,6 @@ SetPropertyParIC::update(ForkJoinContext *cx, size_t cacheIndex, HandleObject ob
{
// See note about locking context in GetPropertyParIC::update.
LockedJSContext ncx(cx);
AutoFlushCache afc("SetPropertyParCache", cx->runtime()->jitRuntime());
if (cache.canAttachStub()) {
bool alreadyStubbed;
@ -3425,8 +3418,6 @@ GetElementIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj,
return true;
}
AutoFlushCache afc("GetElementCache", cx->runtime()->jitRuntime());
RootedId id(cx);
if (!ValueToId<CanGC>(cx, idval, &id))
return false;
@ -3968,7 +3959,6 @@ GetElementParIC::update(ForkJoinContext *cx, size_t cacheIndex, HandleObject obj
{
// See note about locking context in GetPropertyParIC::update.
LockedJSContext ncx(cx);
AutoFlushCache afc("GetElementParCache", cx->runtime()->jitRuntime());
if (cache.canAttachStub()) {
bool alreadyStubbed;
@ -4159,8 +4149,6 @@ IsCacheableScopeChain(JSObject *scopeChain, JSObject *holder)
JSObject *
BindNameIC::update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain)
{
AutoFlushCache afc ("BindNameCache", cx->runtime()->jitRuntime());
RootedScript outerScript(cx, GetTopIonJSScript(cx));
IonScript *ion = outerScript->ionScript();
BindNameIC &cache = ion->getCache(cacheIndex).toBindName();
@ -4290,8 +4278,6 @@ bool
NameIC::update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain,
MutableHandleValue vp)
{
AutoFlushCache afc ("GetNameCache", cx->runtime()->jitRuntime());
void *returnAddr;
RootedScript outerScript(cx, GetTopIonJSScript(cx, &returnAddr));
IonScript *ion = outerScript->ionScript();
@ -4354,8 +4340,6 @@ CallsiteCloneIC::attach(JSContext *cx, HandleScript outerScript, IonScript *ion,
JSObject *
CallsiteCloneIC::update(JSContext *cx, size_t cacheIndex, HandleObject callee)
{
AutoFlushCache afc ("CallsiteCloneCache", cx->runtime()->jitRuntime());
// Act as the identity for functions that are not clone-at-callsite, as we
// generate this cache as long as some callees are clone-at-callsite.
RootedFunction fun(cx, &callee->as<JSFunction>());

View File

@ -540,7 +540,7 @@ struct IonScript
return (CacheLocation *) &runtimeData()[locIndex];
}
void toggleBarriers(bool enabled);
void purgeCaches(JS::Zone *zone);
void purgeCaches();
void destroyCaches();
void unlinkFromRuntime(FreeOp *fop);
void copySnapshots(const SnapshotWriter *writer);
@ -774,40 +774,23 @@ struct VMFunction;
class JitCompartment;
class JitRuntime;
struct AutoFlushCache
struct AutoFlushICache
{
private:
uintptr_t start_;
uintptr_t stop_;
const char *name_;
JitRuntime *runtime_;
bool used_;
bool inhibit_;
AutoFlushICache *prev_;
public:
void update(uintptr_t p, size_t len);
static void updateTop(uintptr_t p, size_t len);
~AutoFlushCache();
AutoFlushCache(const char *nonce, JitRuntime *rt);
void flushAnyway();
static void setRange(uintptr_t p, size_t len);
static void flush(uintptr_t p, size_t len);
static void setInhibit();
~AutoFlushICache();
AutoFlushICache(const char *nonce, bool inhibit=false);
};
// If you are currently in the middle of modifing Ion-compiled code, which
// is going to be flushed at *some* point, but determine that you *must*
// call a function *right* *now*, two things can go wrong:
// 1) The flusher that you were using is still active, but you are about to
// enter jitted code, so it needs to be flushed
// 2) the called function can re-enter a compilation/modification path which
// will use your AFC, and thus not flush when his compilation is done
struct AutoFlushInhibitor
{
private:
JitRuntime *runtime_;
AutoFlushCache *afc;
public:
AutoFlushInhibitor(JitRuntime *rt);
~AutoFlushInhibitor();
};
} // namespace jit
namespace gc {

View File

@ -209,9 +209,6 @@ class JitRuntime
// (after returning from JIT code).
uint8_t *osrTempData_;
// Keep track of memoryregions that are going to be flushed.
AutoFlushCache *flusher_;
// Whether all Ion code in the runtime is protected, and will fault if it
// is accessed.
bool ionCodeProtected_;
@ -262,14 +259,6 @@ class JitRuntime
static void Mark(JSTracer *trc);
AutoFlushCache *flusher() {
return flusher_;
}
void setFlusher(AutoFlushCache *fl) {
if (!flusher_ || !fl)
flusher_ = fl;
}
JSC::ExecutableAllocator *execAlloc() const {
return execAlloc_;
}

View File

@ -568,7 +568,7 @@ Assembler::executableCopy(uint8_t *buffer)
{
JS_ASSERT(isFinished);
m_buffer.executableCopy(buffer);
AutoFlushCache::updateTop((uintptr_t)buffer, m_buffer.size());
AutoFlushICache::setRange(uintptr_t(buffer), m_buffer.size());
}
void
@ -2415,7 +2415,7 @@ Assembler::retargetNearBranch(Instruction *i, int offset, Condition cond, bool f
// Flush the cache, since an instruction was overwritten
if (final)
AutoFlushCache::updateTop(uintptr_t(i), 4);
AutoFlushICache::flush(uintptr_t(i), 4);
}
void
@ -2424,7 +2424,7 @@ Assembler::retargetFarBranch(Instruction *i, uint8_t **slot, uint8_t *dest, Cond
int32_t offset = reinterpret_cast<uint8_t*>(slot) - reinterpret_cast<uint8_t*>(i);
if (!i->is<InstLDR>()) {
new (i) InstLDR(Offset, pc, DTRAddr(pc, DtrOffImm(offset - 8)), cond);
AutoFlushCache::updateTop(uintptr_t(i), 4);
AutoFlushICache::flush(uintptr_t(i), 4);
}
*slot = dest;
@ -2526,7 +2526,7 @@ Assembler::patchWrite_NearCall(CodeLocationLabel start, CodeLocationLabel toCall
new (inst) InstBLImm(BOffImm(dest - (uint8_t*)inst) , Always);
// Ensure everyone sees the code that was just written into memory.
AutoFlushCache::updateTop(uintptr_t(inst), 4);
AutoFlushICache::flush(uintptr_t(inst), 4);
}
void
@ -2543,8 +2543,8 @@ Assembler::patchDataWithValueCheck(CodeLocationLabel label, PatchedImmPtr newVal
dest, Always, rs, ptr);
// L_LDR won't cause any instructions to be updated.
if (rs != L_LDR) {
AutoFlushCache::updateTop(uintptr_t(ptr), 4);
AutoFlushCache::updateTop(uintptr_t(ptr->next()), 4);
AutoFlushICache::flush(uintptr_t(ptr), 4);
AutoFlushICache::flush(uintptr_t(ptr->next()), 4);
}
}
@ -2676,7 +2676,7 @@ Assembler::ToggleToJmp(CodeLocationLabel inst_)
// Zero bits 20-27, then set 24-27 to be correct for a branch.
// 20-23 will be party of the B's immediate, and should be 0.
*ptr = (*ptr & ~(0xff << 20)) | (0xa0 << 20);
AutoFlushCache::updateTop((uintptr_t)ptr, 4);
AutoFlushICache::flush(uintptr_t(ptr), 4);
}
void
@ -2699,7 +2699,7 @@ Assembler::ToggleToCmp(CodeLocationLabel inst_)
// Zero out bits 20-27, then set them to be correct for a compare.
*ptr = (*ptr & ~(0xff << 20)) | (0x35 << 20);
AutoFlushCache::updateTop((uintptr_t)ptr, 4);
AutoFlushICache::flush(uintptr_t(ptr), 4);
}
void
@ -2729,7 +2729,7 @@ Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled)
else
*inst = InstNOP();
AutoFlushCache::updateTop(uintptr_t(inst), 4);
AutoFlushICache::flush(uintptr_t(inst), 4);
}
void Assembler::updateBoundsCheck(uint32_t heapSize, Instruction *inst)
@ -2751,78 +2751,6 @@ void Assembler::updateBoundsCheck(uint32_t heapSize, Instruction *inst)
// within AsmJSModule::patchHeapAccesses, which does that for us. Don't call this!
}
static uintptr_t
PageStart(uintptr_t p)
{
static const size_t PageSize = 4096;
return p & ~(PageSize - 1);
}
static bool
OnSamePage(uintptr_t start1, uintptr_t stop1, uintptr_t start2, uintptr_t stop2)
{
// Return true if (parts of) the two ranges are on the same memory page.
return PageStart(stop1) == PageStart(start2) || PageStart(stop2) == PageStart(start1);
}
void
AutoFlushCache::update(uintptr_t newStart, size_t len)
{
uintptr_t newStop = newStart + len;
used_ = true;
if (!start_) {
IonSpewCont(IonSpew_CacheFlush, ".");
start_ = newStart;
stop_ = newStop;
return;
}
if (!OnSamePage(start_, stop_, newStart, newStop)) {
// Flush now if the two ranges have no memory page in common, to avoid
// problems on Linux where the kernel only flushes the first VMA that
// covers the range. This also ensures we don't add too many pages to
// the range.
IonSpewCont(IonSpew_CacheFlush, "*");
JSC::ExecutableAllocator::cacheFlush((void*)newStart, len);
return;
}
start_ = Min(start_, newStart);
stop_ = Max(stop_, newStop);
IonSpewCont(IonSpew_CacheFlush, ".");
}
AutoFlushCache::~AutoFlushCache()
{
if (!runtime_)
return;
flushAnyway();
IonSpewCont(IonSpew_CacheFlush, ">", name_);
if (runtime_->flusher() == this) {
IonSpewFin(IonSpew_CacheFlush);
runtime_->setFlusher(nullptr);
}
}
void
AutoFlushCache::flushAnyway()
{
if (!runtime_)
return;
IonSpewCont(IonSpew_CacheFlush, "|", name_);
if (!used_)
return;
if (start_) {
JSC::ExecutableAllocator::cacheFlush((void *)start_, size_t(stop_ - start_ + sizeof(Instruction)));
} else {
JSC::ExecutableAllocator::cacheFlush(nullptr, 0xff000000);
}
used_ = false;
}
InstructionIterator::InstructionIterator(Instruction *i_) : i(i_) {
const PoolHeader *ph;
// If this is a guard, and the next instruction is a header, always work around the pool

View File

@ -1106,6 +1106,7 @@ Simulator::setLastDebuggerInput(char *input)
void
Simulator::FlushICache(void *start_addr, size_t size)
{
IonSpewCont(IonSpew_CacheFlush, "[%p %zx]", start_addr, size);
if (!Simulator::ICacheCheckingEnabled)
return;
SimulatorRuntime *srt = Simulator::Current()->srt_;

View File

@ -113,7 +113,6 @@ JitRuntime::generateEnterJIT(JSContext *cx, EnterJitType type)
JS_ASSERT(OsrFrameReg == r3);
MacroAssembler masm(cx);
AutoFlushCache afc("GenerateEnterJIT", cx->runtime()->jitRuntime());
Assembler *aasm = &masm;
// Save non-volatile registers. These must be saved by the trampoline,
@ -334,6 +333,7 @@ JitRuntime::generateEnterJIT(JSContext *cx, EnterJitType type)
GenerateReturn(masm, true, &cx->runtime()->spsProfiler);
Linker linker(masm);
AutoFlushICache afc("EnterJIT");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -395,6 +395,7 @@ JitRuntime::generateInvalidator(JSContext *cx)
masm.branch(bailoutTail);
Linker linker(masm);
AutoFlushICache afc("Invalidator");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
IonSpew(IonSpew_Invalidate, " invalidation thunk created at %p", (void *) code->raw());
@ -497,6 +498,7 @@ JitRuntime::generateArgumentsRectifier(JSContext *cx, ExecutionMode mode, void *
masm.ret();
Linker linker(masm);
AutoFlushICache afc("ArgumentsRectifier");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
CodeOffsetLabel returnLabel(returnOffset);
@ -621,6 +623,7 @@ JitRuntime::generateBailoutTable(JSContext *cx, uint32_t frameClass)
GenerateBailoutThunk(cx, masm, frameClass);
Linker linker(masm);
AutoFlushICache afc("BailoutTable");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -637,6 +640,7 @@ JitRuntime::generateBailoutHandler(JSContext *cx)
GenerateBailoutThunk(cx, masm, NO_FRAME_SIZE_CLASS_ID);
Linker linker(masm);
AutoFlushICache afc("BailoutHandler");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -809,6 +813,7 @@ JitRuntime::generateVMWrapper(JSContext *cx, const VMFunction &f)
f.extraValuesToPop * sizeof(Value)));
Linker linker(masm);
AutoFlushICache afc("VMWrapper");
JitCode *wrapper = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
if (!wrapper)
return nullptr;
@ -857,6 +862,7 @@ JitRuntime::generatePreBarrier(JSContext *cx, MIRType type)
masm.ret();
Linker linker(masm);
AutoFlushICache afc("PreBarrier");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -912,6 +918,7 @@ JitRuntime::generateDebugTrapHandler(JSContext *cx)
masm.ret();
Linker linker(masm);
AutoFlushICache afc("DebugTrapHandler");
JitCode *codeDbg = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -929,6 +936,7 @@ JitRuntime::generateExceptionTailStub(JSContext *cx)
masm.handleFailureWithHandlerTail();
Linker linker(masm);
AutoFlushICache afc("ExceptionTailStub");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -946,6 +954,7 @@ JitRuntime::generateBailoutTailStub(JSContext *cx)
masm.generateBailoutTail(r1, r2);
Linker linker(masm);
AutoFlushICache afc("BailoutTailStub");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF

View File

@ -126,7 +126,7 @@ jit::PatchJump(CodeLocationJump &jump_, CodeLocationLabel label)
Assembler::updateLuiOriValue(inst1, inst2, (uint32_t)label.raw());
AutoFlushCache::updateTop(uintptr_t(inst1), 8);
AutoFlushICache::flush(uintptr_t(inst1), 8);
}
void
@ -150,7 +150,7 @@ Assembler::executableCopy(uint8_t *buffer)
updateLuiOriValue(inst1, inst1->next(), (uint32_t)buffer + value);
}
AutoFlushCache::updateTop((uintptr_t)buffer, m_buffer.size());
AutoFlushICache::setRange(uintptr_t(buffer), m_buffer.size());
}
uint32_t
@ -1336,7 +1336,7 @@ Assembler::patchWrite_NearCall(CodeLocationLabel start, CodeLocationLabel toCall
inst[3] = InstNOP();
// Ensure everyone sees the code that was just written into memory.
AutoFlushCache::updateTop(uintptr_t(inst), patchWrite_NearCallSize());
AutoFlushICache::flush(uintptr_t(inst), patchWrite_NearCallSize());
}
uint32_t
@ -1383,7 +1383,7 @@ Assembler::patchDataWithValueCheck(CodeLocationLabel label, PatchedImmPtr newVal
// Replace with new value
Assembler::updateLuiOriValue(inst, inst->next(), uint32_t(newValue.value));
AutoFlushCache::updateTop(uintptr_t(inst), 8);
AutoFlushICache::flush(uintptr_t(inst), 8);
}
void
@ -1485,7 +1485,7 @@ Assembler::ToggleToJmp(CodeLocationLabel inst_)
// We converted beq to andi, so now we restore it.
inst->setOpcode(op_beq);
AutoFlushCache::updateTop((uintptr_t)inst, 4);
AutoFlushICache::flush(uintptr_t(inst), 4);
}
void
@ -1498,7 +1498,7 @@ Assembler::ToggleToCmp(CodeLocationLabel inst_)
// Replace "beq $zero, $zero, offset" with "andi $zero, $zero, offset"
inst->setOpcode(op_andi);
AutoFlushCache::updateTop((uintptr_t)inst, 4);
AutoFlushICache::flush(uintptr_t(inst), 4);
}
void
@ -1520,75 +1520,10 @@ Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled)
*i2 = nop;
}
AutoFlushCache::updateTop((uintptr_t)i2, 4);
AutoFlushICache::flush(uintptr_t(i2), 4);
}
void Assembler::updateBoundsCheck(uint32_t heapSize, Instruction *inst)
{
MOZ_ASSUME_UNREACHABLE("NYI");
}
void
AutoFlushCache::update(uintptr_t newStart, size_t len)
{
uintptr_t newStop = newStart + len;
if (this == nullptr) {
// just flush right here and now.
JSC::ExecutableAllocator::cacheFlush((void*)newStart, len);
return;
}
used_ = true;
if (!start_) {
IonSpewCont(IonSpew_CacheFlush, ".");
start_ = newStart;
stop_ = newStop;
return;
}
if (newStop < start_ - 4096 || newStart > stop_ + 4096) {
// If this would add too many pages to the range. Flush recorded range
// and make a new range.
IonSpewCont(IonSpew_CacheFlush, "*");
JSC::ExecutableAllocator::cacheFlush((void*)start_, stop_);
start_ = newStart;
stop_ = newStop;
return;
}
start_ = Min(start_, newStart);
stop_ = Max(stop_, newStop);
IonSpewCont(IonSpew_CacheFlush, ".");
}
AutoFlushCache::~AutoFlushCache()
{
if (!runtime_)
return;
flushAnyway();
IonSpewCont(IonSpew_CacheFlush, ">", name_);
if (runtime_->flusher() == this) {
IonSpewFin(IonSpew_CacheFlush);
runtime_->setFlusher(nullptr);
}
}
void
AutoFlushCache::flushAnyway()
{
if (!runtime_)
return;
IonSpewCont(IonSpew_CacheFlush, "|", name_);
if (!used_)
return;
if (start_) {
JSC::ExecutableAllocator::cacheFlush((void *)start_,
size_t(stop_ - start_ + sizeof(Instruction)));
} else {
JSC::ExecutableAllocator::cacheFlush(nullptr, 0xff000000);
}
used_ = false;
}

View File

@ -137,8 +137,6 @@ JitRuntime::generateEnterJIT(JSContext *cx, EnterJitType type)
MOZ_ASSERT(OsrFrameReg == reg_frame);
MacroAssembler masm(cx);
AutoFlushCache afc("GenerateEnterJIT", cx->runtime()->jitRuntime());
GeneratePrologue(masm);
const Address slotToken(sp, sizeof(EnterJITRegs) + offsetof(EnterJITArgs, calleeToken));
@ -304,6 +302,7 @@ JitRuntime::generateEnterJIT(JSContext *cx, EnterJitType type)
GenerateReturn(masm, ShortJump);
Linker linker(masm);
AutoFlushICache afc("GenerateEnterJIT");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -376,6 +375,7 @@ JitRuntime::generateInvalidator(JSContext *cx)
masm.branch(bailoutTail);
Linker linker(masm);
AutoFlushICache afc("Invalidator");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
IonSpew(IonSpew_Invalidate, " invalidation thunk created at %p", (void *) code->raw());
@ -507,6 +507,7 @@ JitRuntime::generateArgumentsRectifier(JSContext *cx, ExecutionMode mode, void *
masm.ret();
Linker linker(masm);
AutoFlushICache afc("ArgumentsRectifier");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
CodeOffsetLabel returnLabel(returnOffset);
@ -630,6 +631,7 @@ JitRuntime::generateBailoutTable(JSContext *cx, uint32_t frameClass)
GenerateBailoutThunk(cx, masm, frameClass);
Linker linker(masm);
AutoFlushICache afc("BailoutTable");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -646,6 +648,7 @@ JitRuntime::generateBailoutHandler(JSContext *cx)
GenerateBailoutThunk(cx, masm, NO_FRAME_SIZE_CLASS_ID);
Linker linker(masm);
AutoFlushICache afc("BailoutHandler");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -842,6 +845,7 @@ JitRuntime::generateVMWrapper(JSContext *cx, const VMFunction &f)
f.extraValuesToPop * sizeof(Value)));
Linker linker(masm);
AutoFlushICache afc("VMWrapper");
JitCode *wrapper = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
if (!wrapper)
return nullptr;
@ -891,6 +895,7 @@ JitRuntime::generatePreBarrier(JSContext *cx, MIRType type)
masm.ret();
Linker linker(masm);
AutoFlushICache afc("PreBarrier");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -950,6 +955,7 @@ JitRuntime::generateDebugTrapHandler(JSContext *cx)
masm.ret();
Linker linker(masm);
AutoFlushICache afc("DebugTrapHandler");
JitCode *codeDbg = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -968,6 +974,7 @@ JitRuntime::generateExceptionTailStub(JSContext *cx)
masm.handleFailureWithHandlerTail();
Linker linker(masm);
AutoFlushICache afc("ExceptionTailStub");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF
@ -985,6 +992,7 @@ JitRuntime::generateBailoutTailStub(JSContext *cx)
masm.generateBailoutTail(a1, a2);
Linker linker(masm);
AutoFlushICache afc("BailoutTailStub");
JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
#ifdef JS_ION_PERF

View File

@ -131,22 +131,3 @@ AssemblerX86Shared::InvertCondition(Condition cond)
MOZ_ASSUME_UNREACHABLE("unexpected condition");
}
}
void
AutoFlushCache::update(uintptr_t newStart, size_t len)
{
}
void
AutoFlushCache::flushAnyway()
{
}
AutoFlushCache::~AutoFlushCache()
{
if (!runtime_)
return;
if (runtime_->flusher() == this)
runtime_->setFlusher(nullptr);
}

View File

@ -5388,7 +5388,7 @@ js::PurgeJITCaches(Zone *zone)
JSScript *script = i.get<JSScript>();
/* Discard Ion caches. */
jit::PurgeCaches(script, zone);
jit::PurgeCaches(script);
}
#endif
}

View File

@ -80,6 +80,7 @@ PerThreadData::PerThreadData(JSRuntime *runtime)
#endif
activation_(nullptr),
asmJSActivationStack_(nullptr),
autoFlushICache_(nullptr),
#if defined(JS_ARM_SIMULATOR) || defined(JS_MIPS_SIMULATOR)
simulator_(nullptr),
simulatorStackLimit_(0),

View File

@ -94,6 +94,7 @@ class JitActivation;
struct PcScriptCache;
class Simulator;
class SimulatorRuntime;
class AutoFlushICache;
}
/*
@ -533,6 +534,9 @@ class PerThreadData : public PerThreadDataFriendFields
/* See AsmJSActivation comment. Protected by rt->interruptLock. */
js::AsmJSActivation *asmJSActivationStack_;
/* Pointer to the current AutoFlushICache. */
js::jit::AutoFlushICache *autoFlushICache_;
#if defined(JS_ARM_SIMULATOR) || defined(JS_MIPS_SIMULATOR)
js::jit::Simulator *simulator_;
uintptr_t simulatorStackLimit_;
@ -608,6 +612,9 @@ class PerThreadData : public PerThreadDataFriendFields
}
};
js::jit::AutoFlushICache *autoFlushICache() const;
void setAutoFlushICache(js::jit::AutoFlushICache *afc);
#if defined(JS_ARM_SIMULATOR) || defined(JS_MIPS_SIMULATOR)
js::jit::Simulator *simulator() const;
void setSimulator(js::jit::Simulator *sim);