diff --git a/js/src/assembler/assembler/X86Assembler.h b/js/src/assembler/assembler/X86Assembler.h index 8e2d614ea26..c03b224f6f2 100644 --- a/js/src/assembler/assembler/X86Assembler.h +++ b/js/src/assembler/assembler/X86Assembler.h @@ -1277,8 +1277,7 @@ public: void testq_i32r(int imm, RegisterID dst) { - spew("testq $0x%x, %s", - imm, nameIReg(dst)); + FIXME_INSN_PRINTING; m_formatter.oneByteOp64(OP_GROUP3_EvIz, GROUP3_OP_TEST, dst); m_formatter.immediate32(imm); } diff --git a/js/src/ion/CodeGenerator.cpp b/js/src/ion/CodeGenerator.cpp index 07ebae2e9a5..31717ef3334 100644 --- a/js/src/ion/CodeGenerator.cpp +++ b/js/src/ion/CodeGenerator.cpp @@ -33,91 +33,6 @@ using mozilla::Maybe; namespace js { namespace ion { -// This out-of-line cache is used to do a double dispatch including it-self and -// the wrapped IonCache. -class OutOfLineUpdateCache : - public OutOfLineCodeBase, - public IonCacheVisitor -{ - private: - LInstruction *lir_; - RepatchLabel repatchEntry_; - size_t cacheIndex_; - - public: - OutOfLineUpdateCache(LInstruction *lir, size_t cacheIndex) - : lir_(lir), - cacheIndex_(cacheIndex) - { } - - void bind(MacroAssembler *masm) { - masm->bind(&repatchEntry_); - } - - size_t getCacheIndex() const { - return cacheIndex_; - } - LInstruction *lir() const { - return lir_; - } - RepatchLabel *repatchEntry() { - return &repatchEntry_; - } - - bool accept(CodeGenerator *codegen) { - return codegen->visitOutOfLineCache(this); - } - - // ICs' visit functions delegating the work to the CodeGen visit funtions. -#define VISIT_CACHE_FUNCTION(op) \ - bool visit##op##IC(CodeGenerator *codegen, op##IC *ic) { \ - return codegen->visit##op##IC(this, ic); \ - } - - IONCACHE_KIND_LIST(VISIT_CACHE_FUNCTION) -#undef VISIT_CACHE_FUNCTION -}; - -// This function is declared here because it needs to instantiate an -// OutOfLineUpdateCache, but we want to keep it visible inside the -// CodeGeneratorShared such as we can specialize inline caches in function of -// the architecture. -bool -CodeGeneratorShared::addCache(LInstruction *lir, size_t cacheIndex) -{ - IonCache *cache = static_cast(getCache(cacheIndex)); - MInstruction *mir = lir->mirRaw()->toInstruction(); - if (mir->resumePoint()) - cache->setScriptedLocation(mir->block()->info().script(), - mir->resumePoint()->pc()); - else - cache->setIdempotent(); - - OutOfLineUpdateCache *ool = new OutOfLineUpdateCache(lir, cacheIndex); - if (!addOutOfLineCode(ool)) - return false; - - CodeOffsetJump jump = masm.jumpWithPatch(ool->repatchEntry()); - CodeOffsetLabel label = masm.labelForPatch(); - masm.bind(ool->rejoin()); - - cache->setInlineJump(jump, label); - return true; -} - -bool -CodeGenerator::visitOutOfLineCache(OutOfLineUpdateCache *ool) -{ - size_t cacheIndex = ool->getCacheIndex(); - IonCache *cache = static_cast(getCache(cacheIndex)); - - // Register the location of the OOL path in the IC. - cache->setFallbackLabel(masm.labelForPatch()); - - // Dispatch to ICs' accept functions. - return cache->accept(this, ool); -} - StringObject * MNewStringObject::templateObj() const { return &templateObj_->asString(); @@ -2198,7 +2113,7 @@ CodeGenerator::visitNewArray(LNewArray *lir) JS_ASSERT(gen->info().executionMode() == SequentialExecution); Register objReg = ToRegister(lir->output()); JSObject *templateObject = lir->mir()->templateObject(); - DebugOnly count = lir->mir()->count(); + uint32_t count = lir->mir()->count(); JS_ASSERT(count < JSObject::NELEMENTS_LIMIT); @@ -4301,9 +4216,9 @@ CodeGenerator::link() IonScript::New(cx, graph.totalSlotCount(), scriptFrameSize, snapshots_.size(), bailouts_.length(), graph.numConstants(), safepointIndices_.length(), osiIndices_.length(), - cacheList_.length(), runtimeData_.length(), - safepoints_.size(), graph.mir().numScripts(), - executionMode == ParallelExecution ? ForkJoinSlices(cx) : 0); + cacheList_.length(), safepoints_.size(), + graph.mir().numScripts(), + executionMode == ParallelExecution ? ForkJoinSlices(cx) : 0); SetIonScript(script, executionMode, ionScript); if (!ionScript) @@ -4324,28 +4239,21 @@ CodeGenerator::link() ionScript->setMethod(code); ionScript->setDeoptTable(deoptTable_); - - // for generating inline caches during the execution. - if (runtimeData_.length()) - ionScript->copyRuntimeData(&runtimeData_[0]); - if (cacheList_.length()) - ionScript->copyCacheEntries(&cacheList_[0], masm); - - // for marking during GC. + if (snapshots_.size()) + ionScript->copySnapshots(&snapshots_); + if (bailouts_.length()) + ionScript->copyBailoutTable(&bailouts_[0]); + if (graph.numConstants()) + ionScript->copyConstants(graph.constantPool()); if (safepointIndices_.length()) ionScript->copySafepointIndices(&safepointIndices_[0], masm); + if (osiIndices_.length()) + ionScript->copyOsiIndices(&osiIndices_[0], masm); + if (cacheList_.length()) + ionScript->copyCacheEntries(&cacheList_[0], masm); if (safepoints_.size()) ionScript->copySafepoints(&safepoints_); - // for reconvering from an Ion Frame. - if (bailouts_.length()) - ionScript->copyBailoutTable(&bailouts_[0]); - if (osiIndices_.length()) - ionScript->copyOsiIndices(&osiIndices_[0], masm); - if (snapshots_.size()) - ionScript->copySnapshots(&snapshots_); - if (graph.numConstants()) - ionScript->copyConstants(graph.constantPool()); JS_ASSERT(graph.mir().numScripts() > 0); ionScript->copyScriptEntries(graph.mir().scripts()); @@ -4520,186 +4428,321 @@ CodeGenerator::visitStoreFixedSlotT(LStoreFixedSlotT *ins) return true; } -bool -CodeGenerator::visitCallsiteCloneCache(LCallsiteCloneCache *ins) +// An out-of-line path to call an inline cache function. +class OutOfLineCache : public OutOfLineCodeBase { - const MCallsiteCloneCache *mir = ins->mir(); - Register callee = ToRegister(ins->callee()); - Register output = ToRegister(ins->output()); + LInstruction *ins; + RepatchLabel repatchEntry_; + CodeOffsetJump inlineJump; + CodeOffsetLabel inlineLabel; - CallsiteCloneIC cache(callee, mir->block()->info().script(), mir->callPc(), output); - return addCache(ins, allocateCache(cache)); + public: + OutOfLineCache(LInstruction *ins) + : ins(ins) + {} + + void setInlineJump(CodeOffsetJump jump, CodeOffsetLabel label) { + inlineJump = jump; + inlineLabel = label; + } + + CodeOffsetJump getInlineJump() const { + return inlineJump; + } + + CodeOffsetLabel getInlineLabel() const { + return inlineLabel; + } + + bool accept(CodeGenerator *codegen) { + switch (ins->op()) { + case LInstruction::LOp_GetPropertyCacheT: + case LInstruction::LOp_GetPropertyCacheV: + return codegen->visitOutOfLineCacheGetProperty(this); + case LInstruction::LOp_GetElementCacheV: + return codegen->visitOutOfLineGetElementCache(this); + case LInstruction::LOp_SetPropertyCacheT: + case LInstruction::LOp_SetPropertyCacheV: + return codegen->visitOutOfLineSetPropertyCache(this); + case LInstruction::LOp_BindNameCache: + return codegen->visitOutOfLineBindNameCache(this); + case LInstruction::LOp_GetNameCache: + return codegen->visitOutOfLineGetNameCache(this); + case LInstruction::LOp_CallsiteCloneCache: + return codegen->visitOutOfLineCallsiteCloneCache(this); + default: + JS_NOT_REACHED("Bad instruction"); + return false; + } + } + + LInstruction *cache() { + return ins; + } + void bind(MacroAssembler *masm) { + masm->bind(&repatchEntry_); + } + RepatchLabel *repatchEntry() { + return &repatchEntry_; + } +}; + +bool +CodeGenerator::visitCache(LInstruction *ins) +{ + OutOfLineCache *ool = new OutOfLineCache(ins); + if (!addOutOfLineCode(ool)) + return false; + +#if 0 + // NOTE: This is currently disabled. OSI and IC interaction are protected + // through other means in ICs, since the nops incur significant overhead. + // + // ensureOsiSpace(); +#endif + + CodeOffsetJump jump = masm.jumpWithPatch(ool->repatchEntry()); + CodeOffsetLabel label = masm.labelForPatch(); + masm.bind(ool->rejoin()); + + ool->setInlineJump(jump, label); + return true; } -typedef JSObject *(*CallsiteCloneICFn)(JSContext *, size_t, HandleObject); -const VMFunction CallsiteCloneIC::UpdateInfo = - FunctionInfo(CallsiteCloneIC::update); +typedef JSObject *(*CallsiteCloneCacheFn)(JSContext *, size_t, HandleObject); +static const VMFunction CallsiteCloneCacheInfo = + FunctionInfo(CallsiteCloneCache); bool -CodeGenerator::visitCallsiteCloneIC(OutOfLineUpdateCache *ool, CallsiteCloneIC *ic) +CodeGenerator::visitOutOfLineCallsiteCloneCache(OutOfLineCache *ool) { - AssertCanGC(); - LInstruction *lir = ool->lir(); + LCallsiteCloneCache *lir = ool->cache()->toCallsiteCloneCache(); + const MCallsiteCloneCache *mir = lir->mir(); + Register callee = ToRegister(lir->callee()); + RegisterSet liveRegs = lir->safepoint()->liveRegs(); + Register output = ToRegister(lir->output()); + + IonCacheCallsiteClone cache(ool->getInlineJump(), ool->getInlineLabel(), + masm.labelForPatch(), liveRegs, + callee, mir->block()->info().script(), mir->callPc(), output); + + JS_ASSERT(!mir->resumePoint()); + + size_t cacheIndex = allocateCache(cache); + saveLive(lir); - pushArg(ic->calleeReg()); - pushArg(Imm32(ool->getCacheIndex())); - if (!callVM(CallsiteCloneIC::UpdateInfo, lir)) + pushArg(callee); + pushArg(Imm32(cacheIndex)); + if (!callVM(CallsiteCloneCacheInfo, lir)) return false; - StoreRegisterTo(ic->outputReg()).generate(this); - restoreLiveIgnore(lir, StoreRegisterTo(ic->outputReg()).clobbered()); + + masm.storeCallResult(output); + restoreLive(lir); masm.jump(ool->rejoin()); return true; } -bool -CodeGenerator::visitGetNameCache(LGetNameCache *ins) -{ - Register scopeChain = ToRegister(ins->scopeObj()); - TypedOrValueRegister output(GetValueOutput(ins)); - bool isTypeOf = ins->mir()->accessKind() != MGetNameCache::NAME; - - NameIC cache(isTypeOf, scopeChain, ins->mir()->name(), output); - return addCache(ins, allocateCache(cache)); -} - -typedef bool (*NameICFn)(JSContext *, size_t, HandleObject, MutableHandleValue); -const VMFunction NameIC::UpdateInfo = FunctionInfo(NameIC::update); +typedef bool (*GetNameCacheFn)(JSContext *, size_t, HandleObject, MutableHandleValue); +static const VMFunction GetNameCacheInfo = + FunctionInfo(GetNameCache); bool -CodeGenerator::visitNameIC(OutOfLineUpdateCache *ool, NameIC *ic) +CodeGenerator::visitOutOfLineGetNameCache(OutOfLineCache *ool) { - AssertCanGC(); - LInstruction *lir = ool->lir(); + LGetNameCache *lir = ool->cache()->toGetNameCache(); + const MGetNameCache *mir = lir->mir(); + Register scopeChain = ToRegister(lir->scopeObj()); + RegisterSet liveRegs = lir->safepoint()->liveRegs(); + TypedOrValueRegister output(GetValueOutput(lir)); + + IonCache::Kind kind = (mir->accessKind() == MGetNameCache::NAME) + ? IonCache::Name + : IonCache::NameTypeOf; + IonCacheName cache(kind, ool->getInlineJump(), ool->getInlineLabel(), + masm.labelForPatch(), liveRegs, + scopeChain, mir->name(), output); + + cache.setScriptedLocation(mir->block()->info().script(), mir->resumePoint()->pc()); + size_t cacheIndex = allocateCache(cache); + saveLive(lir); - pushArg(ic->scopeChainReg()); - pushArg(Imm32(ool->getCacheIndex())); - if (!callVM(NameIC::UpdateInfo, lir)) + pushArg(scopeChain); + pushArg(Imm32(cacheIndex)); + if (!callVM(GetNameCacheInfo, lir)) return false; - StoreValueTo(ic->outputReg()).generate(this); - restoreLiveIgnore(lir, StoreValueTo(ic->outputReg()).clobbered()); + + masm.storeCallResultValue(output); + restoreLive(lir); masm.jump(ool->rejoin()); return true; } -bool -CodeGenerator::visitGetPropertyCacheV(LGetPropertyCacheV *ins) -{ - RegisterSet liveRegs = ins->safepoint()->liveRegs(); - Register objReg = ToRegister(ins->getOperand(0)); - PropertyName *name = ins->mir()->name(); - bool allowGetters = ins->mir()->allowGetters(); - TypedOrValueRegister output = TypedOrValueRegister(GetValueOutput(ins)); - - GetPropertyIC cache(liveRegs, objReg, name, output, allowGetters); - return addCache(ins, allocateCache(cache)); -} +typedef bool (*GetPropertyCacheFn)(JSContext *, size_t, HandleObject, MutableHandleValue); +static const VMFunction GetPropertyCacheInfo = + FunctionInfo(GetPropertyCache); bool -CodeGenerator::visitGetPropertyCacheT(LGetPropertyCacheT *ins) +CodeGenerator::visitOutOfLineCacheGetProperty(OutOfLineCache *ool) { - RegisterSet liveRegs = ins->safepoint()->liveRegs(); - Register objReg = ToRegister(ins->getOperand(0)); - PropertyName *name = ins->mir()->name(); - bool allowGetters = ins->mir()->allowGetters(); - TypedOrValueRegister output(ins->mir()->type(), ToAnyRegister(ins->getDef(0))); + RegisterSet liveRegs = ool->cache()->safepoint()->liveRegs(); - GetPropertyIC cache(liveRegs, objReg, name, output, allowGetters); - return addCache(ins, allocateCache(cache)); -} + LInstruction *ins = ool->cache(); + MInstruction *mir = ins->mirRaw()->toInstruction(); -typedef bool (*GetPropertyICFn)(JSContext *, size_t, HandleObject, MutableHandleValue); -const VMFunction GetPropertyIC::UpdateInfo = - FunctionInfo(GetPropertyIC::update); + TypedOrValueRegister output; -bool -CodeGenerator::visitGetPropertyIC(OutOfLineUpdateCache *ool, GetPropertyIC *ic) -{ - AssertCanGC(); - LInstruction *lir = ool->lir(); - saveLive(lir); + Register objReg; - pushArg(ic->object()); - pushArg(Imm32(ool->getCacheIndex())); - if (!callVM(GetPropertyIC::UpdateInfo, lir)) + // Define the input and output registers each opcode wants, + // and which atom property it should get + // Note: because all registers are saved, the output register should be + // a def register, else the result will be overriden by restoreLive(ins) + PropertyName *name = NULL; + bool allowGetters = false; + switch (ins->op()) { + case LInstruction::LOp_GetPropertyCacheT: + name = ((LGetPropertyCacheT *) ins)->mir()->name(); + objReg = ToRegister(ins->getOperand(0)); + output = TypedOrValueRegister(mir->type(), ToAnyRegister(ins->getDef(0))); + JS_ASSERT(mir->isGetPropertyCache()); + allowGetters = mir->toGetPropertyCache()->allowGetters(); + break; + case LInstruction::LOp_GetPropertyCacheV: + name = ((LGetPropertyCacheV *) ins)->mir()->name(); + objReg = ToRegister(ins->getOperand(0)); + output = TypedOrValueRegister(GetValueOutput(ins)); + JS_ASSERT(mir->isGetPropertyCache()); + allowGetters = mir->toGetPropertyCache()->allowGetters(); + break; + default: + JS_NOT_REACHED("Bad instruction"); return false; - StoreValueTo(ic->output()).generate(this); - restoreLiveIgnore(lir, StoreValueTo(ic->output()).clobbered()); + } + + IonCacheGetProperty cache(ool->getInlineJump(), ool->getInlineLabel(), + masm.labelForPatch(), liveRegs, + objReg, name, output, allowGetters); + + if (mir->resumePoint()) + cache.setScriptedLocation(mir->block()->info().script(), mir->resumePoint()->pc()); + else + cache.setIdempotent(); + size_t cacheIndex = allocateCache(cache); + + saveLive(ins); + + pushArg(objReg); + pushArg(Imm32(cacheIndex)); + if (!callVM(GetPropertyCacheInfo, ins)) + return false; + + masm.storeCallResultValue(output); + restoreLive(ins); masm.jump(ool->rejoin()); + return true; } +typedef bool (*GetElementCacheFn)(JSContext *, size_t, HandleObject, HandleValue, MutableHandleValue); +static const VMFunction GetElementCacheInfo = FunctionInfo(GetElementCache); + bool -CodeGenerator::visitGetElementCacheV(LGetElementCacheV *ins) +CodeGenerator::visitOutOfLineGetElementCache(OutOfLineCache *ool) { + LGetElementCacheV *ins = ool->cache()->toGetElementCacheV(); + const MGetElementCache *mir = ins->mir(); + Register obj = ToRegister(ins->object()); ConstantOrRegister index = TypedOrValueRegister(ToValue(ins, LGetElementCacheV::Index)); TypedOrValueRegister output = TypedOrValueRegister(GetValueOutput(ins)); - GetElementIC cache(obj, index, output, ins->mir()->monitoredResult()); + RegisterSet liveRegs = ins->safepoint()->liveRegs(); - return addCache(ins, allocateCache(cache)); -} + IonCacheGetElement cache(ool->getInlineJump(), ool->getInlineLabel(), + masm.labelForPatch(), liveRegs, + obj, index, output, + mir->monitoredResult()); -typedef bool (*GetElementICFn)(JSContext *, size_t, HandleObject, HandleValue, MutableHandleValue); -const VMFunction GetElementIC::UpdateInfo = - FunctionInfo(GetElementIC::update); + cache.setScriptedLocation(mir->block()->info().script(), mir->resumePoint()->pc()); + size_t cacheIndex = allocateCache(cache); -bool -CodeGenerator::visitGetElementIC(OutOfLineUpdateCache *ool, GetElementIC *ic) -{ - AssertCanGC(); - LInstruction *lir = ool->lir(); - saveLive(lir); + saveLive(ins); - pushArg(ic->index()); - pushArg(ic->object()); - pushArg(Imm32(ool->getCacheIndex())); - if (!callVM(GetElementIC::UpdateInfo, lir)) + pushArg(index); + pushArg(obj); + pushArg(Imm32(cacheIndex)); + if (!callVM(GetElementCacheInfo, ins)) return false; - StoreValueTo(ic->output()).generate(this); - restoreLiveIgnore(lir, StoreValueTo(ic->output()).clobbered()); + + masm.storeCallResultValue(output); + restoreLive(ins); masm.jump(ool->rejoin()); return true; } +typedef JSObject *(*BindNameCacheFn)(JSContext *, size_t, HandleObject); +static const VMFunction BindNameCacheInfo = + FunctionInfo(BindNameCache); + bool -CodeGenerator::visitBindNameCache(LBindNameCache *ins) +CodeGenerator::visitOutOfLineBindNameCache(OutOfLineCache *ool) { + LBindNameCache *ins = ool->cache()->toBindNameCache(); Register scopeChain = ToRegister(ins->scopeChain()); Register output = ToRegister(ins->output()); - BindNameIC cache(scopeChain, ins->mir()->name(), output); - return addCache(ins, allocateCache(cache)); -} + RegisterSet liveRegs = ins->safepoint()->liveRegs(); -typedef JSObject *(*BindNameICFn)(JSContext *, size_t, HandleObject); -const VMFunction BindNameIC::UpdateInfo = - FunctionInfo(BindNameIC::update); + const MBindNameCache *mir = ins->mir(); + IonCacheBindName cache(ool->getInlineJump(), ool->getInlineLabel(), + masm.labelForPatch(), liveRegs, + scopeChain, mir->name(), output); + cache.setScriptedLocation(mir->script(), mir->pc()); + size_t cacheIndex = allocateCache(cache); -bool -CodeGenerator::visitBindNameIC(OutOfLineUpdateCache *ool, BindNameIC *ic) -{ - AssertCanGC(); - LInstruction *lir = ool->lir(); - saveLive(lir); + saveLive(ins); - pushArg(ic->scopeChainReg()); - pushArg(Imm32(ool->getCacheIndex())); - if (!callVM(BindNameIC::UpdateInfo, lir)) + pushArg(scopeChain); + pushArg(Imm32(cacheIndex)); + if (!callVM(BindNameCacheInfo, ins)) return false; - StoreRegisterTo(ic->outputReg()).generate(this); - restoreLiveIgnore(lir, StoreRegisterTo(ic->outputReg()).clobbered()); + + masm.storeCallResult(output); + restoreLive(ins); masm.jump(ool->rejoin()); return true; } +ConstantOrRegister +CodeGenerator::getSetPropertyValue(LInstruction *ins) +{ + if (ins->getOperand(1)->isConstant()) { + JS_ASSERT(ins->isSetPropertyCacheT()); + return ConstantOrRegister(*ins->getOperand(1)->toConstant()); + } + + switch (ins->op()) { + case LInstruction::LOp_CallSetProperty: + return TypedOrValueRegister(ToValue(ins, LCallSetProperty::Value)); + case LInstruction::LOp_SetPropertyCacheV: + return TypedOrValueRegister(ToValue(ins, LSetPropertyCacheV::Value)); + case LInstruction::LOp_SetPropertyCacheT: { + LSetPropertyCacheT *ins_ = ins->toSetPropertyCacheT(); + return TypedOrValueRegister(ins_->valueType(), ToAnyRegister(ins->getOperand(1))); + } + default: + JS_NOT_REACHED("Bad opcode"); + return ConstantOrRegister(UndefinedValue()); + } +} + typedef bool (*SetPropertyFn)(JSContext *, HandleObject, HandlePropertyName, const HandleValue, bool, bool); static const VMFunction SetPropertyInfo = @@ -4708,7 +4751,7 @@ static const VMFunction SetPropertyInfo = bool CodeGenerator::visitCallSetProperty(LCallSetProperty *ins) { - ConstantOrRegister value = TypedOrValueRegister(ToValue(ins, LCallSetProperty::Value)); + ConstantOrRegister value = getSetPropertyValue(ins); const Register objReg = ToRegister(ins->getOperand(0)); bool isSetName = JSOp(*ins->mir()->resumePoint()->pc()) == JSOP_SETNAME; @@ -4741,56 +4784,43 @@ CodeGenerator::visitCallDeleteProperty(LCallDeleteProperty *lir) return callVM(DeletePropertyNonStrictInfo, lir); } +typedef bool (*SetPropertyCacheFn)(JSContext *, size_t, HandleObject, HandleValue, bool); +static const VMFunction SetPropertyCacheInfo = + FunctionInfo(ion::SetPropertyCache); + bool -CodeGenerator::visitSetPropertyCacheV(LSetPropertyCacheV *ins) +CodeGenerator::visitOutOfLineSetPropertyCache(OutOfLineCache *ool) { - RegisterSet liveRegs = ins->safepoint()->liveRegs(); + LInstruction *ins = ool->cache(); + Register objReg = ToRegister(ins->getOperand(0)); - ConstantOrRegister value = TypedOrValueRegister(ToValue(ins, LSetPropertyCacheV::Value)); - bool isSetName = JSOp(*ins->mir()->resumePoint()->pc()) == JSOP_SETNAME; - - SetPropertyIC cache(liveRegs, objReg, ins->mir()->name(), value, - isSetName, ins->mir()->strict()); - return addCache(ins, allocateCache(cache)); -} - -bool -CodeGenerator::visitSetPropertyCacheT(LSetPropertyCacheT *ins) -{ RegisterSet liveRegs = ins->safepoint()->liveRegs(); - Register objReg = ToRegister(ins->getOperand(0)); - ConstantOrRegister value; - bool isSetName = JSOp(*ins->mir()->resumePoint()->pc()) == JSOP_SETNAME; - if (ins->getOperand(1)->isConstant()) - value = ConstantOrRegister(*ins->getOperand(1)->toConstant()); - else - value = TypedOrValueRegister(ins->valueType(), ToAnyRegister(ins->getOperand(1))); + ConstantOrRegister value = getSetPropertyValue(ins); + const MSetPropertyCache *mir = ins->mirRaw()->toSetPropertyCache(); - SetPropertyIC cache(liveRegs, objReg, ins->mir()->name(), value, - isSetName, ins->mir()->strict()); - return addCache(ins, allocateCache(cache)); -} + IonCacheSetProperty cache(ool->getInlineJump(), ool->getInlineLabel(), + masm.labelForPatch(), liveRegs, + objReg, mir->name(), value, + mir->strict()); -typedef bool (*SetPropertyICFn)(JSContext *, size_t, HandleObject, HandleValue); -const VMFunction SetPropertyIC::UpdateInfo = - FunctionInfo(SetPropertyIC::update); + size_t cacheIndex = allocateCache(cache); + bool isSetName = JSOp(*mir->resumePoint()->pc()) == JSOP_SETNAME; -bool -CodeGenerator::visitSetPropertyIC(OutOfLineUpdateCache *ool, SetPropertyIC *ic) -{ - AssertCanGC(); - LInstruction *lir = ool->lir(); - saveLive(lir); + saveLive(ins); - pushArg(ic->value()); - pushArg(ic->object()); - pushArg(Imm32(ool->getCacheIndex())); - if (!callVM(SetPropertyIC::UpdateInfo, lir)) + pushArg(Imm32(isSetName)); + pushArg(value); + pushArg(objReg); + pushArg(Imm32(cacheIndex)); + + if (!callVM(SetPropertyCacheInfo, ool->cache())) return false; - restoreLive(lir); + + restoreLive(ins); masm.jump(ool->rejoin()); + return true; } diff --git a/js/src/ion/CodeGenerator.h b/js/src/ion/CodeGenerator.h index ed29fddf1f4..ac9c2a1933f 100644 --- a/js/src/ion/CodeGenerator.h +++ b/js/src/ion/CodeGenerator.h @@ -29,11 +29,11 @@ class CheckOverRecursedFailure; class ParCheckOverRecursedFailure; class OutOfLineParCheckInterrupt; class OutOfLineUnboxDouble; +class OutOfLineCache; class OutOfLineStoreElementHole; class OutOfLineTypeOfV; class OutOfLineLoadTypedArray; class OutOfLineParNewGCThing; -class OutOfLineUpdateCache; class CodeGenerator : public CodeGeneratorSpecific { @@ -220,31 +220,49 @@ class CodeGenerator : public CodeGeneratorSpecific bool visitOutOfLineUnboxDouble(OutOfLineUnboxDouble *ool); bool visitOutOfLineStoreElementHole(OutOfLineStoreElementHole *ool); + bool visitOutOfLineCacheGetProperty(OutOfLineCache *ool); + bool visitOutOfLineGetElementCache(OutOfLineCache *ool); + bool visitOutOfLineSetPropertyCache(OutOfLineCache *ool); + bool visitOutOfLineBindNameCache(OutOfLineCache *ool); + bool visitOutOfLineGetNameCache(OutOfLineCache *ool); + bool visitOutOfLineCallsiteCloneCache(OutOfLineCache *ool); + bool visitOutOfLineParNewGCThing(OutOfLineParNewGCThing *ool); + bool visitOutOfLineParallelAbort(OutOfLineParallelAbort *ool); - // Inline caches visitors. - bool visitOutOfLineCache(OutOfLineUpdateCache *ool); - - bool visitGetPropertyCacheV(LGetPropertyCacheV *ins); - bool visitGetPropertyCacheT(LGetPropertyCacheT *ins); - bool visitGetElementCacheV(LGetElementCacheV *ins); - bool visitBindNameCache(LBindNameCache *ins); - bool visitCallSetProperty(LInstruction *ins); - bool visitSetPropertyCacheV(LSetPropertyCacheV *ins); - bool visitSetPropertyCacheT(LSetPropertyCacheT *ins); - bool visitGetNameCache(LGetNameCache *ins); - bool visitCallsiteCloneCache(LCallsiteCloneCache *ins); - - bool visitGetPropertyIC(OutOfLineUpdateCache *ool, GetPropertyIC *ic); - bool visitSetPropertyIC(OutOfLineUpdateCache *ool, SetPropertyIC *ic); - bool visitGetElementIC(OutOfLineUpdateCache *ool, GetElementIC *ic); - bool visitBindNameIC(OutOfLineUpdateCache *ool, BindNameIC *ic); - bool visitNameIC(OutOfLineUpdateCache *ool, NameIC *ic); - bool visitCallsiteCloneIC(OutOfLineUpdateCache *ool, CallsiteCloneIC *ic); + bool visitGetPropertyCacheV(LGetPropertyCacheV *ins) { + return visitCache(ins); + } + bool visitGetPropertyCacheT(LGetPropertyCacheT *ins) { + return visitCache(ins); + } + bool visitGetElementCacheV(LGetElementCacheV *ins) { + return visitCache(ins); + } + bool visitBindNameCache(LBindNameCache *ins) { + return visitCache(ins); + } + bool visitSetPropertyCacheV(LSetPropertyCacheV *ins) { + return visitCache(ins); + } + bool visitSetPropertyCacheT(LSetPropertyCacheT *ins) { + return visitCache(ins); + } + bool visitGetNameCache(LGetNameCache *ins) { + return visitCache(ins); + } + bool visitCallsiteCloneCache(LCallsiteCloneCache *ins) { + return visitCache(ins); + } private: + bool visitCache(LInstruction *load); + bool visitCallSetProperty(LInstruction *ins); + bool checkForParallelBailout(); + + ConstantOrRegister getSetPropertyValue(LInstruction *ins); bool generateBranchV(const ValueOperand &value, Label *ifTrue, Label *ifFalse, FloatRegister fr); bool emitParAllocateGCThing(const Register &objReg, diff --git a/js/src/ion/Ion.cpp b/js/src/ion/Ion.cpp index f6db0c78624..956dbabde09 100644 --- a/js/src/ion/Ion.cpp +++ b/js/src/ion/Ion.cpp @@ -451,24 +451,22 @@ IonScript::IonScript() invalidateEpilogueOffset_(0), invalidateEpilogueDataOffset_(0), bailoutExpected_(false), - runtimeData_(0), - runtimeSize_(0), - cacheIndex_(0), - cacheEntries_(0), - safepointIndexOffset_(0), - safepointIndexEntries_(0), - safepointsStart_(0), - safepointsSize_(0), - frameSlots_(0), - frameSize_(0), - bailoutTable_(0), - bailoutEntries_(0), - osiIndexOffset_(0), - osiIndexEntries_(0), snapshots_(0), snapshotsSize_(0), + bailoutTable_(0), + bailoutEntries_(0), constantTable_(0), constantEntries_(0), + safepointIndexOffset_(0), + safepointIndexEntries_(0), + frameSlots_(0), + frameSize_(0), + osiIndexOffset_(0), + osiIndexEntries_(0), + cacheList_(0), + cacheEntries_(0), + safepointsStart_(0), + safepointsSize_(0), scriptList_(0), scriptEntries_(0), parallelInvalidatedScriptList_(0), @@ -483,9 +481,8 @@ static const int DataAlignment = 4; IonScript * IonScript::New(JSContext *cx, uint32_t frameSlots, uint32_t frameSize, size_t snapshotsSize, size_t bailoutEntries, size_t constants, size_t safepointIndices, - size_t osiIndices, size_t cacheEntries, size_t runtimeSize, - size_t safepointsSize, size_t scriptEntries, - size_t parallelInvalidatedScriptEntries) + size_t osiIndices, size_t cacheEntries, size_t safepointsSize, + size_t scriptEntries, size_t parallelInvalidatedScriptEntries) { if (snapshotsSize >= MAX_BUFFER_SIZE || (bailoutEntries >= MAX_BUFFER_SIZE / sizeof(uint32_t))) @@ -502,8 +499,7 @@ IonScript::New(JSContext *cx, uint32_t frameSlots, uint32_t frameSize, size_t sn size_t paddedConstantsSize = AlignBytes(constants * sizeof(Value), DataAlignment); size_t paddedSafepointIndicesSize = AlignBytes(safepointIndices * sizeof(SafepointIndex), DataAlignment); size_t paddedOsiIndicesSize = AlignBytes(osiIndices * sizeof(OsiIndex), DataAlignment); - size_t paddedCacheEntriesSize = AlignBytes(cacheEntries * sizeof(uint32_t), DataAlignment); - size_t paddedRuntimeSize = AlignBytes(runtimeSize, DataAlignment); + size_t paddedCacheEntriesSize = AlignBytes(cacheEntries * sizeof(IonCache), DataAlignment); size_t paddedSafepointSize = AlignBytes(safepointsSize, DataAlignment); size_t paddedScriptSize = AlignBytes(scriptEntries * sizeof(RawScript), DataAlignment); size_t paddedParallelInvalidatedScriptSize = @@ -514,7 +510,6 @@ IonScript::New(JSContext *cx, uint32_t frameSlots, uint32_t frameSize, size_t sn paddedSafepointIndicesSize+ paddedOsiIndicesSize + paddedCacheEntriesSize + - paddedRuntimeSize + paddedSafepointSize + paddedScriptSize + paddedParallelInvalidatedScriptSize; @@ -527,37 +522,33 @@ IonScript::New(JSContext *cx, uint32_t frameSlots, uint32_t frameSize, size_t sn uint32_t offsetCursor = sizeof(IonScript); - script->runtimeData_ = offsetCursor; - script->runtimeSize_ = runtimeSize; - offsetCursor += paddedRuntimeSize; - - script->cacheIndex_ = offsetCursor; - script->cacheEntries_ = cacheEntries; - offsetCursor += paddedCacheEntriesSize; - - script->safepointIndexOffset_ = offsetCursor; - script->safepointIndexEntries_ = safepointIndices; - offsetCursor += paddedSafepointIndicesSize; - - script->safepointsStart_ = offsetCursor; - script->safepointsSize_ = safepointsSize; - offsetCursor += paddedSafepointSize; + script->snapshots_ = offsetCursor; + script->snapshotsSize_ = snapshotsSize; + offsetCursor += paddedSnapshotsSize; script->bailoutTable_ = offsetCursor; script->bailoutEntries_ = bailoutEntries; offsetCursor += paddedBailoutSize; + script->constantTable_ = offsetCursor; + script->constantEntries_ = constants; + offsetCursor += paddedConstantsSize; + + script->safepointIndexOffset_ = offsetCursor; + script->safepointIndexEntries_ = safepointIndices; + offsetCursor += paddedSafepointIndicesSize; + script->osiIndexOffset_ = offsetCursor; script->osiIndexEntries_ = osiIndices; offsetCursor += paddedOsiIndicesSize; - script->snapshots_ = offsetCursor; - script->snapshotsSize_ = snapshotsSize; - offsetCursor += paddedSnapshotsSize; + script->cacheList_ = offsetCursor; + script->cacheEntries_ = cacheEntries; + offsetCursor += paddedCacheEntriesSize; - script->constantTable_ = offsetCursor; - script->constantEntries_ = constants; - offsetCursor += paddedConstantsSize; + script->safepointsStart_ = offsetCursor; + script->safepointsSize_ = safepointsSize; + offsetCursor += paddedSafepointSize; script->scriptList_ = offsetCursor; script->scriptEntries_ = scriptEntries; @@ -632,9 +623,11 @@ IonScript::zeroParallelInvalidatedScripts() void IonScript::copySafepointIndices(const SafepointIndex *si, MacroAssembler &masm) { - // Jumps in the caches reflect the offset of those jumps in the compiled - // code, not the absolute positions of the jumps. Update according to the - // final code address now. + /* + * Jumps in the caches reflect the offset of those jumps in the compiled + * code, not the absolute positions of the jumps. Update according to the + * final code address now. + */ SafepointIndex *table = safepointIndices(); memcpy(table, si, safepointIndexEntries_ * sizeof(SafepointIndex)); for (size_t i = 0; i < safepointIndexEntries_; i++) @@ -650,19 +643,15 @@ IonScript::copyOsiIndices(const OsiIndex *oi, MacroAssembler &masm) } void -IonScript::copyRuntimeData(const uint8_t *data) +IonScript::copyCacheEntries(const IonCache *caches, MacroAssembler &masm) { - memcpy(runtimeData(), data, runtimeSize()); -} + memcpy(cacheList(), caches, numCaches() * sizeof(IonCache)); -void -IonScript::copyCacheEntries(const uint32_t *caches, MacroAssembler &masm) -{ - memcpy(cacheIndex(), caches, numCaches() * sizeof(uint32_t)); - - // Jumps in the caches reflect the offset of those jumps in the compiled - // code, not the absolute positions of the jumps. Update according to the - // final code address now. + /* + * Jumps in the caches reflect the offset of those jumps in the compiled + * code, not the absolute positions of the jumps. Update according to the + * final code address now. + */ for (size_t i = 0; i < numCaches(); i++) getCache(i).updateBaseAddress(method_, masm); } diff --git a/js/src/ion/IonCaches.cpp b/js/src/ion/IonCaches.cpp index dc37dc119a6..681de28378b 100644 --- a/js/src/ion/IonCaches.cpp +++ b/js/src/ion/IonCaches.cpp @@ -46,7 +46,7 @@ CodeLocationJump::repoint(IonCode *code, MacroAssembler *masm) #ifdef JS_SMALL_BRANCH jumpTableEntry_ = Assembler::PatchableJumpAddress(code, (size_t) jumpTableEntryOffset); #endif - setAbsolute(); + absolute_ = true; } void @@ -63,7 +63,7 @@ CodeLocationLabel::repoint(IonCode *code, MacroAssembler *masm) JS_ASSERT(new_off < code->instructionsSize()); raw_ = code->raw() + new_off; - setAbsolute(); + absolute_ = true; } void @@ -81,100 +81,7 @@ CodeOffsetJump::fixup(MacroAssembler *masm) #endif } -const char * -IonCache::CacheName(IonCache::Kind kind) -{ - static const char *names[] = - { -#define NAME(x) #x, - IONCACHE_KIND_LIST(NAME) -#undef NAME - }; - return names[kind]; -} - -IonCache::LinkStatus -IonCache::linkCode(JSContext *cx, MacroAssembler &masm, IonScript *ion, IonCode **code) -{ - AssertCanGC(); - Linker linker(masm); - *code = linker.newCode(cx); - if (!code) - return LINK_ERROR; - - if (ion->invalidated()) - return CACHE_FLUSHED; - - return LINK_GOOD; -} - -const size_t IonCache::MAX_STUBS = 16; - -// Value used instead of the IonCode self-reference of generated stubs. This -// value is needed for marking calls made inside stubs. This value would be -// replaced by the attachStub function after the allocation of the IonCode. The -// self-reference is used to keep the stub path alive even if the IonScript is -// invalidated or if the IC is flushed. -const ImmWord STUB_ADDR = ImmWord(uintptr_t(0xdeadc0de)); - -void -IonCache::attachStub(MacroAssembler &masm, IonCode *code, CodeOffsetJump &rejoinOffset, - CodeOffsetJump *exitOffset, CodeOffsetLabel *stubLabel) -{ - JS_ASSERT(canAttachStub()); - incrementStubCount(); - - rejoinOffset.fixup(&masm); - CodeLocationJump rejoinJump(code, rejoinOffset); - - // Update the success path to continue after the IC initial jump. - PatchJump(rejoinJump, rejoinLabel()); - - // Patch the previous exitJump of the last stub, or the jump from the - // codeGen, to jump into the newly allocated code. - PatchJump(lastJump_, CodeLocationLabel(code)); - - // If this path is not taken, we are producing an entry which can no longer - // go back into the update function. - if (exitOffset) { - exitOffset->fixup(&masm); - CodeLocationJump exitJump(code, *exitOffset); - - // When the last stub fails, it fallback to the ool call which can - // produce a stub. - PatchJump(exitJump, fallbackLabel()); - - // Next time we generate a stub, we will patch the exitJump to try the - // new stub. - lastJump_ = exitJump; - } - - // Replace the STUB_ADDR constant by the address of the generated stub, such - // as it can be kept alive even if the cache is flushed (see - // MarkIonExitFrame). - if (stubLabel) { - stubLabel->fixup(&masm); - Assembler::patchDataWithValueCheck(CodeLocationLabel(code, *stubLabel), - ImmWord(uintptr_t(code)), STUB_ADDR); - } -} - -bool -IonCache::linkAndAttachStub(JSContext *cx, MacroAssembler &masm, IonScript *ion, - const char *attachKind, CodeOffsetJump &rejoinOffset, - CodeOffsetJump *exitOffset, CodeOffsetLabel *stubLabel) -{ - IonCode *code = NULL; - LinkStatus status = linkCode(cx, masm, ion, &code); - if (status != LINK_GOOD) - return status != LINK_ERROR; - - attachStub(masm, code, rejoinOffset, exitOffset, stubLabel); - - IonSpew(IonSpew_InlineCaches, "Generated %s %s stub at %p", - attachKind, CacheName(kind()), code->raw()); - return true; -} +static const size_t MAX_STUBS = 16; static bool IsCacheableListBase(JSObject *obj) @@ -577,7 +484,7 @@ struct GetNativePropertyStub // WARNING: if the IonCode object ever moved, since we'd be rooting a nonsense // WARNING: value here. // WARNING: - stubCodePatchOffset = masm.PushWithPatch(STUB_ADDR); + stubCodePatchOffset = masm.PushWithPatch(ImmWord(uintptr_t(-1))); if (callNative) { JS_ASSERT(shape->hasGetterValue() && shape->getterValue().isObject() && @@ -712,8 +619,8 @@ struct GetNativePropertyStub }; bool -GetPropertyIC::attachReadSlot(JSContext *cx, IonScript *ion, JSObject *obj, JSObject *holder, - HandleShape shape) +IonCacheGetProperty::attachReadSlot(JSContext *cx, IonScript *ion, JSObject *obj, JSObject *holder, + HandleShape shape) { MacroAssembler masm; RepatchLabel failures; @@ -721,17 +628,37 @@ GetPropertyIC::attachReadSlot(JSContext *cx, IonScript *ion, JSObject *obj, JSOb GetNativePropertyStub getprop; getprop.generateReadSlot(cx, masm, obj, name(), holder, shape, object(), output(), &failures); - const char *attachKind = "non idempotent reading"; - if (idempotent()) - attachKind = "idempotent reading"; - return linkAndAttachStub(cx, masm, ion, attachKind, getprop.rejoinOffset, &getprop.exitOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + getprop.rejoinOffset.fixup(&masm); + getprop.exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, getprop.rejoinOffset); + CodeLocationJump exitJump(code, getprop.exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated native GETPROP stub at %p %s", code->raw(), + idempotent() ? "(idempotent)" : "(not idempotent)"); + + return true; } bool -GetPropertyIC::attachCallGetter(JSContext *cx, IonScript *ion, JSObject *obj, - JSObject *holder, HandleShape shape, - const SafepointIndex *safepointIndex, void *returnAddr) +IonCacheGetProperty::attachCallGetter(JSContext *cx, IonScript *ion, JSObject *obj, + JSObject *holder, HandleShape shape, + const SafepointIndex *safepointIndex, void *returnAddr) { + AssertCanGC(); MacroAssembler masm; RepatchLabel failures; @@ -743,21 +670,43 @@ GetPropertyIC::attachCallGetter(JSContext *cx, IonScript *ion, JSObject *obj, masm.setFramePushed(ion->frameSize()); GetNativePropertyStub getprop; - if (!getprop.generateCallGetter(cx, masm, obj, name(), holder, shape, liveRegs_, + if (!getprop.generateCallGetter(cx, masm, obj, name(), holder, shape, liveRegs, object(), output(), returnAddr, pc, &failures)) { return false; } - const char *attachKind = "non idempotent calling"; - if (idempotent()) - attachKind = "idempotent calling"; - return linkAndAttachStub(cx, masm, ion, attachKind, getprop.rejoinOffset, &getprop.exitOffset, - &getprop.stubCodePatchOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + getprop.rejoinOffset.fixup(&masm); + getprop.exitOffset.fixup(&masm); + getprop.stubCodePatchOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + Assembler::patchDataWithValueCheck(CodeLocationLabel(code, getprop.stubCodePatchOffset), + ImmWord(uintptr_t(code)), ImmWord(uintptr_t(-1))); + + CodeLocationJump rejoinJump(code, getprop.rejoinOffset); + CodeLocationJump exitJump(code, getprop.exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated native GETPROP stub at %p %s", code->raw(), + idempotent() ? "(idempotent)" : "(not idempotent)"); + + return true; } bool -GetPropertyIC::attachArrayLength(JSContext *cx, IonScript *ion, JSObject *obj) +IonCacheGetProperty::attachArrayLength(JSContext *cx, IonScript *ion, JSObject *obj) { JS_ASSERT(obj->isArray()); JS_ASSERT(!idempotent()); @@ -791,6 +740,9 @@ GetPropertyIC::attachArrayLength(JSContext *cx, IonScript *ion, JSObject *obj) if (output().hasValue()) masm.tagValue(JSVAL_TYPE_INT32, outReg, output().valueReg()); + u.getprop.hasArrayLengthStub = true; + incrementStubCount(); + /* Success. */ RepatchLabel rejoin_; CodeOffsetJump rejoinOffset = masm.jumpWithPatch(&rejoin_); @@ -802,13 +754,32 @@ GetPropertyIC::attachArrayLength(JSContext *cx, IonScript *ion, JSObject *obj) CodeOffsetJump exitOffset = masm.jumpWithPatch(&exit_); masm.bind(&exit_); - JS_ASSERT(!hasArrayLengthStub_); - hasArrayLengthStub_ = true; - return linkAndAttachStub(cx, masm, ion, "array length", rejoinOffset, &exitOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump exitJump(code, exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated GETPROP dense array length stub at %p", code->raw()); + + return true; } bool -GetPropertyIC::attachTypedArrayLength(JSContext *cx, IonScript *ion, JSObject *obj) +IonCacheGetProperty::attachTypedArrayLength(JSContext *cx, IonScript *ion, JSObject *obj) { JS_ASSERT(obj->isTypedArray()); JS_ASSERT(!idempotent()); @@ -833,6 +804,9 @@ GetPropertyIC::attachTypedArrayLength(JSContext *cx, IonScript *ion, JSObject *o // Load length. masm.loadTypedOrValue(Address(object(), TypedArray::lengthOffset()), output()); + u.getprop.hasTypedArrayLengthStub = true; + incrementStubCount(); + /* Success. */ RepatchLabel rejoin_; CodeOffsetJump rejoinOffset = masm.jumpWithPatch(&rejoin_); @@ -844,14 +818,33 @@ GetPropertyIC::attachTypedArrayLength(JSContext *cx, IonScript *ion, JSObject *o CodeOffsetJump exitOffset = masm.jumpWithPatch(&exit_); masm.bind(&exit_); - JS_ASSERT(!hasTypedArrayLengthStub_); - hasTypedArrayLengthStub_ = true; - return linkAndAttachStub(cx, masm, ion, "typed array length", rejoinOffset, &exitOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump exitJump(code, exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated GETPROP typed array length stub at %p", code->raw()); + + return true; } static bool TryAttachNativeGetPropStub(JSContext *cx, IonScript *ion, - GetPropertyIC &cache, HandleObject obj, + IonCacheGetProperty &cache, HandleObject obj, HandlePropertyName name, const SafepointIndex *safepointIndex, void *returnAddr, bool *isCacheable) @@ -925,20 +918,22 @@ TryAttachNativeGetPropStub(JSContext *cx, IonScript *ion, JS_ASSERT_IF(readSlot, !callGetter); JS_ASSERT_IF(callGetter, !readSlot); - // Falback to the interpreter function. - if (!cache.canAttachStub()) - return true; + if (cache.stubCount() < MAX_STUBS) { + cache.incrementStubCount(); - if (readSlot) - return cache.attachReadSlot(cx, ion, obj, holder, shape); - else if (obj->isArray() && !cache.hasArrayLengthStub() && cx->names().length == name) - return cache.attachArrayLength(cx, ion, obj); - return cache.attachCallGetter(cx, ion, obj, holder, shape, safepointIndex, returnAddr); + if (readSlot) + return cache.attachReadSlot(cx, ion, obj, holder, shape); + else if (obj->isArray() && !cache.hasArrayLengthStub() && cx->names().length == name) + return cache.attachArrayLength(cx, ion, obj); + else + return cache.attachCallGetter(cx, ion, obj, holder, shape, safepointIndex, returnAddr); + } + + return true; } bool -GetPropertyIC::update(JSContext *cx, size_t cacheIndex, - HandleObject obj, MutableHandleValue vp) +js::ion::GetPropertyCache(JSContext *cx, size_t cacheIndex, HandleObject obj, MutableHandleValue vp) { AutoFlushCache afc ("GetPropertyCache"); const SafepointIndex *safepointIndex; @@ -946,9 +941,13 @@ GetPropertyIC::update(JSContext *cx, size_t cacheIndex, RootedScript topScript(cx, GetTopIonJSScript(cx, &safepointIndex, &returnAddr)); IonScript *ion = topScript->ionScript(); - GetPropertyIC &cache = ion->getCache(cacheIndex).toGetProperty(); + IonCacheGetProperty &cache = ion->getCache(cacheIndex).toGetProperty(); RootedPropertyName name(cx, cache.name()); + RootedScript script(cx); + jsbytecode *pc; + cache.getScriptedLocation(&script, &pc); + // Override the return value if we are invalidated (bug 728188). AutoDetectInvalidation adi(cx, vp.address(), ion); @@ -967,9 +966,7 @@ GetPropertyIC::update(JSContext *cx, size_t cacheIndex, return false; } - if (!isCacheable && cache.canAttachStub() && - !cache.idempotent() && cx->names().length == name) - { + if (!isCacheable && !cache.idempotent() && cx->names().length == name) { if (cache.output().type() != MIRType_Value && cache.output().type() != MIRType_Int32) { // The next execution should cause an invalidation because the type // does not fit. @@ -1010,10 +1007,6 @@ GetPropertyIC::update(JSContext *cx, size_t cacheIndex, } if (!cache.idempotent()) { - RootedScript script(cx); - jsbytecode *pc; - cache.getScriptedLocation(&script, &pc); - // If the cache is idempotent, the property exists so we don't have to // call __noSuchMethod__. @@ -1037,7 +1030,7 @@ IonCache::updateBaseAddress(IonCode *code, MacroAssembler &masm) { initialJump_.repoint(code, &masm); lastJump_.repoint(code, &masm); - fallbackLabel_.repoint(code, &masm); + cacheLabel_.repoint(code, &masm); } void @@ -1050,17 +1043,15 @@ IonCache::disable() void IonCache::reset() { - // Skip all generated stub by patching the original stub to go directly to - // the update function. - PatchJump(initialJump_, fallbackLabel_); + PatchJump(initialJump_, cacheLabel_); this->stubCount_ = 0; this->lastJump_ = initialJump_; } bool -SetPropertyIC::attachNativeExisting(JSContext *cx, IonScript *ion, - HandleObject obj, HandleShape shape) +IonCacheSetProperty::attachNativeExisting(JSContext *cx, IonScript *ion, + HandleObject obj, HandleShape shape) { MacroAssembler masm; @@ -1095,13 +1086,34 @@ SetPropertyIC::attachNativeExisting(JSContext *cx, IonScript *ion, CodeOffsetJump rejoinOffset = masm.jumpWithPatch(&rejoin_); masm.bind(&rejoin_); - return linkAndAttachStub(cx, masm, ion, "setting", rejoinOffset, &exitOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump exitJump(code, exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated native SETPROP setting case stub at %p", code->raw()); + + return true; } bool -SetPropertyIC::attachSetterCall(JSContext *cx, IonScript *ion, - HandleObject obj, HandleObject holder, HandleShape shape, - void *returnAddr) +IonCacheSetProperty::attachSetterCall(JSContext *cx, IonScript *ion, + HandleObject obj, HandleObject holder, HandleShape shape, + void *returnAddr) { MacroAssembler masm; @@ -1151,7 +1163,7 @@ SetPropertyIC::attachSetterCall(JSContext *cx, IonScript *ion, // Good to go for invoking setter. // saveLive() - masm.PushRegsInMask(liveRegs_); + masm.PushRegsInMask(liveRegs); // Remaining registers should basically be free, but we need to use |object| still // so leave it alone. @@ -1183,7 +1195,7 @@ SetPropertyIC::attachSetterCall(JSContext *cx, IonScript *ion, // WARNING: if the IonCode object ever moved, since we'd be rooting a nonsense // WARNING: value here. // WARNING: - CodeOffsetLabel stubCodePatchOffset = masm.PushWithPatch(STUB_ADDR); + CodeOffsetLabel stubCodePatchOffset = masm.PushWithPatch(ImmWord(uintptr_t(-1))); StrictPropertyOp target = shape->setterOp(); JS_ASSERT(target); @@ -1244,7 +1256,7 @@ SetPropertyIC::attachSetterCall(JSContext *cx, IonScript *ion, JS_ASSERT(masm.framePushed() == initialStack); // restoreLive() - masm.PopRegsInMask(liveRegs_); + masm.PopRegsInMask(liveRegs); // Rejoin jump. RepatchLabel rejoin; @@ -1257,14 +1269,38 @@ SetPropertyIC::attachSetterCall(JSContext *cx, IonScript *ion, CodeOffsetJump exitOffset = masm.jumpWithPatch(&exit); masm.bind(&exit); - return linkAndAttachStub(cx, masm, ion, "calling", rejoinOffset, &exitOffset, - &stubCodePatchOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + exitOffset.fixup(&masm); + stubCodePatchOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + Assembler::patchDataWithValueCheck(CodeLocationLabel(code, stubCodePatchOffset), + ImmWord(uintptr_t(code)), ImmWord(uintptr_t(-1))); + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump exitJump(code, exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated SETPROP calling case stub at %p", code->raw()); + + return true; } bool -SetPropertyIC::attachNativeAdding(JSContext *cx, IonScript *ion, JSObject *obj, - HandleShape oldShape, HandleShape newShape, - HandleShape propShape) +IonCacheSetProperty::attachNativeAdding(JSContext *cx, IonScript *ion, JSObject *obj, + HandleShape oldShape, HandleShape newShape, + HandleShape propShape) { MacroAssembler masm; @@ -1331,12 +1367,38 @@ SetPropertyIC::attachNativeAdding(JSContext *cx, IonScript *ion, JSObject *obj, CodeOffsetJump exitOffset = masm.jumpWithPatch(&exit_); masm.bind(&exit_); - return linkAndAttachStub(cx, masm, ion, "adding", rejoinOffset, &exitOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump exitJump(code, exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated native SETPROP adding case stub at %p", code->raw()); + + return true; } static bool -IsPropertyInlineable(JSObject *obj) +IsPropertyInlineable(JSObject *obj, IonCacheSetProperty &cache) { + // Stop generating new stubs once we hit the stub count limit, see + // GetPropertyCache. + if (cache.stubCount() >= MAX_STUBS) + return false; + if (!obj->isNative()) return false; @@ -1444,8 +1506,8 @@ IsPropertyAddInlineable(JSContext *cx, HandleObject obj, HandleId id, uint32_t o } bool -SetPropertyIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, - HandleValue value) +js::ion::SetPropertyCache(JSContext *cx, size_t cacheIndex, HandleObject obj, HandleValue value, + bool isSetName) { AutoFlushCache afc ("SetPropertyCache"); @@ -1453,19 +1515,18 @@ SetPropertyIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, const SafepointIndex *safepointIndex; RootedScript script(cx, GetTopIonJSScript(cx, &safepointIndex, &returnAddr)); IonScript *ion = script->ion; - SetPropertyIC &cache = ion->getCache(cacheIndex).toSetProperty(); + IonCacheSetProperty &cache = ion->getCache(cacheIndex).toSetProperty(); RootedPropertyName name(cx, cache.name()); RootedId id(cx, AtomToId(name)); RootedShape shape(cx); RootedObject holder(cx); - // Stop generating new stubs once we hit the stub count limit, see - // GetPropertyCache. - bool inlinable = cache.canAttachStub() && IsPropertyInlineable(obj); + bool inlinable = IsPropertyInlineable(obj, cache); bool addedSetterStub = false; if (inlinable) { RootedShape shape(cx); if (IsPropertySetInlineable(cx, obj, id, &shape)) { + cache.incrementStubCount(); if (!cache.attachNativeExisting(cx, ion, obj, shape)) return false; addedSetterStub = true; @@ -1475,6 +1536,7 @@ SetPropertyIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, return false; if (IsPropertySetterCallInlineable(cx, obj, holder, shape)) { + cache.incrementStubCount(); if (!cache.attachSetterCall(cx, ion, obj, holder, shape, returnAddr)) return false; addedSetterStub = true; @@ -1486,14 +1548,15 @@ SetPropertyIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, RootedShape oldShape(cx, obj->lastProperty()); // Set/Add the property on the object, the inlined cache are setup for the next execution. - if (!SetProperty(cx, obj, name, value, cache.strict(), cache.isSetName())) + if (!SetProperty(cx, obj, name, value, cache.strict(), isSetName)) return false; - // The property did not exist before, now we can try to inline the property add. + // The property did not exist before, now we can try to inline the propery add. if (inlinable && !addedSetterStub && obj->lastProperty() != oldShape && IsPropertyAddInlineable(cx, obj, id, oldSlots, &shape)) { RootedShape newShape(cx, obj->lastProperty()); + cache.incrementStubCount(); if (!cache.attachNativeAdding(cx, ion, obj, oldShape, newShape, shape)) return false; } @@ -1502,8 +1565,8 @@ SetPropertyIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, } bool -GetElementIC::attachGetProp(JSContext *cx, IonScript *ion, HandleObject obj, - const Value &idval, HandlePropertyName name) +IonCacheGetElement::attachGetProp(JSContext *cx, IonScript *ion, HandleObject obj, + const Value &idval, HandlePropertyName name) { RootedObject holder(cx); RootedShape shape(cx); @@ -1531,14 +1594,33 @@ GetElementIC::attachGetProp(JSContext *cx, IonScript *ion, HandleObject obj, masm.branchTestValue(Assembler::NotEqual, val, idval, &nonRepatchFailures); GetNativePropertyStub getprop; - getprop.generateReadSlot(cx, masm, obj, name, holder, shape, object(), output(), &failures, - &nonRepatchFailures); + getprop.generateReadSlot(cx, masm, obj, name, holder, shape, object(), output(), &failures, &nonRepatchFailures); - return linkAndAttachStub(cx, masm, ion, "property", getprop.rejoinOffset, &getprop.exitOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + getprop.rejoinOffset.fixup(&masm); + getprop.exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, getprop.rejoinOffset); + CodeLocationJump exitJump(code, getprop.exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated GETELEM property stub at %p", code->raw()); + return true; } bool -GetElementIC::attachDenseElement(JSContext *cx, IonScript *ion, JSObject *obj, const Value &idval) +IonCacheGetElement::attachDenseElement(JSContext *cx, IonScript *ion, JSObject *obj, const Value &idval) { JS_ASSERT(obj->isNative()); JS_ASSERT(idval.isInt32()); @@ -1592,90 +1674,37 @@ GetElementIC::attachDenseElement(JSContext *cx, IonScript *ion, JSObject *obj, c CodeOffsetJump exitOffset = masm.jumpWithPatch(&exit_); masm.bind(&exit_); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump exitJump(code, exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + setHasDenseStub(); - return linkAndAttachStub(cx, masm, ion, "dense array", rejoinOffset, &exitOffset); + IonSpew(IonSpew_InlineCaches, "Generated GETELEM dense array stub at %p", code->raw()); + + return true; } bool -GetElementIC::attachTypedArrayElement(JSContext *cx, IonScript *ion, JSObject *obj, - const Value &idval) -{ - JS_ASSERT(obj->isTypedArray()); - JS_ASSERT(idval.isInt32()); - - Label failures; - MacroAssembler masm; - - // The array type is the object within the table of typed array classes. - int arrayType = obj->getClass() - &TypedArray::classes[0]; - - Register tmpReg; - if (output().hasValue()) { - tmpReg = output().valueReg().scratchReg(); - } else { - JS_ASSERT(output().type() == MIRType_Int32); - tmpReg = output().typedReg().gpr(); - } - JS_ASSERT(object() != tmpReg); - - // Check that the typed array is of the same type as the current object - // because load size differ in function of the typed array data width. - masm.branchTestObjClass(Assembler::NotEqual, object(), tmpReg, obj->getClass(), &failures); - - // Ensure the index is an int32 value. - ValueOperand val = index().reg().valueReg(); - masm.branchTestInt32(Assembler::NotEqual, val, &failures); - - // Unbox the index. - Register indexReg = tmpReg; - masm.unboxInt32(val, indexReg); - - // Guard on the initialized length. - Address length(object(), TypedArray::lengthOffset()); - masm.branch32(Assembler::BelowOrEqual, length, indexReg, &failures); - - // Save the object register on the stack in case of failure. - Label popAndFail; - Register elementReg = object(); - masm.push(object()); - - // Load elements vector. - masm.loadPtr(Address(object(), TypedArray::dataOffset()), elementReg); - - // Load the value. We use an invalid register because the destination - // register is necessary a non double register. - int width = TypedArray::slotWidth(arrayType); - BaseIndex source(elementReg, indexReg, ScaleFromElemWidth(width)); - if (output().hasValue()) - masm.loadFromTypedArray(arrayType, source, output().valueReg(), true, - elementReg, &popAndFail); - else - masm.loadFromTypedArray(arrayType, source, output().typedReg(), - elementReg, &popAndFail); - - masm.pop(object()); - RepatchLabel rejoin_; - CodeOffsetJump rejoinOffset = masm.jumpWithPatch(&rejoin_); - masm.bind(&rejoin_); - - // Restore the object before continuing to the next stub. - masm.bind(&popAndFail); - masm.pop(object()); - masm.bind(&failures); - - RepatchLabel exit_; - CodeOffsetJump exitOffset = masm.jumpWithPatch(&exit_); - masm.bind(&exit_); - - return linkAndAttachStub(cx, masm, ion, "typed array", rejoinOffset, &exitOffset); -} - -bool -GetElementIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, - HandleValue idval, MutableHandleValue res) +js::ion::GetElementCache(JSContext *cx, size_t cacheIndex, HandleObject obj, HandleValue idval, + MutableHandleValue res) { IonScript *ion = GetTopIonJSScript(cx)->ionScript(); - GetElementIC &cache = ion->getCache(cacheIndex).toGetElement(); + IonCacheGetElement &cache = ion->getCache(cacheIndex).toGetElement(); RootedScript script(cx); jsbytecode *pc; cache.getScriptedLocation(&script, &pc); @@ -1689,6 +1718,7 @@ GetElementIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, } // Override the return value if we are invalidated (bug 728188). + AutoFlushCache afc ("GetElementCache"); AutoDetectInvalidation adi(cx, res.address(), ion); RootedId id(cx); @@ -1696,8 +1726,10 @@ GetElementIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, return false; bool attachedStub = false; - if (cache.canAttachStub()) { + if (cache.stubCount() < MAX_STUBS) { if (obj->isNative() && cache.monitoredResult()) { + cache.incrementStubCount(); + uint32_t dummy; if (idval.isString() && JSID_IS_ATOM(id) && !JSID_TO_ATOM(id)->isIndex(&dummy)) { RootedPropertyName name(cx, JSID_TO_ATOM(id)->asPropertyName()); @@ -1706,13 +1738,12 @@ GetElementIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, attachedStub = true; } } else if (!cache.hasDenseStub() && obj->isNative() && idval.isInt32()) { + // Generate at most one dense array stub. + cache.incrementStubCount(); + if (!cache.attachDenseElement(cx, ion, obj, idval)) return false; attachedStub = true; - } else if (obj->isTypedArray() && idval.isInt32()) { - if (!cache.attachTypedArrayElement(cx, ion, obj, idval)) - return false; - attachedStub = true; } } @@ -1721,7 +1752,7 @@ GetElementIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, // If no new attach was done, and we've reached maximum number of stubs, then // disable the cache. - if (!attachedStub && !cache.canAttachStub()) + if (!attachedStub && cache.stubCount() >= MAX_STUBS) cache.disable(); types::TypeScript::Monitor(cx, script, pc, res); @@ -1729,7 +1760,7 @@ GetElementIC::update(JSContext *cx, size_t cacheIndex, HandleObject obj, } bool -BindNameIC::attachGlobal(JSContext *cx, IonScript *ion, JSObject *scopeChain) +IonCacheBindName::attachGlobal(JSContext *cx, IonScript *ion, JSObject *scopeChain) { JS_ASSERT(scopeChain->isGlobal()); @@ -1746,7 +1777,27 @@ BindNameIC::attachGlobal(JSContext *cx, IonScript *ion, JSObject *scopeChain) CodeOffsetJump rejoinOffset = masm.jumpWithPatch(&rejoin_); masm.bind(&rejoin_); - return linkAndAttachStub(cx, masm, ion, "global", rejoinOffset, &exitOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump exitJump(code, exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated BINDNAME global stub at %p", code->raw()); + return true; } static inline void @@ -1799,7 +1850,7 @@ GenerateScopeChainGuards(MacroAssembler &masm, JSObject *scopeChain, JSObject *h } bool -BindNameIC::attachNonGlobal(JSContext *cx, IonScript *ion, JSObject *scopeChain, JSObject *holder) +IonCacheBindName::attachNonGlobal(JSContext *cx, IonScript *ion, JSObject *scopeChain, JSObject *holder) { JS_ASSERT(IsCacheableNonGlobalScope(scopeChain)); @@ -1838,7 +1889,27 @@ BindNameIC::attachNonGlobal(JSContext *cx, IonScript *ion, JSObject *scopeChain, masm.bind(&exit_); } - return linkAndAttachStub(cx, masm, ion, "non-global", rejoinOffset, &exitOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump exitJump(code, exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated BINDNAME non-global stub at %p", code->raw()); + return true; } static bool @@ -1865,12 +1936,12 @@ IsCacheableScopeChain(JSObject *scopeChain, JSObject *holder) } JSObject * -BindNameIC::update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain) +js::ion::BindNameCache(JSContext *cx, size_t cacheIndex, HandleObject scopeChain) { AutoFlushCache afc ("BindNameCache"); IonScript *ion = GetTopIonJSScript(cx)->ionScript(); - BindNameIC &cache = ion->getCache(cacheIndex).toBindName(); + IonCacheBindName &cache = ion->getCache(cacheIndex).toBindName(); HandlePropertyName name = cache.name(); RootedObject holder(cx); @@ -1883,7 +1954,9 @@ BindNameIC::update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain) // Stop generating new stubs once we hit the stub count limit, see // GetPropertyCache. - if (cache.canAttachStub()) { + if (cache.stubCount() < MAX_STUBS) { + cache.incrementStubCount(); + if (scopeChain->isGlobal()) { if (!cache.attachGlobal(cx, ion, scopeChain)) return NULL; @@ -1899,7 +1972,8 @@ BindNameIC::update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain) } bool -NameIC::attach(JSContext *cx, IonScript *ion, HandleObject scopeChain, HandleObject holder, HandleShape shape) +IonCacheName::attach(JSContext *cx, IonScript *ion, HandleObject scopeChain, HandleObject holder, + HandleShape shape) { AssertCanGC(); MacroAssembler masm; @@ -1935,8 +2009,30 @@ NameIC::attach(JSContext *cx, IonScript *ion, HandleObject scopeChain, HandleObj masm.bind(&exit); } - return linkAndAttachStub(cx, masm, ion, "generic", rejoinOffset, - (failures.bound() ? &exitOffset : NULL)); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + if (failures.bound()) + exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + if (failures.bound()) { + CodeLocationJump exitJump(code, exitOffset); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + } + + IonSpew(IonSpew_InlineCaches, "Generated NAME stub at %p", code->raw()); + return true; } static bool @@ -1979,14 +2075,13 @@ IsCacheableName(JSContext *cx, HandleObject scopeChain, HandleObject obj, Handle } bool -NameIC::update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain, - MutableHandleValue vp) +js::ion::GetNameCache(JSContext *cx, size_t cacheIndex, HandleObject scopeChain, MutableHandleValue vp) { AutoFlushCache afc ("GetNameCache"); IonScript *ion = GetTopIonJSScript(cx)->ionScript(); - NameIC &cache = ion->getCache(cacheIndex).toName(); + IonCacheName &cache = ion->getCache(cacheIndex).toName(); RootedPropertyName name(cx, cache.name()); RootedScript script(cx); @@ -1999,11 +2094,12 @@ NameIC::update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain, if (!LookupName(cx, name, scopeChain, &obj, &holder, &shape)) return false; - if (cache.canAttachStub() && + if (cache.stubCount() < MAX_STUBS && IsCacheableName(cx, scopeChain, obj, holder, shape, pc, cache.outputReg())) { if (!cache.attach(cx, ion, scopeChain, obj, shape)) return false; + cache.incrementStubCount(); } if (cache.isTypeOf()) { @@ -2021,8 +2117,8 @@ NameIC::update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain, } bool -CallsiteCloneIC::attach(JSContext *cx, IonScript *ion, HandleFunction original, - HandleFunction clone) +IonCacheCallsiteClone::attach(JSContext *cx, IonScript *ion, HandleFunction original, + HandleFunction clone) { MacroAssembler masm; @@ -2039,11 +2135,31 @@ CallsiteCloneIC::attach(JSContext *cx, IonScript *ion, HandleFunction original, CodeOffsetJump rejoinOffset = masm.jumpWithPatch(&rejoin); masm.bind(&rejoin); - return linkAndAttachStub(cx, masm, ion, "generic", rejoinOffset, &exitOffset); + Linker linker(masm); + IonCode *code = linker.newCode(cx); + if (!code) + return false; + + rejoinOffset.fixup(&masm); + exitOffset.fixup(&masm); + + if (ion->invalidated()) + return true; + + CodeLocationJump rejoinJump(code, rejoinOffset); + CodeLocationJump exitJump(code, exitOffset); + CodeLocationJump lastJump_ = lastJump(); + PatchJump(lastJump_, CodeLocationLabel(code)); + PatchJump(rejoinJump, rejoinLabel()); + PatchJump(exitJump, cacheLabel()); + updateLastJump(exitJump); + + IonSpew(IonSpew_InlineCaches, "Generated CALL callee clone stub at %p", code->raw()); + return true; } JSObject * -CallsiteCloneIC::update(JSContext *cx, size_t cacheIndex, HandleObject callee) +js::ion::CallsiteCloneCache(JSContext *cx, size_t cacheIndex, HandleObject callee) { AutoFlushCache afc ("CallsiteCloneCache"); @@ -2054,15 +2170,16 @@ CallsiteCloneIC::update(JSContext *cx, size_t cacheIndex, HandleObject callee) return fun; IonScript *ion = GetTopIonJSScript(cx)->ionScript(); - CallsiteCloneIC &cache = ion->getCache(cacheIndex).toCallsiteClone(); + IonCacheCallsiteClone &cache = ion->getCache(cacheIndex).toCallsiteClone(); RootedFunction clone(cx, CloneFunctionAtCallsite(cx, fun, cache.callScript(), cache.callPc())); if (!clone) return NULL; - if (cache.canAttachStub()) { + if (cache.stubCount() < MAX_STUBS) { if (!cache.attach(cx, ion, fun, clone)) return NULL; + cache.incrementStubCount(); } return clone; diff --git a/js/src/ion/IonCaches.h b/js/src/ion/IonCaches.h index e514e1ed9ce..64f6973c56d 100644 --- a/js/src/ion/IonCaches.h +++ b/js/src/ion/IonCaches.h @@ -18,31 +18,12 @@ class JSScript; namespace js { namespace ion { -#define IONCACHE_KIND_LIST(_) \ - _(GetProperty) \ - _(SetProperty) \ - _(GetElement) \ - _(BindName) \ - _(Name) \ - _(CallsiteClone) - -// Forward declarations of Cache kinds. -#define FORWARD_DECLARE(kind) class kind##IC; -IONCACHE_KIND_LIST(FORWARD_DECLARE) -#undef FORWARD_DECLARE - -class IonCacheVisitor -{ - public: -#define VISIT_INS(op) \ - virtual bool visit##op##IC(CodeGenerator *codegen, op##IC *) { \ - JS_NOT_REACHED("NYI: " #op "IC"); \ - return false; \ - } - - IONCACHE_KIND_LIST(VISIT_INS) -#undef VISIT_INS -}; +class IonCacheGetProperty; +class IonCacheSetProperty; +class IonCacheGetElement; +class IonCacheBindName; +class IonCacheName; +class IonCacheCallsiteClone; // Common structure encoding the state of a polymorphic inline cache contained // in the code for an IonScript. IonCaches are used for polymorphic operations @@ -74,70 +55,45 @@ class IonCacheVisitor // for a cache, the cache itself may be marked as idempotent and become hoisted // or coalesced by LICM or GVN. This also constrains the stubs which can be // generated for the cache. -// -// * IonCache usage -// -// IonCache is the base structure of an inline cache, which generates code stubs -// dynamically and attaches them to an IonScript. -// -// A cache must at least provide a static update function which will usualy have -// a JSContext*, followed by the cache index. The rest of the arguments of the -// update function are usualy corresponding to the register inputs of the cache, -// as it must perform the same operation as any of the stubs that it might -// produce. The update function call is handled by the visit function of -// CodeGenerator corresponding to this IC. -// -// The CodeGenerator visit function, as opposed to other visit functions, has -// two arguments. The first one is the OutOfLineUpdateCache which stores the LIR -// instruction. The second one is the IC object. This function would be called -// once the IC is registered with the addCache function of CodeGeneratorShared. -// -// To register a cache, you must call the addCache function as follow: -// -// MyCodeIC cache(inputReg1, inputValueReg2, outputReg); -// if (!addCache(lir, allocateCache(cache))) -// return false; -// -// Once the cache is allocated with the allocateCache function, any modification -// made to the cache would be ignored. -// -// The addCache function will produce a patchable jump at the location where -// it is called. This jump will execute generated stubs and fallback on the code -// of the visitMyCodeIC function if no stub match. -// -// Warning: As the addCache function fallback on a VMCall, calls to -// addCache should not be in the same path as another VMCall or in the same -// path of another addCache as this is not supported by the invalidation -// procedure. + +struct TypedOrValueRegisterSpace +{ + mozilla::AlignedStorage2 data_; + TypedOrValueRegister &data() { + return *data_.addr(); + } + const TypedOrValueRegister &data() const { + return *data_.addr(); + } +}; + +struct ConstantOrRegisterSpace +{ + mozilla::AlignedStorage2 data_; + ConstantOrRegister &data() { + return *data_.addr(); + } + const ConstantOrRegister &data() const { + return *data_.addr(); + } +}; + class IonCache { public: enum Kind { -# define DEFINE_CACHEKINDS(ickind) Cache_##ickind, - IONCACHE_KIND_LIST(DEFINE_CACHEKINDS) -# undef DEFINE_CACHEKINDS - Cache_Invalid + Invalid = 0, + GetProperty, + SetProperty, + GetElement, + BindName, + Name, + NameTypeOf, + CallsiteClone }; - // Cache testing and cast. -# define CACHEKIND_CASTS(ickind) \ - bool is##ickind() const { \ - return kind() == Cache_##ickind; \ - } \ - inline ickind##IC &to##ickind(); - - IONCACHE_KIND_LIST(CACHEKIND_CASTS) -# undef CACHEKIND_CASTS - - virtual Kind kind() const = 0; - - virtual bool accept(CodeGenerator *codegen, IonCacheVisitor *visitor) = 0; - - public: - - static const char *CacheName(Kind kind); - protected: + Kind kind_ : 8; bool pure_ : 1; bool idempotent_ : 1; bool disabled_ : 1; @@ -145,7 +101,7 @@ class IonCache CodeLocationJump initialJump_; CodeLocationJump lastJump_; - CodeLocationLabel fallbackLabel_; + CodeLocationLabel cacheLabel_; // Offset from the initial jump to the rejoin label. #ifdef JS_CPU_ARM @@ -153,22 +109,85 @@ class IonCache #else static const size_t REJOIN_LABEL_OFFSET = 0; #endif + union { + struct { + Register object; + PropertyName *name; + TypedOrValueRegisterSpace output; + bool allowGetters : 1; + bool hasArrayLengthStub : 1; + bool hasTypedArrayLengthStub : 1; + } getprop; + struct { + Register object; + PropertyName *name; + ConstantOrRegisterSpace value; + bool strict; + } setprop; + struct { + Register object; + ConstantOrRegisterSpace index; + TypedOrValueRegisterSpace output; + bool monitoredResult : 1; + bool hasDenseStub : 1; + } getelem; + struct { + Register scopeChain; + PropertyName *name; + Register output; + } bindname; + struct { + Register scopeChain; + PropertyName *name; + TypedOrValueRegisterSpace output; + } name; + struct { + Register callee; + Register output; + JSScript *callScript; + jsbytecode *callPc; + } callsiteclone; + } u; + + // Registers live after the cache, excluding output registers. The initial + // value of these registers must be preserved by the cache. + RegisterSet liveRegs; // Location of this operation, NULL for idempotent caches. JSScript *script; jsbytecode *pc; - private: - static const size_t MAX_STUBS; - void incrementStubCount() { - // The IC should stop generating stubs before wrapping stubCount. - stubCount_++; - JS_ASSERT(stubCount_); + void init(Kind kind, RegisterSet liveRegs, + CodeOffsetJump initialJump, + CodeOffsetLabel rejoinLabel, + CodeOffsetLabel cacheLabel) { + this->kind_ = kind; + this->liveRegs = liveRegs; + this->initialJump_ = initialJump; + this->lastJump_ = initialJump; + this->cacheLabel_ = cacheLabel; + + JS_ASSERT(rejoinLabel.offset() == initialJump.offset() + REJOIN_LABEL_OFFSET); } - CodeLocationLabel fallbackLabel() const { - return fallbackLabel_; + public: + + IonCache() { PodZero(this); } + + void updateBaseAddress(IonCode *code, MacroAssembler &masm); + + // disable the IC. + void disable(); + inline bool isDisabled() const { + return disabled_; } + + // Reset the cache around garbage collection. + void reset(); + + CodeLocationJump lastJump() const { return lastJump_; } + CodeLocationLabel cacheLabel() const { return cacheLabel_; } + CodeLocationLabel rejoinLabel() const { uint8_t *ptr = initialJump_.raw(); #ifdef JS_CPU_ARM @@ -179,77 +198,6 @@ class IonCache return CodeLocationLabel(ptr); } - public: - - IonCache() - : pure_(false), - idempotent_(false), - disabled_(false), - stubCount_(0), - initialJump_(), - lastJump_(), - fallbackLabel_(), - script(NULL), - pc(NULL) - { - } - - void disable(); - inline bool isDisabled() const { - return disabled_; - } - - // Set the initial jump state of the cache. The initialJump is the inline - // jump that will point to out-of-line code (such as the slow path, or - // stubs), and the rejoinLabel is the position that all out-of-line paths - // will rejoin to. - void setInlineJump(CodeOffsetJump initialJump, CodeOffsetLabel rejoinLabel) { - initialJump_ = initialJump; - lastJump_ = initialJump; - - JS_ASSERT(rejoinLabel.offset() == initialJump.offset() + REJOIN_LABEL_OFFSET); - } - - // Set the initial 'out-of-line' jump state of the cache. The fallbackLabel is - // the location of the out-of-line update (slow) path. This location will - // be set to the exitJump of the last generated stub. - void setFallbackLabel(CodeOffsetLabel fallbackLabel) { - fallbackLabel_ = fallbackLabel; - } - - // Update labels once the code is copied and finalized. - void updateBaseAddress(IonCode *code, MacroAssembler &masm); - - // Reset the cache around garbage collection. - void reset(); - - bool canAttachStub() const { - return stubCount_ < MAX_STUBS; - } - - enum LinkStatus { - LINK_ERROR, - CACHE_FLUSHED, - LINK_GOOD - }; - - // Use the Linker to link the generated code and check if any - // monitoring/allocation caused an invalidation of the running ion script, - // this function returns CACHE_FLUSHED. In case of allocation issue this - // function returns LINK_ERROR. - LinkStatus linkCode(JSContext *cx, MacroAssembler &masm, IonScript *ion, IonCode **code); - - // Fixup variables and update jumps in the list of stubs. Increment the - // number of attached stubs accordingly. - void attachStub(MacroAssembler &masm, IonCode *code, CodeOffsetJump &rejoinOffset, - CodeOffsetJump *exitOffset, CodeOffsetLabel *stubOffset = NULL); - - // Combine both linkCode and attachStub into one function. In addition, it - // produces a spew augmented with the attachKind string. - bool linkAndAttachStub(JSContext *cx, MacroAssembler &masm, IonScript *ion, - const char *attachKind, CodeOffsetJump &rejoinOffset, - CodeOffsetJump *exitOffset, CodeOffsetLabel *stubOffset = NULL); - bool pure() { return pure_; } @@ -263,6 +211,44 @@ class IonCache idempotent_ = true; } + void updateLastJump(CodeLocationJump jump) { + lastJump_ = jump; + } + + size_t stubCount() const { + return stubCount_; + } + void incrementStubCount() { + // The IC should stop generating stubs before wrapping stubCount. + stubCount_++; + JS_ASSERT(stubCount_); + } + + IonCacheGetProperty &toGetProperty() { + JS_ASSERT(kind_ == GetProperty); + return *(IonCacheGetProperty *)this; + } + IonCacheSetProperty &toSetProperty() { + JS_ASSERT(kind_ == SetProperty); + return *(IonCacheSetProperty *)this; + } + IonCacheGetElement &toGetElement() { + JS_ASSERT(kind_ == GetElement); + return *(IonCacheGetElement *)this; + } + IonCacheBindName &toBindName() { + JS_ASSERT(kind_ == BindName); + return *(IonCacheBindName *)this; + } + IonCacheName &toName() { + JS_ASSERT(kind_ == Name || kind_ == NameTypeOf); + return *(IonCacheName *)this; + } + IonCacheCallsiteClone &toCallsiteClone() { + JS_ASSERT(kind_ == CallsiteClone); + return *(IonCacheCallsiteClone *)this; + } + void setScriptedLocation(UnrootedScript script, jsbytecode *pc) { JS_ASSERT(!idempotent_); this->script = script; @@ -275,71 +261,41 @@ class IonCache } }; -// Define the cache kind and pre-declare data structures used for calling inline -// caches. -#define CACHE_HEADER(ickind) \ - Kind kind() const { \ - return IonCache::Cache_##ickind; \ - } \ - \ - bool accept(CodeGenerator *codegen, IonCacheVisitor *visitor) { \ - return visitor->visit##ickind##IC(codegen, this); \ - } \ - \ - static const VMFunction UpdateInfo; +inline IonCache & +IonScript::getCache(size_t index) { + JS_ASSERT(index < numCaches()); + return cacheList()[index]; +} // Subclasses of IonCache for the various kinds of caches. These do not define // new data members; all caches must be of the same size. -class GetPropertyIC : public IonCache +class IonCacheGetProperty : public IonCache { - protected: - // Registers live after the cache, excluding output registers. The initial - // value of these registers must be preserved by the cache. - RegisterSet liveRegs_; - - Register object_; - PropertyName *name_; - TypedOrValueRegister output_; - bool allowGetters_ : 1; - bool hasArrayLengthStub_ : 1; - bool hasTypedArrayLengthStub_ : 1; - public: - GetPropertyIC(RegisterSet liveRegs, - Register object, PropertyName *name, - TypedOrValueRegister output, - bool allowGetters) - : liveRegs_(liveRegs), - object_(object), - name_(name), - output_(output), - allowGetters_(allowGetters), - hasArrayLengthStub_(false), - hasTypedArrayLengthStub_(false) + IonCacheGetProperty(CodeOffsetJump initialJump, + CodeOffsetLabel rejoinLabel, + CodeOffsetLabel cacheLabel, + RegisterSet liveRegs, + Register object, PropertyName *name, + TypedOrValueRegister output, + bool allowGetters) { + init(GetProperty, liveRegs, initialJump, rejoinLabel, cacheLabel); + u.getprop.object = object; + u.getprop.name = name; + u.getprop.output.data() = output; + u.getprop.allowGetters = allowGetters; + u.getprop.hasArrayLengthStub = false; + u.getprop.hasTypedArrayLengthStub = false; } - CACHE_HEADER(GetProperty) - - Register object() const { - return object_; - } - PropertyName *name() const { - return name_; - } - TypedOrValueRegister output() const { - return output_; - } - bool allowGetters() const { - return allowGetters_; - } - bool hasArrayLengthStub() const { - return hasArrayLengthStub_; - } - bool hasTypedArrayLengthStub() const { - return hasTypedArrayLengthStub_; - } + Register object() const { return u.getprop.object; } + PropertyName *name() const { return u.getprop.name; } + TypedOrValueRegister output() const { return u.getprop.output.data(); } + bool allowGetters() const { return u.getprop.allowGetters; } + bool hasArrayLengthStub() const { return u.getprop.hasArrayLengthStub; } + bool hasTypedArrayLengthStub() const { return u.getprop.hasTypedArrayLengthStub; } bool attachReadSlot(JSContext *cx, IonScript *ion, JSObject *obj, JSObject *holder, HandleShape shape); @@ -348,237 +304,196 @@ class GetPropertyIC : public IonCache const SafepointIndex *safepointIndex, void *returnAddr); bool attachArrayLength(JSContext *cx, IonScript *ion, JSObject *obj); bool attachTypedArrayLength(JSContext *cx, IonScript *ion, JSObject *obj); - - static bool update(JSContext *cx, size_t cacheIndex, HandleObject obj, MutableHandleValue vp); }; -class SetPropertyIC : public IonCache +class IonCacheSetProperty : public IonCache { - protected: - // Registers live after the cache, excluding output registers. The initial - // value of these registers must be preserved by the cache. - RegisterSet liveRegs_; - - Register object_; - PropertyName *name_; - ConstantOrRegister value_; - bool isSetName_; - bool strict_; - public: - SetPropertyIC(RegisterSet liveRegs, Register object, PropertyName *name, - ConstantOrRegister value, bool isSetName, bool strict) - : liveRegs_(liveRegs), - object_(object), - name_(name), - value_(value), - isSetName_(isSetName), - strict_(strict) + IonCacheSetProperty(CodeOffsetJump initialJump, + CodeOffsetLabel rejoinLabel, + CodeOffsetLabel cacheLabel, + RegisterSet liveRegs, + Register object, PropertyName *name, + ConstantOrRegister value, + bool strict) { + init(SetProperty, liveRegs, initialJump, rejoinLabel, cacheLabel); + u.setprop.object = object; + u.setprop.name = name; + u.setprop.value.data() = value; + u.setprop.strict = strict; } - CACHE_HEADER(SetProperty) - - Register object() const { - return object_; - } - PropertyName *name() const { - return name_; - } - ConstantOrRegister value() const { - return value_; - } - bool isSetName() const { - return isSetName_; - } - bool strict() const { - return strict_; - } + Register object() const { return u.setprop.object; } + PropertyName *name() const { return u.setprop.name; } + ConstantOrRegister value() const { return u.setprop.value.data(); } + bool strict() const { return u.setprop.strict; } bool attachNativeExisting(JSContext *cx, IonScript *ion, HandleObject obj, HandleShape shape); bool attachSetterCall(JSContext *cx, IonScript *ion, HandleObject obj, HandleObject holder, HandleShape shape, void *returnAddr); bool attachNativeAdding(JSContext *cx, IonScript *ion, JSObject *obj, HandleShape oldshape, HandleShape newshape, HandleShape propshape); - - static bool - update(JSContext *cx, size_t cacheIndex, HandleObject obj, HandleValue value); }; -class GetElementIC : public IonCache +class IonCacheGetElement : public IonCache { - protected: - Register object_; - ConstantOrRegister index_; - TypedOrValueRegister output_; - bool monitoredResult_ : 1; - bool hasDenseStub_ : 1; - public: - GetElementIC(Register object, ConstantOrRegister index, - TypedOrValueRegister output, bool monitoredResult) - : object_(object), - index_(index), - output_(output), - monitoredResult_(monitoredResult), - hasDenseStub_(false) + IonCacheGetElement(CodeOffsetJump initialJump, + CodeOffsetLabel rejoinLabel, + CodeOffsetLabel cacheLabel, + RegisterSet liveRegs, + Register object, ConstantOrRegister index, + TypedOrValueRegister output, bool monitoredResult) { + init(GetElement, liveRegs, initialJump, rejoinLabel, cacheLabel); + u.getelem.object = object; + u.getelem.index.data() = index; + u.getelem.output.data() = output; + u.getelem.monitoredResult = monitoredResult; + u.getelem.hasDenseStub = false; } - CACHE_HEADER(GetElement) - Register object() const { - return object_; + return u.getelem.object; } ConstantOrRegister index() const { - return index_; + return u.getelem.index.data(); } TypedOrValueRegister output() const { - return output_; + return u.getelem.output.data(); } bool monitoredResult() const { - return monitoredResult_; + return u.getelem.monitoredResult; } bool hasDenseStub() const { - return hasDenseStub_; + return u.getelem.hasDenseStub; } void setHasDenseStub() { JS_ASSERT(!hasDenseStub()); - hasDenseStub_ = true; + u.getelem.hasDenseStub = true; } bool attachGetProp(JSContext *cx, IonScript *ion, HandleObject obj, const Value &idval, HandlePropertyName name); bool attachDenseElement(JSContext *cx, IonScript *ion, JSObject *obj, const Value &idval); - bool attachTypedArrayElement(JSContext *cx, IonScript *ion, JSObject *obj, const Value &idval); - - static bool - update(JSContext *cx, size_t cacheIndex, HandleObject obj, HandleValue idval, - MutableHandleValue vp); }; -class BindNameIC : public IonCache +class IonCacheBindName : public IonCache { - protected: - Register scopeChain_; - PropertyName *name_; - Register output_; - public: - BindNameIC(Register scopeChain, PropertyName *name, Register output) - : scopeChain_(scopeChain), - name_(name), - output_(output) + IonCacheBindName(CodeOffsetJump initialJump, + CodeOffsetLabel rejoinLabel, + CodeOffsetLabel cacheLabel, + RegisterSet liveRegs, + Register scopeChain, PropertyName *name, + Register output) { + init(BindName, liveRegs, initialJump, rejoinLabel, cacheLabel); + u.bindname.scopeChain = scopeChain; + u.bindname.name = name; + u.bindname.output = output; } - CACHE_HEADER(BindName) - Register scopeChainReg() const { - return scopeChain_; + return u.bindname.scopeChain; } HandlePropertyName name() const { - return HandlePropertyName::fromMarkedLocation(&name_); + return HandlePropertyName::fromMarkedLocation(&u.bindname.name); } Register outputReg() const { - return output_; + return u.bindname.output; } bool attachGlobal(JSContext *cx, IonScript *ion, JSObject *scopeChain); bool attachNonGlobal(JSContext *cx, IonScript *ion, JSObject *scopeChain, JSObject *holder); - - static JSObject * - update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain); }; -class NameIC : public IonCache +class IonCacheName : public IonCache { - protected: - bool typeOf_; - Register scopeChain_; - PropertyName *name_; - TypedOrValueRegister output_; - public: - NameIC(bool typeOf, - Register scopeChain, PropertyName *name, - TypedOrValueRegister output) - : typeOf_(typeOf), - scopeChain_(scopeChain), - name_(name), - output_(output) + IonCacheName(Kind kind, + CodeOffsetJump initialJump, + CodeOffsetLabel rejoinLabel, + CodeOffsetLabel cacheLabel, + RegisterSet liveRegs, + Register scopeChain, PropertyName *name, + TypedOrValueRegister output) { + init(kind, liveRegs, initialJump, rejoinLabel, cacheLabel); + u.name.scopeChain = scopeChain; + u.name.name = name; + u.name.output.data() = output; } - CACHE_HEADER(Name) - Register scopeChainReg() const { - return scopeChain_; + return u.name.scopeChain; } HandlePropertyName name() const { - return HandlePropertyName::fromMarkedLocation(&name_); + return HandlePropertyName::fromMarkedLocation(&u.name.name); } TypedOrValueRegister outputReg() const { - return output_; + return u.name.output.data(); } bool isTypeOf() const { - return typeOf_; + return kind_ == NameTypeOf; } bool attach(JSContext *cx, IonScript *ion, HandleObject scopeChain, HandleObject obj, HandleShape shape); - - static bool - update(JSContext *cx, size_t cacheIndex, HandleObject scopeChain, MutableHandleValue vp); }; -class CallsiteCloneIC : public IonCache +class IonCacheCallsiteClone : public IonCache { - protected: - Register callee_; - Register output_; - JSScript *callScript_; - jsbytecode *callPc_; - public: - CallsiteCloneIC(Register callee, JSScript *callScript, jsbytecode *callPc, Register output) - : callee_(callee), - output_(output), - callScript_(callScript), - callPc_(callPc) + IonCacheCallsiteClone(CodeOffsetJump initialJump, + CodeOffsetLabel rejoinLabel, + CodeOffsetLabel cacheLabel, + RegisterSet liveRegs, + Register callee, JSScript *callScript, jsbytecode *callPc, + Register output) { + init(CallsiteClone, liveRegs, initialJump, rejoinLabel, cacheLabel); + u.callsiteclone.callee = callee; + u.callsiteclone.callScript = callScript; + u.callsiteclone.callPc = callPc; + u.callsiteclone.output = output; } - CACHE_HEADER(CallsiteClone) - Register calleeReg() const { - return callee_; + return u.callsiteclone.callee; } HandleScript callScript() const { - return HandleScript::fromMarkedLocation(&callScript_); + return HandleScript::fromMarkedLocation(&u.callsiteclone.callScript); } jsbytecode *callPc() const { - return callPc_; + return u.callsiteclone.callPc; } Register outputReg() const { - return output_; + return u.callsiteclone.output; } bool attach(JSContext *cx, IonScript *ion, HandleFunction original, HandleFunction clone); - - static JSObject *update(JSContext *cx, size_t cacheIndex, HandleObject callee); }; -#undef CACHE_HEADER +bool +GetPropertyCache(JSContext *cx, size_t cacheIndex, HandleObject obj, MutableHandleValue vp); -// Implement cache casts now that the compiler can see the inheritance. -#define CACHE_CASTS(ickind) \ - ickind##IC &IonCache::to##ickind() \ - { \ - JS_ASSERT(is##ickind()); \ - return *static_cast(this); \ - } -IONCACHE_KIND_LIST(CACHE_CASTS) -#undef OPCODE_CASTS +bool +SetPropertyCache(JSContext *cx, size_t cacheIndex, HandleObject obj, HandleValue value, + bool isSetName); + +bool +GetElementCache(JSContext *cx, size_t cacheIndex, HandleObject obj, HandleValue idval, + MutableHandleValue vp); + +JSObject * +BindNameCache(JSContext *cx, size_t cacheIndex, HandleObject scopeChain); + +bool +GetNameCache(JSContext *cx, size_t cacheIndex, HandleObject scopeChain, MutableHandleValue vp); + +JSObject * +CallsiteCloneCache(JSContext *cx, size_t cacheIndex, HandleObject callee); } // namespace ion } // namespace js diff --git a/js/src/ion/IonCode.h b/js/src/ion/IonCode.h index 04e71667de3..2112d12a289 100644 --- a/js/src/ion/IonCode.h +++ b/js/src/ion/IonCode.h @@ -171,25 +171,22 @@ struct IonScript // Flag set when we bailout, to avoid frequent bailouts. bool bailoutExpected_; - // Any kind of data needed by the runtime, these can be either cache - // information or profiling info. - uint32_t runtimeData_; - uint32_t runtimeSize_; + // Offset from the start of the code buffer to its snapshot buffer. + uint32_t snapshots_; + uint32_t snapshotsSize_; - // State for polymorphic caches in the compiled code. All caches are stored - // in the runtimeData buffer and indexed by the cacheIndex which give a - // relative offset in the runtimeData array. - uint32_t cacheIndex_; - uint32_t cacheEntries_; + // Table mapping bailout IDs to snapshot offsets. + uint32_t bailoutTable_; + uint32_t bailoutEntries_; + + // Constant table for constants stored in snapshots. + uint32_t constantTable_; + uint32_t constantEntries_; // Map code displacement to safepoint / OSI-patch-delta. uint32_t safepointIndexOffset_; uint32_t safepointIndexEntries_; - // Offset to and length of the safepoint table in bytes. - uint32_t safepointsStart_; - uint32_t safepointsSize_; - // Number of STACK_SLOT_SIZE-length slots this function reserves on the // stack. uint32_t frameSlots_; @@ -198,21 +195,17 @@ struct IonScript // with the frame prefix to get a valid IonJSFrameLayout. uint32_t frameSize_; - // Table mapping bailout IDs to snapshot offsets. - uint32_t bailoutTable_; - uint32_t bailoutEntries_; - // Map OSI-point displacement to snapshot. uint32_t osiIndexOffset_; uint32_t osiIndexEntries_; - // Offset from the start of the code buffer to its snapshot buffer. - uint32_t snapshots_; - uint32_t snapshotsSize_; + // State for polymorphic caches in the compiled code. + uint32_t cacheList_; + uint32_t cacheEntries_; - // Constant table for constants stored in snapshots. - uint32_t constantTable_; - uint32_t constantEntries_; + // Offset to and length of the safepoint table in bytes. + uint32_t safepointsStart_; + uint32_t safepointsSize_; // List of compiled/inlined JSScript's. uint32_t scriptList_; @@ -233,50 +226,39 @@ struct IonScript // Number of references from invalidation records. size_t refcount_; - // Identifier of the compilation which produced this code. types::RecompileInfo recompileInfo_; - private: - inline uint8_t *bottomBuffer() { - return reinterpret_cast(this); - } - inline const uint8_t *bottomBuffer() const { - return reinterpret_cast(this); - } - public: // Number of times this function has tried to call a non-IM compileable function uint32_t slowCallCount; SnapshotOffset *bailoutTable() { - return (SnapshotOffset *) &bottomBuffer()[bailoutTable_]; + return (SnapshotOffset *)(reinterpret_cast(this) + bailoutTable_); } HeapValue *constants() { - return (HeapValue *) &bottomBuffer()[constantTable_]; + return (HeapValue *)(reinterpret_cast(this) + constantTable_); } const SafepointIndex *safepointIndices() const { return const_cast(this)->safepointIndices(); } SafepointIndex *safepointIndices() { - return (SafepointIndex *) &bottomBuffer()[safepointIndexOffset_]; + return (SafepointIndex *)(reinterpret_cast(this) + safepointIndexOffset_); } const OsiIndex *osiIndices() const { return const_cast(this)->osiIndices(); } OsiIndex *osiIndices() { - return (OsiIndex *) &bottomBuffer()[osiIndexOffset_]; + return (OsiIndex *)(reinterpret_cast(this) + osiIndexOffset_); } - uint32_t *cacheIndex() { - return (uint32_t *) &bottomBuffer()[cacheIndex_]; - } - uint8_t *runtimeData() { - return &bottomBuffer()[runtimeData_]; + IonCache *cacheList() { + return (IonCache *)(reinterpret_cast(this) + cacheList_); } JSScript **scriptList() const { - return (JSScript **) &bottomBuffer()[scriptList_]; + return (JSScript **)(reinterpret_cast(this) + scriptList_); } JSScript **parallelInvalidatedScriptList() { - return (JSScript **) &bottomBuffer()[parallelInvalidatedScriptList_]; + return (JSScript **)(reinterpret_cast(this) + + parallelInvalidatedScriptList_); } private: @@ -289,8 +271,7 @@ struct IonScript static IonScript *New(JSContext *cx, uint32_t frameLocals, uint32_t frameSize, size_t snapshotsSize, size_t snapshotEntries, size_t constants, size_t safepointIndexEntries, size_t osiIndexEntries, - size_t cacheEntries, size_t runtimeSize, - size_t safepointsSize, size_t scriptEntries, + size_t cacheEntries, size_t safepointsSize, size_t scriptEntries, size_t parallelInvalidatedScriptEntries); static void Trace(JSTracer *trc, IonScript *script); static void Destroy(FreeOp *fop, IonScript *script); @@ -411,18 +392,10 @@ struct IonScript } const OsiIndex *getOsiIndex(uint32_t disp) const; const OsiIndex *getOsiIndex(uint8_t *retAddr) const; - inline IonCache &getCache(uint32_t index) { - JS_ASSERT(index < cacheEntries_); - uint32_t offset = cacheIndex()[index]; - JS_ASSERT(offset < runtimeSize_); - return *(IonCache *) &runtimeData()[offset]; - } + inline IonCache &getCache(size_t index); size_t numCaches() const { return cacheEntries_; } - size_t runtimeSize() const { - return runtimeSize_; - } void toggleBarriers(bool enabled); void purgeCaches(JSCompartment *c); void copySnapshots(const SnapshotWriter *writer); @@ -430,8 +403,7 @@ struct IonScript void copyConstants(const HeapValue *vp); void copySafepointIndices(const SafepointIndex *firstSafepointIndex, MacroAssembler &masm); void copyOsiIndices(const OsiIndex *firstOsiIndex, MacroAssembler &masm); - void copyRuntimeData(const uint8_t *data); - void copyCacheEntries(const uint32_t *caches, MacroAssembler &masm); + void copyCacheEntries(const IonCache *caches, MacroAssembler &masm); void copySafepoints(const SafepointWriter *writer); void copyScriptEntries(JSScript **scripts); void zeroParallelInvalidatedScripts(); diff --git a/js/src/ion/IonFrames.cpp b/js/src/ion/IonFrames.cpp index bdd8e71b46e..8cd4d268ae1 100644 --- a/js/src/ion/IonFrames.cpp +++ b/js/src/ion/IonFrames.cpp @@ -570,7 +570,7 @@ MarkIonExitFrame(JSTracer *trc, const IonFrameIterator &frame) gc::MarkValueRoot(trc, oolgetter->thisp(), "ion-ool-getter-this"); return; } - + if (frame.isOOLPropertyOp()) { IonOOLPropertyOpExitFrameLayout *oolgetter = frame.exitFrame()->oolPropertyOpExit(); gc::MarkIonCodeRoot(trc, oolgetter->stubCode(), "ion-ool-property-op-code"); diff --git a/js/src/ion/IonMacroAssembler.h b/js/src/ion/IonMacroAssembler.h index 906e8644e24..d57722298b4 100644 --- a/js/src/ion/IonMacroAssembler.h +++ b/js/src/ion/IonMacroAssembler.h @@ -113,7 +113,7 @@ class MacroAssembler : public MacroAssemblerSpecific return size(); } - void propagateOOM(bool success) { + void reportMemory(bool success) { enoughMemory_ &= success; } bool oom() const { diff --git a/js/src/ion/shared/Assembler-shared.h b/js/src/ion/shared/Assembler-shared.h index c144385595e..a85d20cc41e 100644 --- a/js/src/ion/shared/Assembler-shared.h +++ b/js/src/ion/shared/Assembler-shared.h @@ -396,33 +396,14 @@ class CodeOffsetLabel class CodeLocationJump { uint8_t *raw_; -#ifdef DEBUG - bool absolute_; - void setAbsolute() { - absolute_ = true; - } - void setRelative() { - absolute_ = false; - } -#else - void setAbsolute() const { - } - void setRelative() const { - } -#endif + mozilla::DebugOnly absolute_; #ifdef JS_SMALL_BRANCH uint8_t *jumpTableEntry_; #endif public: - CodeLocationJump() { - raw_ = (uint8_t *) 0xdeadc0de; - setAbsolute(); -#ifdef JS_SMALL_BRANCH - jumpTableEntry_ = (uint8_t *) 0xdeadab1e; -#endif - } + CodeLocationJump() {} CodeLocationJump(IonCode *code, CodeOffsetJump base) { *this = base; repoint(code); @@ -430,7 +411,7 @@ class CodeLocationJump void operator = (CodeOffsetJump base) { raw_ = (uint8_t *) base.offset(); - setRelative(); + absolute_ = false; #ifdef JS_SMALL_BRANCH jumpTableEntry_ = (uint8_t *) base.jumpTableIndex(); #endif @@ -439,11 +420,11 @@ class CodeLocationJump void repoint(IonCode *code, MacroAssembler* masm = NULL); uint8_t *raw() const { - JS_ASSERT(absolute_ && raw_ != (uint8_t *) 0xdeadc0de); + JS_ASSERT(absolute_); return raw_; } uint8_t *offset() const { - JS_ASSERT(!absolute_ && raw_ != (uint8_t *) 0xdeadc0de); + JS_ASSERT(!absolute_); return raw_; } @@ -458,42 +439,26 @@ class CodeLocationJump class CodeLocationLabel { uint8_t *raw_; -#ifdef DEBUG - bool absolute_; - void setAbsolute() { - absolute_ = true; - } - void setRelative() { - absolute_ = false; - } -#else - void setAbsolute() const { - } - void setRelative() const { - } -#endif + mozilla::DebugOnly absolute_; public: - CodeLocationLabel() { - raw_ = (uint8_t *) 0xdeadc0de; - setAbsolute(); - } + CodeLocationLabel() {} CodeLocationLabel(IonCode *code, CodeOffsetLabel base) { *this = base; repoint(code); } CodeLocationLabel(IonCode *code) { raw_ = code->raw(); - setAbsolute(); + absolute_ = true; } CodeLocationLabel(uint8_t *raw) { raw_ = raw; - setAbsolute(); + absolute_ = true; } void operator = (CodeOffsetLabel base) { raw_ = (uint8_t *)base.offset(); - setRelative(); + absolute_ = false; } ptrdiff_t operator - (const CodeLocationLabel &other) { return raw_ - other.raw_; @@ -502,11 +467,11 @@ class CodeLocationLabel void repoint(IonCode *code, MacroAssembler *masm = NULL); uint8_t *raw() { - JS_ASSERT(absolute_ && raw_ != (uint8_t *) 0xdeadc0de); + JS_ASSERT(absolute_); return raw_; } uint8_t *offset() { - JS_ASSERT(!absolute_ && raw_ != (uint8_t *) 0xdeadc0de); + JS_ASSERT(!absolute_); return raw_; } }; diff --git a/js/src/ion/shared/CodeGenerator-shared.h b/js/src/ion/shared/CodeGenerator-shared.h index a3d8a61c8d1..ab98cadac8a 100644 --- a/js/src/ion/shared/CodeGenerator-shared.h +++ b/js/src/ion/shared/CodeGenerator-shared.h @@ -25,12 +25,10 @@ namespace ion { class OutOfLineCode; class CodeGenerator; class MacroAssembler; -class IonCache; class OutOfLineParallelAbort; template class OutOfLineCallVM; - class OutOfLineTruncateSlow; class CodeGeneratorShared : public LInstructionVisitor @@ -62,11 +60,8 @@ class CodeGeneratorShared : public LInstructionVisitor // Mapping from bailout table ID to an offset in the snapshot buffer. js::Vector bailouts_; - // Allocated data space needed at runtime. - js::Vector runtimeData_; - // Vector of information about generated polymorphic inline caches. - js::Vector cacheList_; + js::Vector cacheList_; // List of stack slots that have been pushed as arguments to an MCall. js::Vector pushedArgumentSlots_; @@ -159,35 +154,10 @@ class CodeGeneratorShared : public LInstructionVisitor } protected: - // Ensure the cache is an IonCache while expecting the size of the derived - // class. - size_t allocateCache(const IonCache &, size_t size) { - size_t dataOffset = allocateData(size); + + size_t allocateCache(const IonCache &cache) { size_t index = cacheList_.length(); - masm.propagateOOM(cacheList_.append(dataOffset)); - return index; - } - - // This is needed by addCache to update the cache with the jump - // informations provided by the out-of-line path. - IonCache *getCache(size_t index) { - return reinterpret_cast(&runtimeData_[cacheList_[index]]); - } - - protected: - - size_t allocateData(size_t size) { - JS_ASSERT(size % sizeof(void *) == 0); - size_t dataOffset = runtimeData_.length(); - masm.propagateOOM(runtimeData_.appendN(0, size)); - return dataOffset; - } - - template - inline size_t allocateCache(const T &cache) { - size_t index = allocateCache(cache, sizeof(mozilla::AlignedStorage2)); - // Use the copy constructor on the allocated space. - new (&runtimeData_[cacheList_.back()]) T(cache); + masm.reportMemory(cacheList_.append(cache)); return index; } @@ -307,8 +277,6 @@ class CodeGeneratorShared : public LInstructionVisitor inline OutOfLineCode *oolCallVM(const VMFunction &fun, LInstruction *ins, const ArgSeq &args, const StoreOutputTo &out); - bool addCache(LInstruction *lir, size_t cacheIndex); - protected: bool addOutOfLineCode(OutOfLineCode *code); bool generateOutOfLineCode(); @@ -405,7 +373,7 @@ class OutOfLineCodeBase : public OutOfLineCode // ArgSeq store arguments for OutOfLineCallVM. // -// OutOfLineCallVM are created with "oolCallVM" function. The third argument of +// OutOfLineCallVM are created with "oolCallVM" function. The last argument of // this function is an instance of a class which provides a "generate" function // to call the "pushArg" needed by the VMFunction call. The list of argument // can be created by using the ArgList function which create an empty list of @@ -444,7 +412,6 @@ class ArgSeq : public SeqType } }; -// Mark the end of an argument list. template <> class ArgSeq { diff --git a/js/src/jit-test/tests/ion/getPropertyCacheOverflow.js b/js/src/jit-test/tests/ion/getPropertyCacheOverflow.js deleted file mode 100644 index 9e87bc0c5e3..00000000000 --- a/js/src/jit-test/tests/ion/getPropertyCacheOverflow.js +++ /dev/null @@ -1,37 +0,0 @@ -// This test is made to be effective with --no-jm. - -var list = [ - { entry00: 0, length: 1 }, - { entry01: 0, length: 1 }, - { entry02: 0, length: 1 }, - { entry03: 0, length: 1 }, - { entry04: 0, length: 1 }, - { entry05: 0, length: 1 }, - { entry06: 0, length: 1 }, - { entry07: 0, length: 1 }, - { entry08: 0, length: 1 }, - { entry09: 0, length: 1 }, - { entry10: 0, length: 1 }, - { entry11: 0, length: 1 }, - { entry12: 0, length: 1 }, - { entry13: 0, length: 1 }, - { entry14: 0, length: 1 }, - { entry15: 0, length: 1 }, - { entry16: 0, length: 1 }, // cause an overflow. - { entry17: 0, length: 1 }, - [0], - (new Uint8Array(new ArrayBuffer(1))) -]; - -function f(obj) { - return obj.length; -} - -// Cook the f function on the top of the list to make sure we do not register -// our test cases. -for (var i = 0; i < 100; i++) - f(list[i % 10]); - -// Register & check stubs. -for (var i = 0; i < 40; i++) - assertEq(f(list[i % 20]), 1); diff --git a/js/src/jsinterpinlines.h b/js/src/jsinterpinlines.h index c2900a4b29d..c09a618e0fa 100644 --- a/js/src/jsinterpinlines.h +++ b/js/src/jsinterpinlines.h @@ -748,7 +748,7 @@ GetObjectElementOperation(JSContext *cx, JSOp op, JSObject *objArg, bool wasObje uint32_t index; if (IsDefinitelyIndex(rref, &index)) { - if (analyze && !objArg->isNative() && !objArg->isTypedArray()) { + if (analyze && !objArg->isNative()) { JSScript *script = NULL; jsbytecode *pc = NULL; types::TypeScript::GetPcScript(cx, &script, &pc); @@ -775,7 +775,7 @@ GetObjectElementOperation(JSContext *cx, JSOp op, JSObject *objArg, bool wasObje if (script->hasAnalysis()) { script->analysis()->getCode(pc).getStringElement = true; - if (!objArg->isArray() && !objArg->isNative() && !objArg->isTypedArray()) + if (!objArg->isArray() && !objArg->isNative()) script->analysis()->getCode(pc).nonNativeGetElement = true; } }