diff --git a/js/src/methodjit/BaseAssembler.h b/js/src/methodjit/BaseAssembler.h index bd230d6b0b4..04f3d9c90b5 100644 --- a/js/src/methodjit/BaseAssembler.h +++ b/js/src/methodjit/BaseAssembler.h @@ -50,6 +50,8 @@ #include "methodjit/MethodJIT.h" #include "methodjit/MachineRegs.h" #include "CodeGenIncludes.h" +#include "jsobjinlines.h" +#include "jsscopeinlines.h" namespace js { namespace mjit { @@ -209,9 +211,9 @@ static const JSC::MacroAssembler::RegisterID JSParamReg_Argc = JSC::ARMRegiste load32(Address(obj, offsetof(JSObject, objShape)), shape); } - Jump guardShape(RegisterID obj, uint32 shape) { - return branch32(NotEqual, Address(obj, offsetof(JSObject, objShape)), - Imm32(shape)); + Jump guardShape(RegisterID objReg, JSObject *obj) { + return branch32(NotEqual, Address(objReg, offsetof(JSObject, objShape)), + Imm32(obj->shape())); } Jump testFunction(Condition cond, RegisterID fun) { @@ -427,6 +429,24 @@ static const JSC::MacroAssembler::RegisterID JSParamReg_Argc = JSC::ARMRegiste else move(remat.reg(), reg); } + + void loadDynamicSlot(RegisterID objReg, uint32 slot, + RegisterID typeReg, RegisterID dataReg) { + loadPtr(Address(objReg, offsetof(JSObject, slots)), dataReg); + loadValueAsComponents(Address(dataReg, slot * sizeof(Value)), typeReg, dataReg); + } + + void loadObjProp(JSObject *obj, RegisterID objReg, + const js::Shape *shape, + RegisterID typeReg, RegisterID dataReg) + { + if (shape->isMethod()) + loadValueAsComponents(ObjectValue(shape->methodObject()), typeReg, dataReg); + else if (obj->hasSlotsArray()) + loadDynamicSlot(objReg, shape->slot, typeReg, dataReg); + else + loadInlineSlot(objReg, shape->slot, typeReg, dataReg); + } }; /* Return f if the script is strict mode code, f otherwise. */ diff --git a/js/src/methodjit/BaseCompiler.h b/js/src/methodjit/BaseCompiler.h index ad1ff5ad804..d4ccce0ca1e 100644 --- a/js/src/methodjit/BaseCompiler.h +++ b/js/src/methodjit/BaseCompiler.h @@ -48,19 +48,7 @@ namespace js { namespace mjit { -class BaseCompiler -{ - protected: - JSContext *cx; - - public: - BaseCompiler() : cx(NULL) - { } - - BaseCompiler(JSContext *cx) : cx(cx) - { } - - protected: +struct MacroAssemblerTypedefs { typedef JSC::MacroAssembler::Label Label; typedef JSC::MacroAssembler::Imm32 Imm32; typedef JSC::MacroAssembler::ImmPtr ImmPtr; @@ -77,11 +65,25 @@ class BaseCompiler typedef JSC::MacroAssembler::DataLabel32 DataLabel32; typedef JSC::FunctionPtr FunctionPtr; typedef JSC::RepatchBuffer RepatchBuffer; - typedef JSC::CodeBlock CodeBlock; typedef JSC::CodeLocationLabel CodeLocationLabel; - typedef JSC::JITCode JITCode; + typedef JSC::CodeLocationCall CodeLocationCall; typedef JSC::ReturnAddressPtr ReturnAddressPtr; typedef JSC::MacroAssemblerCodePtr MacroAssemblerCodePtr; +}; + +class BaseCompiler : public MacroAssemblerTypedefs +{ + protected: + JSContext *cx; + + public: + BaseCompiler() : cx(NULL) + { } + + BaseCompiler(JSContext *cx) : cx(cx) + { } + + protected: JSC::ExecutablePool * getExecPool(size_t size) { @@ -127,6 +129,12 @@ class LinkerHelper : public JSC::LinkBuffer } return ep; } + + void maybeLink(MaybeJump jump, JSC::CodeLocationLabel label) { + if (!jump.isSet()) + return; + link(jump.get(), label); + } }; } /* namespace js */ diff --git a/js/src/methodjit/Compiler.cpp b/js/src/methodjit/Compiler.cpp index 38430a33672..0a895bbc838 100644 --- a/js/src/methodjit/Compiler.cpp +++ b/js/src/methodjit/Compiler.cpp @@ -103,6 +103,7 @@ mjit::Compiler::Compiler(JSContext *cx, JSStackFrame *fp) #endif #if defined JS_POLYIC pics(ContextAllocPolicy(cx)), + getElemICs(ContextAllocPolicy(cx)), #endif callPatches(ContextAllocPolicy(cx)), callSites(ContextAllocPolicy(cx)), @@ -390,6 +391,7 @@ mjit::Compiler::finishThisUp(JITScript **jitp) #endif #if defined JS_POLYIC sizeof(ic::PICInfo) * pics.length() + + sizeof(ic::GetElementIC) * getElemICs.length() + #endif sizeof(CallSite) * callSites.length(); @@ -586,6 +588,38 @@ mjit::Compiler::finishThisUp(JITScript **jitp) } #if defined JS_POLYIC + jit->nGetElems = getElemICs.length(); + if (getElemICs.length()) { + jit->getElems = (ic::GetElementIC *)cursor; + cursor += sizeof(ic::GetElementIC) * getElemICs.length(); + } else { + jit->getElems = NULL; + } + + for (size_t i = 0; i < getElemICs.length(); i++) { + ic::GetElementIC &to = jit->getElems[i]; + GetElementICInfo &from = getElemICs[i]; + to.init(); + from.copyTo(to, fullCode, stubCode); + + to.typeReg = from.typeReg; + to.objReg = from.objReg; + to.idRemat = from.id; + + if (from.typeGuard.isSet()) { + int inlineTypeGuard = fullCode.locationOf(from.typeGuard.get()) - + fullCode.locationOf(from.fastPathStart); + to.inlineTypeGuard = inlineTypeGuard; + JS_ASSERT(to.inlineTypeGuard == inlineTypeGuard); + } + int inlineClaspGuard = fullCode.locationOf(from.claspGuard) - + fullCode.locationOf(from.fastPathStart); + to.inlineClaspGuard = inlineClaspGuard; + JS_ASSERT(to.inlineClaspGuard == inlineClaspGuard); + + stubCode.patch(from.paramAddr, &to); + } + jit->nPICs = pics.length(); if (pics.length()) { jit->pics = (ic::PICInfo *)cursor; @@ -596,11 +630,9 @@ mjit::Compiler::finishThisUp(JITScript **jitp) if (ic::PICInfo *scriptPICs = jit->pics) { for (size_t i = 0; i < pics.length(); i++) { + scriptPICs[i].init(); + pics[i].copyTo(scriptPICs[i], fullCode, stubCode); pics[i].copySimpleMembersTo(scriptPICs[i]); - scriptPICs[i].fastPathStart = fullCode.locationOf(pics[i].fastPathStart); - scriptPICs[i].fastPathRejoin = fullCode.locationOf(pics[i].fastPathRejoin); - scriptPICs[i].slowPathStart = stubCode.locationOf(pics[i].slowPathStart); - scriptPICs[i].slowPathCall = stubCode.locationOf(pics[i].slowPathCall); scriptPICs[i].shapeGuard = masm.distanceOf(pics[i].shapeGuard) - masm.distanceOf(pics[i].fastPathStart); JS_ASSERT(scriptPICs[i].shapeGuard == masm.distanceOf(pics[i].shapeGuard) - @@ -623,8 +655,6 @@ mjit::Compiler::finishThisUp(JITScript **jitp) scriptPICs[i].u.get.typeCheckOffset = distance; } } - new (&scriptPICs[i].execPools) ic::PICInfo::ExecPoolVector(SystemAllocPolicy()); - scriptPICs[i].reset(); stubCode.patch(pics[i].paramAddr, &scriptPICs[i]); } } @@ -2593,9 +2623,9 @@ mjit::Compiler::passMICAddress(MICGenInfo &mic) #if defined JS_POLYIC void -mjit::Compiler::passPICAddress(PICGenInfo &pic) +mjit::Compiler::passICAddress(BaseICInfo *ic) { - pic.paramAddr = stubcc.masm.moveWithPatch(ImmPtr(NULL), Registers::ArgReg1); + ic->paramAddr = stubcc.masm.moveWithPatch(ImmPtr(NULL), Registers::ArgReg1); } bool @@ -2623,7 +2653,7 @@ mjit::Compiler::jsop_getprop(JSAtom *atom, bool doTypeCheck, bool usePropCache) shapeReg = frame.allocReg(); } - PICGenInfo pic(ic::PICInfo::GET, usePropCache); + PICGenInfo pic(ic::PICInfo::GET, JSOp(*PC), usePropCache); /* Guard that the type is an object. */ Jump typeCheck; @@ -2654,7 +2684,6 @@ mjit::Compiler::jsop_getprop(JSAtom *atom, bool doTypeCheck, bool usePropCache) pic.shapeReg = shapeReg; pic.atom = atom; - pic.objRemat = frame.dataRematInfo(top); /* Guard on shape. */ masm.loadShape(objReg, shapeReg); @@ -2669,7 +2698,7 @@ mjit::Compiler::jsop_getprop(JSAtom *atom, bool doTypeCheck, bool usePropCache) pic.slowPathStart = stubcc.linkExit(j, Uses(1)); stubcc.leave(); - passPICAddress(pic); + passICAddress(&pic); pic.slowPathCall = stubcc.call(ic::GetProp); /* Load dslots. */ @@ -2727,113 +2756,6 @@ mjit::Compiler::jsop_getprop(JSAtom *atom, bool doTypeCheck, bool usePropCache) return true; } -#ifdef JS_POLYIC -bool -mjit::Compiler::jsop_getelem_pic(FrameEntry *obj, FrameEntry *id, RegisterID objReg, - RegisterID idReg, RegisterID shapeReg) -{ - PICGenInfo pic(ic::PICInfo::GETELEM, true); - - pic.objRemat = frame.dataRematInfo(obj); - pic.idRemat = frame.dataRematInfo(id); - pic.shapeReg = shapeReg; - pic.hasTypeCheck = false; - - pic.fastPathStart = masm.label(); - - /* Guard on shape. */ - masm.loadShape(objReg, shapeReg); - pic.shapeGuard = masm.label(); - - DataLabel32 inlineShapeOffsetLabel; - Jump jmpShapeGuard = masm.branch32WithPatch(Assembler::NotEqual, shapeReg, - Imm32(int32(JSObjectMap::INVALID_SHAPE)), - inlineShapeOffsetLabel); - DBGLABEL(dbgInlineShapeJump); - - /* Guard on id identity. */ -#if defined JS_NUNBOX32 - static const void *BOGUS_ATOM = (void *)0xdeadbeef; -#elif defined JS_PUNBOX64 - static const void *BOGUS_ATOM = (void *)0xfeedfacedeadbeef; -#endif - - DataLabelPtr inlineAtomOffsetLabel; - Jump idGuard = masm.branchPtrWithPatch(Assembler::NotEqual, idReg, - inlineAtomOffsetLabel, ImmPtr(BOGUS_ATOM)); - DBGLABEL(dbgInlineAtomJump); - - /* - * The state between these two exits is identical, so this safe. The - * GETELEM PIC repatches both jumps to the slowPathStart on reset. - */ - stubcc.linkExit(idGuard, Uses(2)); - pic.slowPathStart = stubcc.linkExit(jmpShapeGuard, Uses(2)); - - stubcc.leave(); - passPICAddress(pic); - pic.slowPathCall = stubcc.call(ic::GetElem); - - /* Load dslots. */ -#if defined JS_NUNBOX32 - DBGLABEL(dbgDslotsLoad); -#elif defined JS_PUNBOX64 - Label dslotsLoadLabel = masm.label(); -#endif - masm.loadPtr(Address(objReg, offsetof(JSObject, slots)), objReg); - - /* Copy the slot value to the expression stack. */ - Address slot(objReg, 1 << 24); -#if defined JS_NUNBOX32 - masm.loadTypeTag(slot, shapeReg); - DBGLABEL(dbgTypeLoad); - masm.loadPayload(slot, objReg); - DBGLABEL(dbgDataLoad); -#elif defined JS_PUNBOX64 - Label inlineValueOffsetLabel = - masm.loadValueAsComponents(slot, shapeReg, objReg); -#endif - pic.fastPathRejoin = masm.label(); - - pic.objReg = objReg; - pic.idReg = idReg; - - RETURN_IF_OOM(false); -#if defined JS_NUNBOX32 - JS_ASSERT(masm.differenceBetween(pic.fastPathRejoin, dbgDslotsLoad) == GETPROP_DSLOTS_LOAD); - JS_ASSERT(masm.differenceBetween(pic.fastPathRejoin, dbgTypeLoad) == GETPROP_TYPE_LOAD); - JS_ASSERT(masm.differenceBetween(pic.fastPathRejoin, dbgDataLoad) == GETPROP_DATA_LOAD); - JS_ASSERT(masm.differenceBetween(pic.shapeGuard, inlineAtomOffsetLabel) == GETELEM_INLINE_ATOM_OFFSET); - JS_ASSERT(masm.differenceBetween(pic.shapeGuard, dbgInlineAtomJump) == GETELEM_INLINE_ATOM_JUMP); - JS_ASSERT(masm.differenceBetween(pic.shapeGuard, inlineShapeOffsetLabel) == GETELEM_INLINE_SHAPE_OFFSET); - JS_ASSERT(masm.differenceBetween(pic.shapeGuard, dbgInlineShapeJump) == GETELEM_INLINE_SHAPE_JUMP); -#elif defined JS_PUNBOX64 - pic.labels.getprop.dslotsLoadOffset = masm.differenceBetween(pic.fastPathRejoin, dslotsLoadLabel); - JS_ASSERT(pic.labels.getprop.dslotsLoadOffset == masm.differenceBetween(pic.fastPathRejoin, dslotsLoadLabel)); - - pic.labels.getelem.inlineShapeOffset = masm.differenceBetween(pic.shapeGuard, inlineShapeOffsetLabel); - JS_ASSERT(pic.labels.getelem.inlineShapeOffset == masm.differenceBetween(pic.shapeGuard, inlineShapeOffsetLabel)); - - pic.labels.getelem.inlineAtomOffset = masm.differenceBetween(pic.shapeGuard, inlineAtomOffsetLabel); - JS_ASSERT(pic.labels.getelem.inlineAtomOffset == masm.differenceBetween(pic.shapeGuard, inlineAtomOffsetLabel)); - - pic.labels.getelem.inlineValueOffset = masm.differenceBetween(pic.fastPathRejoin, inlineValueOffsetLabel); - JS_ASSERT(pic.labels.getelem.inlineValueOffset == masm.differenceBetween(pic.fastPathRejoin, inlineValueOffsetLabel)); - - JS_ASSERT(masm.differenceBetween(inlineShapeOffsetLabel, dbgInlineShapeJump) == GETELEM_INLINE_SHAPE_JUMP); - JS_ASSERT(masm.differenceBetween(pic.shapeGuard, dbgInlineAtomJump) == - pic.labels.getelem.inlineAtomOffset + GETELEM_INLINE_ATOM_JUMP); -#endif - - JS_ASSERT(pic.idReg != pic.objReg); - JS_ASSERT(pic.idReg != pic.shapeReg); - JS_ASSERT(pic.objReg != pic.shapeReg); - - pics.append(pic); - return true; -} -#endif - bool mjit::Compiler::jsop_callprop_generic(JSAtom *atom) { @@ -2847,7 +2769,7 @@ mjit::Compiler::jsop_callprop_generic(JSAtom *atom) RegisterID objReg = frame.copyDataIntoReg(top); RegisterID shapeReg = frame.allocReg(); - PICGenInfo pic(ic::PICInfo::CALL, true); + PICGenInfo pic(ic::PICInfo::CALL, JSOp(*PC), true); pic.pc = PC; @@ -2871,7 +2793,6 @@ mjit::Compiler::jsop_callprop_generic(JSAtom *atom) pic.objReg = objReg; pic.shapeReg = shapeReg; pic.atom = atom; - pic.objRemat = frame.dataRematInfo(top); /* * Store the type and object back. Don't bother keeping them in registers, @@ -2901,7 +2822,7 @@ mjit::Compiler::jsop_callprop_generic(JSAtom *atom) /* Slow path. */ stubcc.leave(); - passPICAddress(pic); + passICAddress(&pic); pic.slowPathCall = stubcc.call(ic::CallProp); /* Adjust the frame. None of this will generate code. */ @@ -3019,7 +2940,7 @@ mjit::Compiler::jsop_callprop_obj(JSAtom *atom) { FrameEntry *top = frame.peek(-1); - PICGenInfo pic(ic::PICInfo::CALL, true); + PICGenInfo pic(ic::PICInfo::CALL, JSOp(*PC), true); JS_ASSERT(top->isTypeKnown()); JS_ASSERT(top->getKnownType() == JSVAL_TYPE_OBJECT); @@ -3034,7 +2955,6 @@ mjit::Compiler::jsop_callprop_obj(JSAtom *atom) pic.shapeReg = shapeReg; pic.atom = atom; - pic.objRemat = frame.dataRematInfo(top); /* Guard on shape. */ masm.loadShape(objReg, shapeReg); @@ -3049,7 +2969,7 @@ mjit::Compiler::jsop_callprop_obj(JSAtom *atom) pic.slowPathStart = stubcc.linkExit(j, Uses(1)); stubcc.leave(); - passPICAddress(pic); + passICAddress(&pic); pic.slowPathCall = stubcc.call(ic::CallProp); /* Load dslots. */ @@ -3151,7 +3071,10 @@ mjit::Compiler::jsop_setprop(JSAtom *atom, bool usePropCache) JSOp op = JSOp(*PC); - PICGenInfo pic(op == JSOP_SETMETHOD ? ic::PICInfo::SETMETHOD : ic::PICInfo::SET, usePropCache); + ic::PICInfo::Kind kind = (op == JSOP_SETMETHOD) + ? ic::PICInfo::SETMETHOD + : ic::PICInfo::SET; + PICGenInfo pic(kind, op, usePropCache); pic.atom = atom; /* Guard that the type is an object. */ @@ -3191,7 +3114,6 @@ mjit::Compiler::jsop_setprop(JSAtom *atom, bool usePropCache) RegisterID shapeReg = frame.allocReg(); pic.shapeReg = shapeReg; - pic.objRemat = frame.dataRematInfo(lhs); frame.unpinEntry(vr); @@ -3209,7 +3131,7 @@ mjit::Compiler::jsop_setprop(JSAtom *atom, bool usePropCache) pic.slowPathStart = stubcc.linkExit(j, Uses(2)); stubcc.leave(); - passPICAddress(pic); + passICAddress(&pic); pic.slowPathCall = stubcc.call(ic::SetProp); } @@ -3276,7 +3198,7 @@ mjit::Compiler::jsop_setprop(JSAtom *atom, bool usePropCache) void mjit::Compiler::jsop_name(JSAtom *atom) { - PICGenInfo pic(ic::PICInfo::NAME, true); + PICGenInfo pic(ic::PICInfo::NAME, JSOp(*PC), true); pic.shapeReg = frame.allocReg(); pic.objReg = frame.allocReg(); @@ -3291,7 +3213,7 @@ mjit::Compiler::jsop_name(JSAtom *atom) { pic.slowPathStart = stubcc.linkExit(j, Uses(0)); stubcc.leave(); - passPICAddress(pic); + passICAddress(&pic); pic.slowPathCall = stubcc.call(ic::Name); } @@ -3308,7 +3230,7 @@ mjit::Compiler::jsop_name(JSAtom *atom) bool mjit::Compiler::jsop_xname(JSAtom *atom) { - PICGenInfo pic(ic::PICInfo::XNAME, true); + PICGenInfo pic(ic::PICInfo::XNAME, JSOp(*PC), true); FrameEntry *fe = frame.peek(-1); if (fe->isNotType(JSVAL_TYPE_OBJECT)) { @@ -3333,7 +3255,7 @@ mjit::Compiler::jsop_xname(JSAtom *atom) { pic.slowPathStart = stubcc.linkExit(j, Uses(1)); stubcc.leave(); - passPICAddress(pic); + passICAddress(&pic); pic.slowPathCall = stubcc.call(ic::XName); } @@ -3352,7 +3274,7 @@ mjit::Compiler::jsop_xname(JSAtom *atom) void mjit::Compiler::jsop_bindname(uint32 index, bool usePropCache) { - PICGenInfo pic(ic::PICInfo::BIND, usePropCache); + PICGenInfo pic(ic::PICInfo::BIND, JSOp(*PC), usePropCache); // This code does not check the frame flags to see if scopeChain has been // set. Rather, it relies on the up-front analysis statically determining @@ -3382,7 +3304,7 @@ mjit::Compiler::jsop_bindname(uint32 index, bool usePropCache) { pic.slowPathStart = stubcc.linkExit(j, Uses(0)); stubcc.leave(); - passPICAddress(pic); + passICAddress(&pic); pic.slowPathCall = stubcc.call(ic::BindName); } diff --git a/js/src/methodjit/Compiler.h b/js/src/methodjit/Compiler.h index 4483e4f8185..3bf27b46e14 100644 --- a/js/src/methodjit/Compiler.h +++ b/js/src/methodjit/Compiler.h @@ -160,18 +160,39 @@ class Compiler : public BaseCompiler bool hasSlowNcode; }; -#if defined JS_POLYIC struct BaseICInfo { + BaseICInfo(JSOp op) : op(op) + { } Label fastPathStart; Label fastPathRejoin; Label slowPathStart; Call slowPathCall; DataLabelPtr paramAddr; + JSOp op; + + void copyTo(ic::BaseIC &to, JSC::LinkBuffer &full, JSC::LinkBuffer &stub) { + to.fastPathStart = full.locationOf(fastPathStart); + to.fastPathRejoin = full.locationOf(fastPathRejoin); + to.slowPathStart = stub.locationOf(slowPathStart); + to.slowPathCall = stub.locationOf(slowPathCall); + to.op = op; + JS_ASSERT(to.op == op); + } + }; + + struct GetElementICInfo : public BaseICInfo { + GetElementICInfo(JSOp op) : BaseICInfo(op) + { } + RegisterID typeReg; + RegisterID objReg; + ValueRemat id; + MaybeJump typeGuard; + Jump claspGuard; }; struct PICGenInfo : public BaseICInfo { - PICGenInfo(ic::PICInfo::Kind kind, bool usePropCache) - : kind(kind), usePropCache(usePropCache) + PICGenInfo(ic::PICInfo::Kind kind, JSOp op, bool usePropCache) + : BaseICInfo(op), kind(kind), usePropCache(usePropCache) { } ic::PICInfo::Kind kind; Label typeCheck; @@ -183,8 +204,6 @@ class Compiler : public BaseCompiler Label shapeGuard; jsbytecode *pc; JSAtom *atom; - StateRemat objRemat; - StateRemat idRemat; bool hasTypeCheck; ValueRemat vr; # if defined JS_CPU_X64 @@ -200,15 +219,12 @@ class Compiler : public BaseCompiler if (ic.isSet()) { ic.u.vr = vr; } else if (ic.isGet()) { - ic.u.get.idReg = idReg; ic.u.get.typeReg = typeReg; ic.u.get.hasTypeCheck = hasTypeCheck; - ic.setObjRemat(objRemat); } } }; -#endif struct Defs { Defs(uint32 ndefs) @@ -250,6 +266,7 @@ class Compiler : public BaseCompiler #endif #if defined JS_POLYIC js::Vector pics; + js::Vector getElemICs; #endif js::Vector callPatches; js::Vector callSites; @@ -298,7 +315,7 @@ class Compiler : public BaseCompiler void iterEnd(); MaybeJump loadDouble(FrameEntry *fe, FPRegisterID fpReg); #ifdef JS_POLYIC - void passPICAddress(PICGenInfo &pic); + void passICAddress(BaseICInfo *ic); #endif #ifdef JS_MONOIC void passMICAddress(MICGenInfo &mic); @@ -400,13 +417,6 @@ class Compiler : public BaseCompiler void jsop_localinc(JSOp op, uint32 slot, bool popped); void jsop_setelem(); bool jsop_getelem(); - bool jsop_getelem_known_type(FrameEntry *obj, FrameEntry *id, RegisterID tmpReg); - bool jsop_getelem_with_pic(FrameEntry *obj, FrameEntry *id, RegisterID tmpReg); - void jsop_getelem_nopic(FrameEntry *obj, FrameEntry *id, RegisterID tmpReg); - bool jsop_getelem_pic(FrameEntry *obj, FrameEntry *id, RegisterID objReg, RegisterID idReg, - RegisterID shapeReg); - void jsop_getelem_dense(FrameEntry *obj, FrameEntry *id, RegisterID objReg, - MaybeRegisterID &idReg, RegisterID shapeReg); void jsop_stricteq(JSOp op); void jsop_equality(JSOp op, BoolStub stub, jsbytecode *target, JSOp fused); void jsop_equality_int_string(JSOp op, BoolStub stub, jsbytecode *target, JSOp fused); diff --git a/js/src/methodjit/FastOps.cpp b/js/src/methodjit/FastOps.cpp index 76cf06f3f24..2f9dc0c77b5 100644 --- a/js/src/methodjit/FastOps.cpp +++ b/js/src/methodjit/FastOps.cpp @@ -1360,135 +1360,6 @@ mjit::Compiler::jsop_setelem() stubcc.rejoin(Changes(0)); } -void -mjit::Compiler::jsop_getelem_dense(FrameEntry *obj, FrameEntry *id, RegisterID objReg, - MaybeRegisterID &idReg, RegisterID tmpReg) -{ - /* Note: idReg is only valid if id is not a constant. */ - Jump guardDense = masm.testObjClass(Assembler::NotEqual, objReg, &js_ArrayClass); - stubcc.linkExit(guardDense, Uses(2)); - - Int32Key key = idReg.isSet() - ? Int32Key::FromRegister(idReg.reg()) - : Int32Key::FromConstant(id->getValue().toInt32()); - - Assembler::FastArrayLoadFails fails = - masm.fastArrayLoad(objReg, key, tmpReg, objReg); - - stubcc.linkExit(fails.rangeCheck, Uses(2)); - stubcc.linkExit(fails.holeCheck, Uses(2)); -} - -bool -mjit::Compiler::jsop_getelem_known_type(FrameEntry *obj, FrameEntry *id, RegisterID tmpReg) -{ - switch (id->getKnownType()) { - case JSVAL_TYPE_INT32: - { - /* Prologue. */ - RegisterID objReg = frame.copyDataIntoReg(obj); - MaybeRegisterID idReg; - if (!id->isConstant()) - idReg.setReg(frame.copyDataIntoReg(id)); - - /* Meat. */ - jsop_getelem_dense(obj, id, objReg, idReg, tmpReg); - stubcc.leave(); - stubcc.call(stubs::GetElem); - - /* Epilogue. */ - if (idReg.isSet()) - frame.freeReg(idReg.reg()); - frame.popn(2); - frame.pushRegs(tmpReg, objReg); - stubcc.rejoin(Changes(1)); - break; - } -#ifdef JS_POLYIC - case JSVAL_TYPE_STRING: - { - /* Prologue. */ - RegisterID objReg = frame.copyDataIntoReg(obj); - RegisterID idReg = frame.copyDataIntoReg(id); - - /* Meat. */ - if (!jsop_getelem_pic(obj, id, objReg, idReg, tmpReg)) - return false; - - /* Epilogue. */ - frame.popn(2); - frame.pushRegs(tmpReg, objReg); - frame.freeReg(idReg); - stubcc.rejoin(Changes(1)); - break; - } -#endif - default: - JS_NOT_REACHED("Invalid known id type."); - } - return true; -} - -#ifdef JS_POLYIC -bool -mjit::Compiler::jsop_getelem_with_pic(FrameEntry *obj, FrameEntry *id, RegisterID tmpReg) -{ - JS_ASSERT(!id->isTypeKnown()); - RegisterID objReg = frame.copyDataIntoReg(obj); - MaybeRegisterID idReg(frame.copyDataIntoReg(id)); - - RegisterID typeReg = frame.tempRegForType(id, tmpReg); - Jump intGuard = masm.testInt32(Assembler::NotEqual, typeReg); - - JaegerSpew(JSpew_Insns, " ==== BEGIN DENSE ARRAY CODE ==== \n"); - - jsop_getelem_dense(obj, id, objReg, idReg, tmpReg); - Jump performedDense = masm.jump(); - - JaegerSpew(JSpew_Insns, " ==== END DENSE ARRAY CODE ==== \n"); - - intGuard.linkTo(masm.label(), &masm); - Jump stringGuard = masm.testString(Assembler::NotEqual, typeReg); - stubcc.linkExit(stringGuard, Uses(2)); /* Neither int nor string at this point. */ - - stubcc.leave(); - stubcc.call(stubs::GetElem); - Jump toFinalMerge = stubcc.masm.jump(); - - if (!jsop_getelem_pic(obj, id, objReg, idReg.reg(), tmpReg)) - return false; - performedDense.linkTo(masm.label(), &masm); - frame.popn(2); - frame.pushRegs(tmpReg, objReg); - frame.freeReg(idReg.reg()); - toFinalMerge.linkTo(stubcc.masm.label(), &stubcc.masm); - stubcc.rejoin(Changes(1)); - return true; -} -#endif - -void -mjit::Compiler::jsop_getelem_nopic(FrameEntry *obj, FrameEntry *id, RegisterID tmpReg) -{ - /* Only handle the int32 case. */ - RegisterID objReg = frame.copyDataIntoReg(obj); - MaybeRegisterID idReg(frame.copyDataIntoReg(id)); - RegisterID typeReg = frame.tempRegForType(id, tmpReg); - Jump intGuard = masm.testInt32(Assembler::NotEqual, typeReg); - stubcc.linkExit(intGuard, Uses(2)); - - /* Meat. */ - jsop_getelem_dense(obj, id, objReg, idReg, tmpReg); - stubcc.leave(); - stubcc.call(stubs::GetElem); - - /* Epilogue. */ - frame.freeReg(idReg.reg()); - frame.popn(2); - frame.pushRegs(tmpReg, objReg); - stubcc.rejoin(Changes(1)); -} - bool mjit::Compiler::jsop_getelem() { @@ -1516,30 +1387,107 @@ mjit::Compiler::jsop_getelem() return true; } - if (id->isTypeKnown() && id->getKnownType() == JSVAL_TYPE_STRING && id->isConstant()) { - /* Never happens, or I'd optimize it. */ - jsop_getelem_slow(); - return true; + GetElementICInfo ic = GetElementICInfo(JSOp(*PC)); + + // Pin the top of the stack to avoid spills, before allocating registers. + MaybeRegisterID pinnedIdData = frame.maybePinData(id); + MaybeRegisterID pinnedIdType = frame.maybePinType(id); + + MaybeJump objTypeGuard; + if (!obj->isTypeKnown()) { + // Test the type of the object without spilling the payload. + MaybeRegisterID pinnedObjData = frame.maybePinData(obj); + Jump guard = frame.testObject(Assembler::NotEqual, obj); + frame.maybeUnpinReg(pinnedObjData); + + // Create a sync path, which we'll rejoin manually later. This is safe + // as long as the IC does not build a stub; it won't, because |obj| + // won't be an object. If we extend this IC to support strings, all + // that needs to change is a little code movement. + stubcc.linkExit(guard, Uses(2)); + objTypeGuard = stubcc.masm.jump(); } - RegisterID tmpReg; - if (obj->isTypeKnown()) { - tmpReg = frame.allocReg(); + // Get a mutable register for the object. This will be the data reg. + ic.objReg = frame.copyDataIntoReg(obj); + + // Get a mutable register for pushing the result type. We kill two birds + // with one stone by making sure, if the key type is not known, to be loaded + // into this register. In this case it is both an input and an output. + frame.maybeUnpinReg(pinnedIdType); + if (id->isConstant() || id->isTypeKnown()) + ic.typeReg = frame.allocReg(); + else + ic.typeReg = frame.copyTypeIntoReg(id); + + // Fill in the id value. + frame.maybeUnpinReg(pinnedIdData); + if (id->isConstant()) { + ic.id = ValueRemat::FromConstant(id->getValue()); } else { - tmpReg = frame.copyTypeIntoReg(obj); - Jump objGuard = masm.testObject(Assembler::NotEqual, tmpReg); - stubcc.linkExit(objGuard, Uses(2)); + RegisterID dataReg = frame.tempRegForData(id); + if (id->isTypeKnown()) + ic.id = ValueRemat::FromKnownType(id->getKnownType(), dataReg); + else + ic.id = ValueRemat::FromRegisters(ic.typeReg, dataReg); } - if (id->isTypeKnown()) - return jsop_getelem_known_type(obj, id, tmpReg); + ic.fastPathStart = masm.label(); + + // Note: slow path here is safe, since the frame will not be modified. + ic.slowPathStart = stubcc.masm.label(); + frame.sync(stubcc.masm, Uses(2)); + + if (id->mightBeType(JSVAL_TYPE_INT32)) { + // Always test the type first (see comment in PolyIC.h). + if (!id->isTypeKnown()) { + ic.typeGuard = masm.testInt32(Assembler::NotEqual, ic.typeReg); + stubcc.linkExitDirect(ic.typeGuard.get(), ic.slowPathStart); + } + + // Guard on the clasp. + ic.claspGuard = masm.testObjClass(Assembler::NotEqual, ic.objReg, &js_ArrayClass); + stubcc.linkExitDirect(ic.claspGuard, ic.slowPathStart); + + Int32Key key = id->isConstant() + ? Int32Key::FromConstant(id->getValue().toInt32()) + : Int32Key::FromRegister(ic.id.dataReg()); + + Assembler::FastArrayLoadFails fails = + masm.fastArrayLoad(ic.objReg, key, ic.typeReg, ic.objReg); + + stubcc.linkExitDirect(fails.rangeCheck, ic.slowPathStart); + stubcc.linkExitDirect(fails.holeCheck, ic.slowPathStart); + } else { + // The type is known to not be dense-friendly ahead of time, so always + // fall back to a slow path. + ic.claspGuard = masm.jump(); + stubcc.linkExitDirect(ic.claspGuard, ic.slowPathStart); + } + + stubcc.leave(); + if (objTypeGuard.isSet()) + objTypeGuard.get().linkTo(stubcc.masm.label(), &stubcc.masm); +#ifdef JS_POLYIC + passICAddress(&ic); + ic.slowPathCall = stubcc.call(ic::GetElement); +#else + ic.slowPathCall = stubcc.call(stubs::GetElem); +#endif + + ic.fastPathRejoin = masm.label(); + + frame.popn(2); + frame.pushRegs(ic.typeReg, ic.objReg); + + stubcc.rejoin(Changes(2)); #ifdef JS_POLYIC - return jsop_getelem_with_pic(obj, id, tmpReg); -#else - jsop_getelem_nopic(obj, id, tmpReg); - return true; + if (!getElemICs.append(ic)) + return false; #endif + + return true; } static inline bool diff --git a/js/src/methodjit/FrameEntry.h b/js/src/methodjit/FrameEntry.h index 56a4dc72ac1..7cea4a464e3 100644 --- a/js/src/methodjit/FrameEntry.h +++ b/js/src/methodjit/FrameEntry.h @@ -97,6 +97,12 @@ class FrameEntry return isTypeKnown() && getKnownType() != type_; } + // Return true if the type of this value is definitely type_, or is unknown + // and thus potentially type_ at runtime. + bool mightBeType(JSValueType type_) const { + return !isNotType(type_); + } + #if defined JS_NUNBOX32 uint32 getPayload() const { //JS_ASSERT(!Valueify(v_.asBits).isDouble() || type.synced()); diff --git a/js/src/methodjit/MethodJIT.cpp b/js/src/methodjit/MethodJIT.cpp index 64e6ffe984a..ae9ad37af7f 100644 --- a/js/src/methodjit/MethodJIT.cpp +++ b/js/src/methodjit/MethodJIT.cpp @@ -811,10 +811,10 @@ mjit::JITScript::release() code.m_executablePool->release(); #if defined JS_POLYIC - for (uint32 i = 0; i < nPICs; i++) { - pics[i].releasePools(); - Destroy(pics[i].execPools); - } + for (uint32 i = 0; i < nPICs; i++) + pics[i].finish(); + for (uint32 i = 0; i < nGetElems; i++) + getElems[i].finish(); #endif #if defined JS_MONOIC diff --git a/js/src/methodjit/MethodJIT.h b/js/src/methodjit/MethodJIT.h index e5332942a09..1bcca198b9b 100644 --- a/js/src/methodjit/MethodJIT.h +++ b/js/src/methodjit/MethodJIT.h @@ -143,6 +143,7 @@ namespace mjit { namespace ic { # if defined JS_POLYIC struct PICInfo; + struct GetElementIC; # endif # if defined JS_MONOIC struct MICInfo; @@ -182,6 +183,7 @@ typedef void * (JS_FASTCALL *VoidPtrStubTraceIC)(VMFrame &, js::mjit::ic::TraceI #endif #ifdef JS_POLYIC typedef void (JS_FASTCALL *VoidStubPIC)(VMFrame &, js::mjit::ic::PICInfo *); +typedef void (JS_FASTCALL *VoidStubGetElemIC)(VMFrame &, js::mjit::ic::GetElementIC *); #endif namespace mjit { @@ -212,6 +214,8 @@ struct JITScript { #ifdef JS_POLYIC ic::PICInfo *pics; /* PICs in this script */ uint32 nPICs; /* number of PolyICs */ + ic::GetElementIC *getElems; + uint32 nGetElems; #endif void *invokeEntry; /* invoke address */ void *fastEntry; /* cached entry, fastest */ diff --git a/js/src/methodjit/NunboxAssembler.h b/js/src/methodjit/NunboxAssembler.h index 6715b388688..d2dc6191641 100644 --- a/js/src/methodjit/NunboxAssembler.h +++ b/js/src/methodjit/NunboxAssembler.h @@ -90,22 +90,15 @@ class NunboxAssembler : public JSC::MacroAssembler return BaseIndex(address.base, address.index, address.scale, address.offset + TAG_OFFSET); } - void loadSlot(RegisterID obj, RegisterID clobber, uint32 slot, bool inlineAccess, - RegisterID type, RegisterID data) { - JS_ASSERT(type != data); - Address address(obj, JSObject::getFixedSlotOffset(slot)); - RegisterID activeAddressReg = obj; - if (!inlineAccess) { - loadPtr(Address(obj, offsetof(JSObject, slots)), clobber); - address = Address(clobber, slot * sizeof(Value)); - activeAddressReg = clobber; - } - if (activeAddressReg == type) { - loadPayload(address, data); - loadTypeTag(address, type); + void loadInlineSlot(RegisterID objReg, uint32 slot, + RegisterID typeReg, RegisterID dataReg) { + Address address(objReg, JSObject::getFixedSlotOffset(slot)); + if (objReg == typeReg) { + loadPayload(address, dataReg); + loadTypeTag(address, typeReg); } else { - loadTypeTag(address, type); - loadPayload(address, data); + loadTypeTag(address, typeReg); + loadPayload(address, dataReg); } } diff --git a/js/src/methodjit/PolyIC.cpp b/js/src/methodjit/PolyIC.cpp index d0084df2927..df2b29e24e5 100644 --- a/js/src/methodjit/PolyIC.cpp +++ b/js/src/methodjit/PolyIC.cpp @@ -45,6 +45,7 @@ #include "assembler/assembler/RepatchBuffer.h" #include "jsscope.h" #include "jsnum.h" +#include "jsatominlines.h" #include "jsobjinlines.h" #include "jsscopeinlines.h" #include "jspropertycache.h" @@ -57,6 +58,9 @@ using namespace js; using namespace js::mjit; using namespace js::mjit::ic; +typedef JSC::RepatchBuffer RepatchBuffer; +typedef JSC::FunctionPtr FunctionPtr; + /* Rough over-estimate of how much memory we need to unprotect. */ static const uint32 INLINE_PATH_LENGTH = 64; @@ -65,18 +69,18 @@ static const uint32 INLINE_PATH_LENGTH = 64; // are instantiated and rooted. class PICLinker : public LinkerHelper { - ic::PICInfo &pic; + ic::BaseIC ⁣ public: - PICLinker(JSContext *cx, ic::PICInfo &pic) - : LinkerHelper(cx), pic(pic) + PICLinker(JSContext *cx, ic::BaseIC &ic) + : LinkerHelper(cx), ic(ic) { } bool init(Assembler &masm) { JSC::ExecutablePool *pool = LinkerHelper::init(masm); if (!pool) return false; - if (!pic.execPools.append(pool)) { + if (!ic.execPools.append(pool)) { pool->release(); js_ReportOutOfMemory(cx); return false; @@ -92,42 +96,38 @@ class PICStubCompiler : public BaseCompiler VMFrame &f; JSScript *script; ic::PICInfo &pic; + void *stub; public: - PICStubCompiler(const char *type, VMFrame &f, JSScript *script, ic::PICInfo &pic) - : BaseCompiler(f.cx), type(type), f(f), script(script), pic(pic) + PICStubCompiler(const char *type, VMFrame &f, JSScript *script, ic::PICInfo &pic, void *stub) + : BaseCompiler(f.cx), type(type), f(f), script(script), pic(pic), stub(stub) { } - bool isCallOp() const - { + bool isCallOp() const { if (pic.kind == ic::PICInfo::CALL) return true; - JSOp op = JSOp(*f.regs.pc); - return !!(js_CodeSpec[op].format & JOF_CALLOP); + return !!(js_CodeSpec[pic.op].format & JOF_CALLOP); } - bool disable(const char *reason, VoidStub stub) - { - return disable(reason, JS_FUNC_TO_DATA_PTR(void *, stub)); + LookupStatus error() { + disable("error"); + return Lookup_Error; } - bool disable(const char *reason, VoidStubPIC stub) { - return disable(reason, JS_FUNC_TO_DATA_PTR(void *, stub)); + LookupStatus error(JSContext *cx) { + return error(); } - bool disable(const char *reason, void *stub) - { - spew("disabled", reason); - JITCode jitCode(pic.slowPathStart.executableAddress(), INLINE_PATH_LENGTH); - CodeBlock codeBlock(jitCode); - RepatchBuffer repatcher(&codeBlock); - repatcher.relink(pic.slowPathCall, FunctionPtr(stub)); - return true; + LookupStatus disable(const char *reason) { + return disable(f.cx, reason); + } + + LookupStatus disable(JSContext *cx, const char *reason) { + return pic.disable(cx, reason, stub); } protected: - void spew(const char *event, const char *op) - { + void spew(const char *event, const char *op) { #ifdef JS_METHODJIT_SPEW JaegerSpew(JSpew_PICs, "%s %s: %s (%s: %d)\n", type, event, op, script->filename, @@ -138,13 +138,13 @@ class PICStubCompiler : public BaseCompiler class PICRepatchBuffer : public JSC::RepatchBuffer { - ic::PICInfo &pic; + ic::BaseIC ⁣ JSC::CodeLocationLabel label; public: - PICRepatchBuffer(ic::PICInfo &ic, JSC::CodeLocationLabel path) + PICRepatchBuffer(ic::BaseIC &ic, JSC::CodeLocationLabel path) : JSC::RepatchBuffer(path.executableAddress(), INLINE_PATH_LENGTH), - pic(ic), label(path) + ic(ic), label(path) { } void relink(int32 offset, JSC::CodeLocationLabel target) { @@ -156,7 +156,6 @@ class SetPropCompiler : public PICStubCompiler { JSObject *obj; JSAtom *atom; - VoidStubPIC stub; int lastStubSecondShapeGuard; static int32 dslotsLoadOffset(ic::PICInfo &pic) { @@ -222,15 +221,10 @@ class SetPropCompiler : public PICStubCompiler public: SetPropCompiler(VMFrame &f, JSScript *script, JSObject *obj, ic::PICInfo &pic, JSAtom *atom, VoidStubPIC stub) - : PICStubCompiler("setprop", f, script, pic), obj(obj), atom(atom), stub(stub), - lastStubSecondShapeGuard(pic.secondShapeGuard) + : PICStubCompiler("setprop", f, script, pic, JS_FUNC_TO_DATA_PTR(void *, stub)), + obj(obj), atom(atom), lastStubSecondShapeGuard(pic.secondShapeGuard) { } - bool disable(const char *reason) - { - return PICStubCompiler::disable(reason, stub); - } - static void reset(ic::PICInfo &pic) { RepatchBuffer repatcher(pic.fastPathStart.executableAddress(), INLINE_PATH_LENGTH); @@ -247,7 +241,7 @@ class SetPropCompiler : public PICStubCompiler repatcher.relink(pic.slowPathCall, target); } - bool patchInline(const Shape *shape, bool inlineSlot) + LookupStatus patchInline(const Shape *shape, bool inlineSlot) { JS_ASSERT(!pic.inlinePathPatched); JaegerSpew(JSpew_PICs, "patch setprop inline at %p\n", pic.fastPathStart.executableAddress()); @@ -286,7 +280,7 @@ class SetPropCompiler : public PICStubCompiler pic.inlinePathPatched = true; - return true; + return Lookup_Cacheable; } void patchPreviousToHere(PICRepatchBuffer &repatcher, CodeLocationLabel cs) @@ -308,7 +302,7 @@ class SetPropCompiler : public PICStubCompiler repatcher.relink(lastStubSecondShapeGuard, cs); } - bool generateStub(uint32 initialShape, const Shape *shape, bool adding, bool inlineSlot) + LookupStatus generateStub(uint32 initialShape, const Shape *shape, bool adding, bool inlineSlot) { /* Exits to the slow path. */ Vector slowExits(cx); @@ -346,9 +340,9 @@ class SetPropCompiler : public PICStubCompiler RegisterID lastReg = pic.objReg; while (proto) { masm.loadPtr(Address(lastReg, offsetof(JSObject, proto)), pic.shapeReg); - Jump protoGuard = masm.guardShape(pic.shapeReg, proto->shape()); + Jump protoGuard = masm.guardShape(pic.shapeReg, proto); if (!otherGuards.append(protoGuard)) - return false; + return error(); proto = proto->getProto(); lastReg = pic.shapeReg; @@ -367,7 +361,7 @@ class SetPropCompiler : public PICStubCompiler Jump mismatchedFunction = masm.branchPtr(Assembler::NotEqual, pic.u.vr.dataReg(), ImmPtr(funobj)); if (!slowExits.append(mismatchedFunction)) - return false; + return error(); } } @@ -382,7 +376,7 @@ class SetPropCompiler : public PICStubCompiler Jump overCapacity = masm.branch32(Assembler::LessThanOrEqual, pic.shapeReg, Imm32(shape->slot)); if (!slowExits.append(overCapacity)) - return false; + return error(); masm.loadPtr(Address(pic.objReg, offsetof(JSObject, slots)), pic.shapeReg); Address address(pic.shapeReg, shape->slot * sizeof(Value)); @@ -420,7 +414,7 @@ class SetPropCompiler : public PICStubCompiler masm.loadPayload(address, pic.shapeReg); Jump rebrand = masm.testFunction(Assembler::Equal, pic.shapeReg); if (!slowExits.append(rebrand)) - return false; + return error(); skip.linkTo(masm.label(), &masm); pic.shapeRegHasBaseShape = false; } @@ -476,7 +470,7 @@ class SetPropCompiler : public PICStubCompiler PICLinker buffer(cx, pic); if (!buffer.init(masm)) - return false; + return error(); buffer.link(shapeGuard, pic.slowPathStart); if (slowExit.isSet()) @@ -511,16 +505,12 @@ class SetPropCompiler : public PICStubCompiler if (pic.stubsGenerated == MAX_PIC_STUBS) disable("max stubs reached"); - return true; + return Lookup_Cacheable; } - bool update() + LookupStatus update() { - if (!pic.hit) { - spew("first hit", "nop"); - pic.hit = true; - return true; - } + JS_ASSERT(pic.hit); if (obj->isDenseArray()) return disable("dense array"); @@ -541,7 +531,7 @@ class SetPropCompiler : public PICStubCompiler JSObject *holder; JSProperty *prop = NULL; if (!obj->lookupProperty(cx, id, &holder, &prop)) - return false; + return error(); /* If the property exists but is on a prototype, treat as addprop. */ if (prop && holder != obj) { @@ -581,7 +571,7 @@ class SetPropCompiler : public PICStubCompiler uint32 initialShape = obj->shape(); if (!obj->ensureClassReservedSlots(cx)) - return false; + return error(); uint32 slots = obj->numSlots(); uintN flags = 0; @@ -604,7 +594,7 @@ class SetPropCompiler : public PICStubCompiler SHAPE_INVALID_SLOT, JSPROP_ENUMERATE, flags, 0); if (!shape) - return false; + return error(); /* * Test after calling putProperty since it can switch obj into @@ -667,11 +657,84 @@ class SetPropCompiler : public PICStubCompiler } }; +static bool +IsCacheableProtoChain(JSObject *obj, JSObject *holder) +{ + while (obj != holder) { + JSObject *proto = obj->getProto(); + if (!proto->isNative()) + return false; + obj = proto; + } + return true; +} + +template +struct GetPropertyHelper { + JSContext *cx; + JSObject *obj; + JSAtom *atom; + IC ⁣ + + JSObject *aobj; + JSObject *holder; + const Shape *shape; + + GetPropertyHelper(JSContext *cx, JSObject *obj, JSAtom *atom, IC &ic) + : cx(cx), obj(obj), atom(atom), ic(ic), holder(NULL), shape(NULL) + { } + + public: + LookupStatus bind() { + JSProperty *prop; + if (!js_FindProperty(cx, ATOM_TO_JSID(atom), &obj, &holder, &prop)) + return ic.error(cx); + if (!prop) + return ic.disable(cx, "lookup failed"); + shape = (const Shape *)prop; + return Lookup_Cacheable; + } + + LookupStatus lookup() { + JSObject *aobj = js_GetProtoIfDenseArray(obj); + if (!aobj->isNative()) + return ic.disable(cx, "non-native"); + JSProperty *prop; + if (!aobj->lookupProperty(cx, ATOM_TO_JSID(atom), &holder, &prop)) + return ic.error(cx); + if (!prop) + return ic.disable(cx, "lookup failed"); + if (!IsCacheableProtoChain(obj, holder)) + return ic.disable(cx, "non-native holder"); + shape = (const Shape *)prop; + return Lookup_Cacheable; + } + + LookupStatus testForGet() { + if (!shape->hasDefaultGetter()) { + if (!shape->isMethod()) + return ic.disable(cx, "getter"); + if (!ic.isCallOp()) + return ic.disable(cx, "method valued shape"); + } else if (!shape->hasSlot()) { + return ic.disable(cx, "no slot"); + } + + return Lookup_Cacheable; + } + + LookupStatus lookupAndTest() { + LookupStatus status = lookup(); + if (status != Lookup_Cacheable) + return status; + return testForGet(); + } +}; + class GetPropCompiler : public PICStubCompiler { JSObject *obj; JSAtom *atom; - VoidStubPIC stub; int lastStubSecondShapeGuard; static int32 inlineShapeOffset(ic::PICInfo &pic) { @@ -713,10 +776,10 @@ class GetPropCompiler : public PICStubCompiler public: GetPropCompiler(VMFrame &f, JSScript *script, JSObject *obj, ic::PICInfo &pic, JSAtom *atom, VoidStubPIC stub) - : PICStubCompiler(pic.kind == ic::PICInfo::CALL ? "callprop" : "getprop", f, script, pic), + : PICStubCompiler(pic.kind == ic::PICInfo::CALL ? "callprop" : "getprop", f, script, pic, + JS_FUNC_TO_DATA_PTR(void *, stub)), obj(obj), atom(atom), - stub(stub), lastStubSecondShapeGuard(pic.secondShapeGuard) { } @@ -754,7 +817,7 @@ class GetPropCompiler : public PICStubCompiler repatcher.relink(pic.slowPathCall, target); } - bool generateArgsLengthStub() + LookupStatus generateArgsLengthStub() { Assembler masm; @@ -772,7 +835,7 @@ class GetPropCompiler : public PICStubCompiler PICLinker buffer(cx, pic); if (!buffer.init(masm)) - return false; + return error(); buffer.link(notArgs, pic.slowPathStart); buffer.link(overridden, pic.slowPathStart); @@ -787,10 +850,10 @@ class GetPropCompiler : public PICStubCompiler disable("args length done"); - return true; + return Lookup_Cacheable; } - bool generateArrayLengthStub() + LookupStatus generateArrayLengthStub() { Assembler masm; @@ -806,7 +869,7 @@ class GetPropCompiler : public PICStubCompiler PICLinker buffer(cx, pic); if (!buffer.init(masm)) - return false; + return error(); buffer.link(notArray, pic.slowPathStart); buffer.link(oob, pic.slowPathStart); @@ -821,10 +884,10 @@ class GetPropCompiler : public PICStubCompiler disable("array length done"); - return true; + return Lookup_Cacheable; } - bool generateStringCallStub() + LookupStatus generateStringCallStub() { JS_ASSERT(pic.hasTypeCheck()); JS_ASSERT(pic.kind == ic::PICInfo::CALL); @@ -832,24 +895,12 @@ class GetPropCompiler : public PICStubCompiler if (!f.fp()->script()->compileAndGo) return disable("String.prototype without compile-and-go"); - JSObject *holder; - JSProperty *prop; - if (!obj->lookupProperty(cx, ATOM_TO_JSID(atom), &holder, &prop)) - return false; - if (!prop) - return disable("property not found"); - - const Shape *shape = (const Shape *)prop; - if (holder != obj) + GetPropertyHelper getprop(cx, obj, atom, *this); + LookupStatus status = getprop.lookupAndTest(); + if (status != Lookup_Cacheable) + return status; + if (getprop.obj != getprop.holder) return disable("proto walk on String.prototype"); - if (!shape->hasDefaultGetterOrIsMethod()) - return disable("getter"); - if (shape->isMethod() && !isCallOp()) - return disable("method valued shape"); - if (!shape->hasSlot()) - return disable("invalid slot"); - - JS_ASSERT(holder->isNative()); Assembler masm; @@ -858,8 +909,7 @@ class GetPropCompiler : public PICStubCompiler ImmType(JSVAL_TYPE_STRING)); /* - * Sink pic.objReg, since we're about to lose it. This is optimistic, - * we could reload it from objRemat if we wanted. + * Sink pic.objReg, since we're about to lose it. * * Note: This is really hacky, and relies on f.regs.sp being set * correctly in ic::CallProp. Should we just move the store higher @@ -880,19 +930,13 @@ class GetPropCompiler : public PICStubCompiler masm.loadShape(pic.objReg, pic.shapeReg); Jump shapeMismatch = masm.branch32(Assembler::NotEqual, pic.shapeReg, Imm32(obj->shape())); - if (!shape->isMethod()) { - masm.loadSlot(pic.objReg, pic.objReg, shape->slot, !obj->hasSlotsArray(), - pic.shapeReg, pic.objReg); - } else { - masm.loadValueAsComponents(ObjectValue(shape->methodObject()), pic.shapeReg, - pic.objReg); - } + masm.loadObjProp(obj, pic.objReg, getprop.shape, pic.shapeReg, pic.objReg); Jump done = masm.jump(); PICLinker buffer(cx, pic); if (!buffer.init(masm)) - return false; + return error(); buffer.link(notString, pic.slowPathStart.labelAtOffset(pic.u.get.typeCheckOffset)); buffer.link(shapeMismatch, pic.slowPathStart); @@ -911,10 +955,10 @@ class GetPropCompiler : public PICStubCompiler /* Disable the PIC so we don't keep generating stubs on the above shape mismatch. */ disable("generated string call stub"); - return true; + return Lookup_Cacheable; } - bool generateStringLengthStub() + LookupStatus generateStringLengthStub() { JS_ASSERT(pic.hasTypeCheck()); @@ -929,7 +973,7 @@ class GetPropCompiler : public PICStubCompiler PICLinker buffer(cx, pic); if (!buffer.init(masm)) - return false; + return error(); buffer.link(notString, pic.slowPathStart.labelAtOffset(pic.u.get.typeCheckOffset)); buffer.link(done, pic.fastPathRejoin); @@ -945,10 +989,10 @@ class GetPropCompiler : public PICStubCompiler disable("generated string length stub"); - return true; + return Lookup_Cacheable; } - bool patchInline(JSObject *holder, const Shape *shape) + LookupStatus patchInline(JSObject *holder, const Shape *shape) { spew("patch", "inline"); PICRepatchBuffer repatcher(pic, pic.fastPathStart); @@ -985,20 +1029,15 @@ class GetPropCompiler : public PICStubCompiler pic.inlinePathPatched = true; - return true; + return Lookup_Cacheable; } - bool generateStub(JSObject *holder, const Shape *shape) + LookupStatus generateStub(JSObject *holder, const Shape *shape) { Vector shapeMismatches(cx); Assembler masm; - if (pic.objNeedsRemat()) { - masm.rematPayload(pic.objRemat(), pic.objReg); - pic.u.get.objNeedsRemat = false; - } - Label start; Jump shapeGuard; Jump argsLenGuard; @@ -1028,63 +1067,33 @@ class GetPropCompiler : public PICStubCompiler Label stubShapeJumpLabel = masm.label(); #endif - if (!shapeMismatches.append(shapeGuard)) - return false; + return error(); + RegisterID holderReg = pic.objReg; if (obj != holder) { - // Emit code that walks the prototype chain. - JSObject *tempObj = obj; - Address proto(pic.objReg, offsetof(JSObject, proto)); - do { - tempObj = tempObj->getProto(); - // FIXME: we should find out why this condition occurs. It is probably - // related to PICs on globals. - if (!tempObj) - return disable("null object in prototype chain"); - JS_ASSERT(tempObj); + // Bake in the holder identity. Careful not to clobber |objReg|, since we can't remat it. + holderReg = pic.shapeReg; + masm.move(ImmPtr(holder), holderReg); + pic.shapeRegHasBaseShape = false; - /* - * If there is a non-native along the prototype chain the shape is technically - * invalid. - */ - if (!tempObj->isNative()) - return disable("non-JS-native in prototype chain"); - - masm.loadPtr(proto, pic.objReg); - pic.shapeRegHasBaseShape = false; - pic.u.get.objNeedsRemat = true; - - Jump j = masm.branchTestPtr(Assembler::Zero, pic.objReg, pic.objReg); - if (!shapeMismatches.append(j)) - return false; - } while (tempObj != holder); - - // Load the shape out of the holder and check it. - masm.loadShape(pic.objReg, pic.shapeReg); - Jump j = masm.branch32_force32(Assembler::NotEqual, pic.shapeReg, - Imm32(holder->shape())); + // Guard on the holder's shape. + Jump j = masm.guardShape(holderReg, holder); if (!shapeMismatches.append(j)) - return false; + return error(); + pic.secondShapeGuard = masm.distanceOf(masm.label()) - masm.distanceOf(start); } else { - JS_ASSERT(holder->isNative()); /* Precondition: already checked. */ pic.secondShapeGuard = 0; } /* Load the value out of the object. */ - if (!shape->isMethod()) { - masm.loadSlot(pic.objReg, pic.objReg, shape->slot, !holder->hasSlotsArray(), - pic.shapeReg, pic.objReg); - } else { - masm.loadValueAsComponents(ObjectValue(shape->methodObject()), pic.shapeReg, - pic.objReg); - } + masm.loadObjProp(holder, holderReg, shape, pic.shapeReg, pic.objReg); Jump done = masm.jump(); PICLinker buffer(cx, pic); if (!buffer.init(masm)) - return false; + return error(); // The guard exit jumps to the original slow case. for (Jump *pj = shapeMismatches.begin(); pj != shapeMismatches.end(); ++pj) @@ -1111,7 +1120,7 @@ class GetPropCompiler : public PICStubCompiler if (obj->isDenseArray()) disable("dense array"); - return true; + return Lookup_Cacheable; } void patchPreviousToHere(PICRepatchBuffer &repatcher, CodeLocationLabel cs) @@ -1133,373 +1142,19 @@ class GetPropCompiler : public PICStubCompiler repatcher.relink(lastStubSecondShapeGuard, cs); } - bool update() + LookupStatus update() { - if (!pic.hit) { - spew("first hit", "nop"); - pic.hit = true; - return true; - } + JS_ASSERT(pic.hit); - JSObject *aobj = js_GetProtoIfDenseArray(obj); - if (!aobj->isNative()) - return disable("non-native"); + GetPropertyHelper getprop(cx, obj, atom, *this); + LookupStatus status = getprop.lookupAndTest(); + if (status != Lookup_Cacheable) + return status; - JSObject *holder; - JSProperty *prop; - if (!aobj->lookupProperty(cx, ATOM_TO_JSID(atom), &holder, &prop)) - return false; - - if (!prop) - return disable("lookup failed"); - - if (!holder->isNative()) - return disable("non-native holder"); - - const Shape *shape = (const Shape *)prop; - if (!shape->hasDefaultGetterOrIsMethod()) - return disable("getter"); - if (shape->isMethod() && !isCallOp()) - return disable("method valued shape"); - if (!shape->hasSlot()) - return disable("invalid slot"); - - if (obj == holder && !pic.inlinePathPatched) - return patchInline(holder, shape); + if (obj == getprop.holder && !pic.inlinePathPatched) + return patchInline(getprop.holder, getprop.shape); - return generateStub(holder, shape); - } - - bool disable(const char *reason) - { - return PICStubCompiler::disable(reason, stub); - } -}; - -class GetElemCompiler : public PICStubCompiler -{ - JSObject *obj; - JSString *id; - void *stub; - int lastStubSecondShapeGuard; - - static int32 dslotsLoad(ic::PICInfo &pic) { -#if defined JS_NUNBOX32 - return GETELEM_DSLOTS_LOAD; -#elif defined JS_PUNBOX64 - return pic.labels.getelem.dslotsLoadOffset; -#endif - } - - inline int32 dslotsLoad() { - return dslotsLoad(pic); - } - - static int32 inlineShapeOffset(ic::PICInfo &pic) { -#if defined JS_NUNBOX32 - return GETELEM_INLINE_SHAPE_OFFSET; -#elif defined JS_PUNBOX64 - return pic.labels.getelem.inlineShapeOffset; -#endif - } - - inline int32 inlineShapeOffset() { - return inlineShapeOffset(pic); - } - - static int32 inlineShapeJump(ic::PICInfo &pic) { -#if defined JS_NUNBOX32 - return GETELEM_INLINE_SHAPE_JUMP; -#elif defined JS_PUNBOX64 - return inlineShapeOffset(pic) + GETELEM_INLINE_SHAPE_JUMP; -#endif - } - - inline int32 inlineShapeJump() { - return inlineShapeJump(pic); - } - - static int32 inlineAtomOffset(ic::PICInfo &pic) { -#if defined JS_NUNBOX32 - return GETELEM_INLINE_ATOM_OFFSET; -#elif defined JS_PUNBOX64 - return pic.labels.getelem.inlineAtomOffset; -#endif - } - - inline int32 inlineAtomOffset() { - return inlineAtomOffset(pic); - } - - static int32 inlineAtomJump(ic::PICInfo &pic) { -#if defined JS_NUNBOX32 - return GETELEM_INLINE_ATOM_JUMP; -#elif defined JS_PUNBOX64 - return inlineAtomOffset(pic) + GETELEM_INLINE_ATOM_JUMP; -#endif - } - - inline int32 inlineAtomJump() { - return inlineAtomJump(pic); - } - - public: - GetElemCompiler(VMFrame &f, JSScript *script, JSObject *obj, ic::PICInfo &pic, JSString *id, - VoidStub stub) - : PICStubCompiler("getelem", f, script, pic), obj(obj), id(id), - stub(JS_FUNC_TO_DATA_PTR(void *, stub)), - lastStubSecondShapeGuard(pic.secondShapeGuard) - {} - - static void reset(ic::PICInfo &pic) - { - JS_ASSERT(pic.kind == ic::PICInfo::GETELEM); - - RepatchBuffer repatcher(pic.fastPathStart.executableAddress(), INLINE_PATH_LENGTH); - repatcher.repatchLEAToLoadPtr(pic.fastPathRejoin.instructionAtOffset(dslotsLoad(pic))); - - /* Only the shape needs to be patched to fail -- atom jump will never be taken. */ - repatcher.repatch(pic.fastPathStart.dataLabel32AtOffset( - pic.shapeGuard + inlineShapeOffset(pic)), - int32(JSObjectMap::INVALID_SHAPE)); - repatcher.relink(pic.fastPathStart.jumpAtOffset(pic.shapeGuard + inlineShapeJump(pic)), - pic.slowPathStart); - repatcher.relink(pic.fastPathStart.jumpAtOffset(pic.shapeGuard + inlineAtomJump(pic)), - pic.slowPathStart); - - RepatchBuffer repatcher2(pic.slowPathStart.executableAddress(), INLINE_PATH_LENGTH); - - FunctionPtr target(JS_FUNC_TO_DATA_PTR(void *, ic::GetElem)); - repatcher.relink(pic.slowPathCall, target); - } - - bool patchInline(JSObject *holder, const Shape *shape) - { - spew("patch", "inline"); - PICRepatchBuffer repatcher(pic, pic.fastPathStart); - - int32 offset; - if (!holder->hasSlotsArray()) { - JSC::CodeLocationInstruction istr = pic.fastPathRejoin.instructionAtOffset(dslotsLoad()); - repatcher.repatchLoadPtrToLEA(istr); - - // - // We've patched | mov dslots, [obj + DSLOTS_OFFSET] - // To: | lea fslots, [obj + DSLOTS_OFFSET] - // - // Because the offset is wrong, it's necessary to correct it - // below. - // - int32 diff = int32(JSObject::getFixedSlotOffset(0)) - int32(offsetof(JSObject, slots)); - JS_ASSERT(diff != 0); - offset = (int32(shape->slot) * sizeof(Value)) + diff; - } else { - offset = shape->slot * sizeof(Value); - } - - uint32 shapeOffset = pic.shapeGuard + inlineShapeOffset(); - repatcher.repatch(pic.fastPathStart.dataLabel32AtOffset(shapeOffset), obj->shape()); - uint32 idOffset = pic.shapeGuard + inlineAtomOffset(); - repatcher.repatch(pic.fastPathStart.dataLabelPtrAtOffset(idOffset), id); -#if defined JS_NUNBOX32 - repatcher.repatch(pic.fastPathRejoin.dataLabel32AtOffset(GETELEM_TYPE_LOAD), offset + 4); - repatcher.repatch(pic.fastPathRejoin.dataLabel32AtOffset(GETELEM_DATA_LOAD), offset); -#elif defined JS_PUNBOX64 - repatcher.repatch(pic.fastPathRejoin.dataLabel32AtOffset(pic.labels.getelem.inlineValueOffset), offset); -#endif - pic.inlinePathPatched = true; - - return true; - } - - void patchPreviousToHere(PICRepatchBuffer &repatcher, CodeLocationLabel cs) - { - // Patch either the inline fast path or a generated stub. The stub - // omits the prefix of the inline fast path that loads the shape, so - // the offsets are different. - int shapeGuardJumpOffset; - int atomGuardJumpOffset; - if (pic.stubsGenerated) { -#if defined JS_NUNBOX32 - shapeGuardJumpOffset = GETELEM_STUB_SHAPE_JUMP; -#elif defined JS_PUNBOX64 - shapeGuardJumpOffset = pic.labels.getelem.stubShapeJump; -#endif - atomGuardJumpOffset = GETELEM_STUB_ATOM_JUMP; - } else { - shapeGuardJumpOffset = pic.shapeGuard + inlineShapeJump(); - atomGuardJumpOffset = pic.shapeGuard + inlineAtomJump(); - } - repatcher.relink(shapeGuardJumpOffset, cs); - repatcher.relink(atomGuardJumpOffset, cs); - if (lastStubSecondShapeGuard) - repatcher.relink(lastStubSecondShapeGuard, cs); - } - - bool generateStub(JSObject *holder, const Shape *shape) - { - JS_ASSERT(pic.u.get.idReg != pic.shapeReg); - Vector shapeMismatches(cx); - - Assembler masm; - - if (pic.objNeedsRemat()) { - masm.rematPayload(pic.objRemat(), pic.objReg); - pic.u.get.objNeedsRemat = false; - } - - if (pic.shapeNeedsRemat()) { - masm.loadShape(pic.objReg, pic.shapeReg); - pic.shapeRegHasBaseShape = true; - } - - Label start = masm.label(); - Jump atomGuard = masm.branchPtr(Assembler::NotEqual, pic.u.get.idReg, ImmPtr(id)); - DBGLABEL(dbgStubAtomJump); - Jump shapeGuard = masm.branch32_force32(Assembler::NotEqual, pic.shapeReg, - Imm32(obj->shape())); - -#if (defined JS_NUNBOX32 && defined DEBUG) || defined JS_PUNBOX64 - Label stubShapeJump = masm.label(); -#endif - - JS_ASSERT(masm.differenceBetween(start, dbgStubAtomJump) == GETELEM_STUB_ATOM_JUMP); -#if defined JS_NUNBOX32 - JS_ASSERT(masm.differenceBetween(start, stubShapeJump) == GETELEM_STUB_SHAPE_JUMP); -#endif - - if (!(shapeMismatches.append(shapeGuard) && shapeMismatches.append(atomGuard))) - return false; - - if (obj != holder) { - // Emit code that walks the prototype chain. - JSObject *tempObj = obj; - Address proto(pic.objReg, offsetof(JSObject, proto)); - do { - tempObj = tempObj->getProto(); - // FIXME: we should find out why this condition occurs. It is probably - // related to PICs on globals. - if (!tempObj) - return disable("null object in prototype chain"); - JS_ASSERT(tempObj); - - /* - * If there is a non-native along the prototype chain the shape is technically - * invalid. - */ - if (!tempObj->isNative()) - return disable("non-JS-native in prototype chain"); - - masm.loadPtr(proto, pic.objReg); - pic.shapeRegHasBaseShape = false; - pic.u.get.objNeedsRemat = true; - - Jump j = masm.branchTestPtr(Assembler::Zero, pic.objReg, pic.objReg); - if (!shapeMismatches.append(j)) - return false; - } while (tempObj != holder); - - // Load the shape out of the holder and check it. - masm.loadShape(pic.objReg, pic.shapeReg); - Jump j = masm.branch32_force32(Assembler::NotEqual, pic.shapeReg, - Imm32(holder->shape())); - if (!shapeMismatches.append(j)) - return false; - pic.secondShapeGuard = masm.distanceOf(masm.label()) - masm.distanceOf(start); - } else { - JS_ASSERT(holder->isNative()); /* Precondition: already checked. */ - pic.secondShapeGuard = 0; - } - - /* Load the value out of the object. */ - if (!shape->isMethod()) { - masm.loadSlot(pic.objReg, pic.objReg, shape->slot, !holder->hasSlotsArray(), - pic.shapeReg, pic.objReg); - } else { - masm.loadValueAsComponents(ObjectValue(shape->methodObject()), pic.shapeReg, - pic.objReg); - } - Jump done = masm.jump(); - - PICLinker buffer(cx, pic); - if (!buffer.init(masm)) - return false; - - // The guard exit jumps to the original slow case. - for (Jump *pj = shapeMismatches.begin(); pj != shapeMismatches.end(); ++pj) - buffer.link(*pj, pic.slowPathStart); - - // The final exit jumps to the store-back in the inline stub. - buffer.link(done, pic.fastPathRejoin); - CodeLocationLabel cs = buffer.finalizeCodeAddendum(); -#if DEBUG - char *chars = js_DeflateString(cx, id->chars(), id->length()); - JaegerSpew(JSpew_PICs, "generated %s stub at %p for atom 0x%x (\"%s\") shape 0x%x (%s: %d)\n", - type, cs.executableAddress(), id, chars, holder->shape(), script->filename, - js_FramePCToLineNumber(cx, f.fp())); - cx->free(chars); -#endif - - PICRepatchBuffer repatcher(pic, pic.lastPathStart()); - patchPreviousToHere(repatcher, cs); - - pic.stubsGenerated++; - pic.lastStubStart = buffer.locationOf(start); - -#if defined JS_PUNBOX64 - pic.labels.getelem.stubShapeJump = masm.differenceBetween(start, stubShapeJump); - JS_ASSERT(pic.labels.getelem.stubShapeJump == masm.differenceBetween(start, stubShapeJump)); -#endif - - if (pic.stubsGenerated == MAX_PIC_STUBS) - disable("max stubs reached"); - if (obj->isDenseArray()) - disable("dense array"); - - return true; - } - - bool update() - { - if (!pic.hit) { - spew("first hit", "nop"); - pic.hit = true; - return true; - } - - JSAtom *atom = js_AtomizeString(cx, id, 0); - if (!atom) - return false; - JSObject *holder; - JSProperty *prop; - if (!obj->lookupProperty(cx, ATOM_TO_JSID(atom), &holder, &prop)) - return false; - - if (!prop) - return disable("lookup failed"); - - if (!obj->isNative()) - return disable("non-native obj"); - if (!holder->isNative()) - return disable("non-native holder"); - - const Shape *shape = (const Shape *)prop; - if (!shape->hasDefaultGetterOrIsMethod()) - return disable("getter"); - if (shape->isMethod() && !isCallOp()) - return disable("method valued shape"); - if (!shape->hasSlot()) - return disable("invalid slot"); - - if (obj == holder && !pic.inlinePathPatched) - return patchInline(holder, shape); - - return generateStub(holder, shape); - } - - bool disable(const char *reason) - { - return PICStubCompiler::disable(reason, stub); + return generateStub(getprop.holder, getprop.shape); } }; @@ -1507,25 +1162,17 @@ class ScopeNameCompiler : public PICStubCompiler { JSObject *scopeChain; JSAtom *atom; - void *stub; - JSObject *obj; - JSObject *holder; - JSProperty *prop; - const Shape *shape; + GetPropertyHelper getprop; public: ScopeNameCompiler(VMFrame &f, JSScript *script, JSObject *scopeChain, ic::PICInfo &pic, JSAtom *atom, VoidStubPIC stub) - : PICStubCompiler("name", f, script, pic), scopeChain(scopeChain), atom(atom), - stub(JS_FUNC_TO_DATA_PTR(void *, stub)), obj(NULL), holder(NULL), prop(NULL) + : PICStubCompiler("name", f, script, pic, JS_FUNC_TO_DATA_PTR(void *, stub)), + scopeChain(scopeChain), atom(atom), + getprop(f.cx, NULL, atom, *this) { } - bool disable(const char *reason) - { - return PICStubCompiler::disable(reason, stub); - } - static void reset(ic::PICInfo &pic) { RepatchBuffer repatcher(pic.fastPathStart.executableAddress(), INLINE_PATH_LENGTH); @@ -1540,16 +1187,16 @@ class ScopeNameCompiler : public PICStubCompiler typedef Vector JumpList; - bool walkScopeChain(Assembler &masm, JumpList &fails, bool &found) + LookupStatus walkScopeChain(Assembler &masm, JumpList &fails) { /* Walk the scope chain. */ JSObject *tobj = scopeChain; /* For GETXPROP, we'll never enter this loop. */ - JS_ASSERT_IF(pic.kind == ic::PICInfo::XNAME, tobj && tobj == holder); - JS_ASSERT_IF(pic.kind == ic::PICInfo::XNAME, obj == tobj); + JS_ASSERT_IF(pic.kind == ic::PICInfo::XNAME, tobj && tobj == getprop.holder); + JS_ASSERT_IF(pic.kind == ic::PICInfo::XNAME, getprop.obj == tobj); - while (tobj && tobj != holder) { + while (tobj && tobj != getprop.holder) { if (!js_IsCacheableNonGlobalScope(tobj)) return disable("non-cacheable scope chain object"); JS_ASSERT(tobj->isNative()); @@ -1558,14 +1205,14 @@ class ScopeNameCompiler : public PICStubCompiler /* scopeChain will never be NULL, but parents can be NULL. */ Jump j = masm.branchTestPtr(Assembler::Zero, pic.objReg, pic.objReg); if (!fails.append(j)) - return false; + return error(); } /* Guard on intervening shapes. */ masm.loadShape(pic.objReg, pic.shapeReg); Jump j = masm.branch32(Assembler::NotEqual, pic.shapeReg, Imm32(tobj->shape())); if (!fails.append(j)) - return false; + return error(); /* Load the next link in the scope chain. */ Address parent(pic.objReg, offsetof(JSObject, parent)); @@ -1574,12 +1221,13 @@ class ScopeNameCompiler : public PICStubCompiler tobj = tobj->getParent(); } - found = tobj == holder; + if (tobj != getprop.holder) + return disable("scope chain walk terminated early"); - return true; + return Lookup_Cacheable; } - bool generateGlobalStub() + LookupStatus generateGlobalStub(JSObject *obj) { Assembler masm; JumpList fails(cx); @@ -1588,29 +1236,21 @@ class ScopeNameCompiler : public PICStubCompiler if (pic.kind == ic::PICInfo::NAME) masm.loadPtr(Address(JSFrameReg, JSStackFrame::offsetOfScopeChain()), pic.objReg); - JS_ASSERT(obj == holder); - JS_ASSERT(holder == scopeChain->getGlobal()); + JS_ASSERT(obj == getprop.holder); + JS_ASSERT(getprop.holder == scopeChain->getGlobal()); - bool found = false; - if (!walkScopeChain(masm, fails, found)) - return false; - if (!found) - return disable("scope chain walk terminated early"); + LookupStatus status = walkScopeChain(masm, fails); + if (status != Lookup_Cacheable) + return status; /* If a scope chain walk was required, the final object needs a NULL test. */ MaybeJump finalNull; if (pic.kind == ic::PICInfo::NAME) finalNull = masm.branchTestPtr(Assembler::Zero, pic.objReg, pic.objReg); masm.loadShape(pic.objReg, pic.shapeReg); - Jump finalShape = masm.branch32(Assembler::NotEqual, pic.shapeReg, Imm32(holder->shape())); - - if (!shape->isMethod()) { - masm.loadSlot(pic.objReg, pic.objReg, shape->slot, false, pic.shapeReg, pic.objReg); - } else { - masm.loadValueAsComponents(ObjectValue(shape->methodObject()), pic.shapeReg, - pic.objReg); - } + Jump finalShape = masm.branch32(Assembler::NotEqual, pic.shapeReg, Imm32(getprop.holder->shape())); + masm.loadObjProp(obj, pic.objReg, getprop.shape, pic.shapeReg, pic.objReg); Jump done = masm.jump(); // All failures flow to here, so there is a common point to patch. @@ -1627,7 +1267,7 @@ class ScopeNameCompiler : public PICStubCompiler PICLinker buffer(cx, pic); if (!buffer.init(masm)) - return false; + return error(); buffer.link(failJump, pic.slowPathStart); buffer.link(done, pic.fastPathRejoin); @@ -1644,7 +1284,7 @@ class ScopeNameCompiler : public PICStubCompiler if (pic.stubsGenerated == MAX_PIC_STUBS) disable("max stubs reached"); - return true; + return Lookup_Cacheable; } enum CallObjPropKind { @@ -1652,7 +1292,7 @@ class ScopeNameCompiler : public PICStubCompiler VAR }; - bool generateCallStub() + LookupStatus generateCallStub(JSObject *obj) { Assembler masm; Vector fails(cx); @@ -1661,10 +1301,11 @@ class ScopeNameCompiler : public PICStubCompiler if (pic.kind == ic::PICInfo::NAME) masm.loadPtr(Address(JSFrameReg, JSStackFrame::offsetOfScopeChain()), pic.objReg); - JS_ASSERT(obj == holder); - JS_ASSERT(holder != scopeChain->getGlobal()); + JS_ASSERT(obj == getprop.holder); + JS_ASSERT(getprop.holder != scopeChain->getGlobal()); CallObjPropKind kind; + const Shape *shape = getprop.shape; if (shape->getterOp() == js_GetCallArg) { kind = ARG; } else if (shape->getterOp() == js_GetCallVar) { @@ -1673,23 +1314,21 @@ class ScopeNameCompiler : public PICStubCompiler return disable("unhandled callobj sprop getter"); } - bool found = false; - if (!walkScopeChain(masm, fails, found)) - return false; - if (!found) - return disable("scope chain walk terminated early"); + LookupStatus status = walkScopeChain(masm, fails); + if (status != Lookup_Cacheable) + return status; /* If a scope chain walk was required, the final object needs a NULL test. */ MaybeJump finalNull; if (pic.kind == ic::PICInfo::NAME) finalNull = masm.branchTestPtr(Assembler::Zero, pic.objReg, pic.objReg); masm.loadShape(pic.objReg, pic.shapeReg); - Jump finalShape = masm.branch32(Assembler::NotEqual, pic.shapeReg, Imm32(holder->shape())); + Jump finalShape = masm.branch32(Assembler::NotEqual, pic.shapeReg, Imm32(getprop.holder->shape())); /* Get callobj's stack frame. */ masm.loadFunctionPrivate(pic.objReg, pic.shapeReg); - JSFunction *fun = holder->getCallObjCalleeFunction(); + JSFunction *fun = getprop.holder->getCallObjCalleeFunction(); uint16 slot = uint16(shape->shortid); Jump skipOver; @@ -1731,7 +1370,7 @@ class ScopeNameCompiler : public PICStubCompiler PICLinker buffer(cx, pic); if (!buffer.init(masm)) - return false; + return error(); buffer.link(failJump, pic.slowPathStart); buffer.link(done, pic.fastPathRejoin); @@ -1747,67 +1386,59 @@ class ScopeNameCompiler : public PICStubCompiler if (pic.stubsGenerated == MAX_PIC_STUBS) disable("max stubs reached"); - return true; + return Lookup_Cacheable; } - bool updateForName() + LookupStatus updateForName() { - if (!js_FindProperty(cx, ATOM_TO_JSID(atom), &obj, &holder, &prop)) - return false; + // |getprop.obj| is filled by bind() + LookupStatus status = getprop.bind(); + if (status != Lookup_Cacheable) + return status; - return update(); + return update(getprop.obj); } - bool updateForXName() + LookupStatus updateForXName() { - obj = scopeChain; + // |obj| and |getprop.obj| are NULL, but should be the given scopeChain. + getprop.obj = scopeChain; + LookupStatus status = getprop.lookup(); + if (status != Lookup_Cacheable) + return status; - if (!obj->lookupProperty(cx, ATOM_TO_JSID(atom), &holder, &prop)) - return false; - - return update(); + return update(getprop.obj); } - bool update() + LookupStatus update(JSObject *obj) { - if (!pic.hit) { - spew("first hit", "nop"); - pic.hit = true; - return true; - } - - if (!prop) - return disable("property not found"); - if (!obj->isNative() || !holder->isNative()) - return disable("non-native scope object"); - if (obj != holder) + if (obj != getprop.holder) return disable("property is on proto of a scope object"); - - shape = (const Shape *)prop; if (obj->getClass() == &js_CallClass) - return generateCallStub(); + return generateCallStub(obj); - if (!shape->hasDefaultGetterOrIsMethod()) - return disable("getter"); - if (shape->isMethod() && !isCallOp()) - return disable("method valued shape"); - if (!shape->hasSlot()) - return disable("invalid slot"); + LookupStatus status = getprop.testForGet(); + if (status != Lookup_Cacheable) + return status; if (!obj->getParent()) - return generateGlobalStub(); + return generateGlobalStub(obj); return disable("scope object not handled yet"); } bool retrieve(Value *vp) { - if (prop && (!obj->isNative() || !holder->isNative())) { + JSObject *obj = getprop.obj; + JSObject *holder = getprop.holder; + const Shape *shape = getprop.shape; + + if (shape && (!obj->isNative() || !holder->isNative())) { if (!obj->getProperty(cx, ATOM_TO_JSID(atom), vp)) return false; } else { - if (!prop) { + if (!shape) { /* Kludge to allow (typeof foo == "undefined") tests. */ disable("property not found"); if (pic.kind == ic::PICInfo::NAME) { @@ -1820,12 +1451,10 @@ class ScopeNameCompiler : public PICStubCompiler ReportAtomNotDefined(cx, atom); return false; } - const Shape *shape = (const Shape *)prop; JSObject *normalized = obj; if (obj->getClass() == &js_WithClass && !shape->hasDefaultGetter()) normalized = js_UnwrapWithObject(cx, obj); - NATIVE_GET(cx, normalized, holder, shape, JSGET_METHOD_BARRIER, vp, - return false); + NATIVE_GET(cx, normalized, holder, shape, JSGET_METHOD_BARRIER, vp, return false); } return true; @@ -1836,7 +1465,6 @@ class BindNameCompiler : public PICStubCompiler { JSObject *scopeChain; JSAtom *atom; - void *stub; static int32 inlineJumpOffset(ic::PICInfo &pic) { #if defined JS_NUNBOX32 @@ -1853,15 +1481,10 @@ class BindNameCompiler : public PICStubCompiler public: BindNameCompiler(VMFrame &f, JSScript *script, JSObject *scopeChain, ic::PICInfo &pic, JSAtom *atom, VoidStubPIC stub) - : PICStubCompiler("bind", f, script, pic), scopeChain(scopeChain), atom(atom), - stub(JS_FUNC_TO_DATA_PTR(void *, stub)) + : PICStubCompiler("bind", f, script, pic, JS_FUNC_TO_DATA_PTR(void *, stub)), + scopeChain(scopeChain), atom(atom) { } - bool disable(const char *reason) - { - return PICStubCompiler::disable(reason, stub); - } - static void reset(ic::PICInfo &pic) { PICRepatchBuffer repatcher(pic, pic.fastPathStart); @@ -1872,7 +1495,7 @@ class BindNameCompiler : public PICStubCompiler repatcher2.relink(pic.slowPathCall, target); } - bool generateStub(JSObject *obj) + LookupStatus generateStub(JSObject *obj) { Assembler masm; js::Vector fails(cx); @@ -1892,7 +1515,7 @@ class BindNameCompiler : public PICStubCompiler masm.loadPtr(parent, pic.objReg); Jump nullTest = masm.branchTestPtr(Assembler::Zero, pic.objReg, pic.objReg); if (!fails.append(nullTest)) - return false; + return error(); masm.loadShape(pic.objReg, pic.shapeReg); Jump shapeTest = masm.branch32(Assembler::NotEqual, pic.shapeReg, Imm32(tobj->shape())); @@ -1915,7 +1538,7 @@ class BindNameCompiler : public PICStubCompiler PICLinker buffer(cx, pic); if (!buffer.init(masm)) - return false; + return error(); buffer.link(failJump, pic.slowPathStart); buffer.link(done, pic.fastPathRejoin); @@ -1934,7 +1557,7 @@ class BindNameCompiler : public PICStubCompiler if (pic.stubsGenerated == MAX_PIC_STUBS) disable("max stubs reached"); - return true; + return Lookup_Cacheable; } JSObject *update() @@ -1951,7 +1574,8 @@ class BindNameCompiler : public PICStubCompiler return obj; } - if (!generateStub(obj)) + LookupStatus status = generateStub(obj); + if (status == Lookup_Error) return NULL; return obj; @@ -1985,10 +1609,9 @@ ic::GetProp(VMFrame &f, ic::PICInfo *pic) if (atom == f.cx->runtime->atomState.lengthAtom) { if (f.regs.sp[-1].isString()) { GetPropCompiler cc(f, script, NULL, *pic, NULL, DisabledLengthIC); - if (!cc.generateStringLengthStub()) { - cc.disable("error"); + LookupStatus status = cc.generateStringLengthStub(); + if (status == Lookup_Error) THROW(); - } JSString *str = f.regs.sp[-1].toString(); f.regs.sp[-1].setInt32(str->length()); return; @@ -1997,16 +1620,14 @@ ic::GetProp(VMFrame &f, ic::PICInfo *pic) if (obj->isArray() || (obj->isArguments() && !obj->isArgsLengthOverridden())) { GetPropCompiler cc(f, script, obj, *pic, NULL, DisabledLengthIC); if (obj->isArray()) { - if (!cc.generateArrayLengthStub()) { - cc.disable("error"); + LookupStatus status = cc.generateArrayLengthStub(); + if (status == Lookup_Error) THROW(); - } f.regs.sp[-1].setNumber(obj->getArrayLength()); } else if (obj->isArguments()) { - if (!cc.generateArgsLengthStub()) { - cc.disable("error"); + LookupStatus status = cc.generateArgsLengthStub(); + if (status == Lookup_Error) THROW(); - } f.regs.sp[-1].setInt32(int32_t(obj->getArgsInitialLength())); } return; @@ -2019,7 +1640,7 @@ ic::GetProp(VMFrame &f, ic::PICInfo *pic) if (!obj) THROW(); - if (pic->shouldGenerate()) { + if (pic->shouldUpdate(f.cx)) { VoidStubPIC stub = pic->usePropCache ? DisabledGetPropIC : DisabledGetPropICNoCache; @@ -2036,35 +1657,6 @@ ic::GetProp(VMFrame &f, ic::PICInfo *pic) f.regs.sp[-1] = v; } -void JS_FASTCALL -ic::GetElem(VMFrame &f, ic::PICInfo *pic) -{ - JSScript *script = f.fp()->script(); - - JSObject *obj = ValueToObject(f.cx, &f.regs.sp[-2]); - if (!obj) - THROW(); - - Value idval = f.regs.sp[-1]; - JS_ASSERT(idval.isString()); - JSString *id = idval.toString(); - if (pic->shouldGenerate()) { - GetElemCompiler cc(f, script, obj, *pic, id, stubs::GetElem); - if (!cc.update()) { - cc.disable("error"); - THROW(); - } - } - - JSAtom *atom = js_AtomizeString(f.cx, id, 0); - if (!atom) - THROW(); - Value v; - if (!obj->getProperty(f.cx, ATOM_TO_JSID(atom), &v)) - THROW(); - f.regs.sp[-2] = v; -} - template static void JS_FASTCALL DisabledSetPropIC(VMFrame &f, ic::PICInfo *pic) @@ -2089,6 +1681,10 @@ ic::SetProp(VMFrame &f, ic::PICInfo *pic) JSScript *script = f.fp()->script(); JS_ASSERT(pic->isSet()); + VoidStubPIC stub = pic->usePropCache + ? STRICT_VARIANT(DisabledSetPropIC) + : STRICT_VARIANT(DisabledSetPropICNoCache); + // // Important: We update the PIC before looking up the property so that the // PIC is updated only if the property already exists. The PIC doesn't try @@ -2096,16 +1692,12 @@ ic::SetProp(VMFrame &f, ic::PICInfo *pic) // // Also note, we can't use SetName for PROPINC PICs because the property // cache can't handle a GET and SET from the same scripted PC. - // + if (pic->shouldUpdate(f.cx)) { - VoidStubPIC stub = pic->usePropCache - ? STRICT_VARIANT(DisabledSetPropIC) - : STRICT_VARIANT(DisabledSetPropICNoCache); - - SetPropCompiler cc(f, script, obj, *pic, pic->atom, stub); - if (!cc.update()) { - cc.disable("error"); - THROW(); + SetPropCompiler cc(f, script, obj, *pic, pic->atom, stub); + LookupStatus status = cc.update(); + if (status == Lookup_Error) + THROW(); } Value rval = f.regs.sp[-1]; @@ -2154,8 +1746,6 @@ ic::CallProp(VMFrame &f, ic::PICInfo *pic) JSObject *aobj = js_GetProtoIfDenseArray(&objv.toObject()); Value rval; - bool usePIC = true; - PropertyCacheEntry *entry; JSObject *obj2; JSAtom *atom; @@ -2208,22 +1798,18 @@ ic::CallProp(VMFrame &f, ic::PICInfo *pic) } GetPropCompiler cc(f, script, &objv.toObject(), *pic, pic->atom, DisabledCallPropIC); - if (usePIC) { - if (lval.isObject()) { - if (!cc.update()) { - cc.disable("error"); + if (lval.isObject()) { + if (pic->shouldUpdate(cx)) { + LookupStatus status = cc.update(); + if (status == Lookup_Error) THROW(); - } - } else if (lval.isString()) { - if (!cc.generateStringCallStub()) { - cc.disable("error"); - THROW(); - } - } else { - cc.disable("non-string primitive"); } + } else if (lval.isString()) { + LookupStatus status = cc.generateStringCallStub(); + if (status == Lookup_Error) + THROW(); } else { - cc.disable("wrapped primitive"); + cc.disable("non-string primitive"); } #if JS_HAS_NO_SUCH_METHOD @@ -2257,10 +1843,9 @@ ic::XName(VMFrame &f, ic::PICInfo *pic) ScopeNameCompiler cc(f, script, obj, *pic, pic->atom, DisabledXNameIC); - if (!cc.updateForXName()) { - cc.disable("error"); + LookupStatus status = cc.updateForXName(); + if (status == Lookup_Error) THROW(); - } Value rval; if (!cc.retrieve(&rval)) @@ -2275,10 +1860,9 @@ ic::Name(VMFrame &f, ic::PICInfo *pic) ScopeNameCompiler cc(f, script, &f.fp()->scopeChain(), *pic, pic->atom, DisabledNameIC); - if (!cc.updateForName()) { - cc.disable("error"); + LookupStatus status = cc.updateForName(); + if (status == Lookup_Error) THROW(); - } Value rval; if (!cc.retrieve(&rval)) @@ -2317,6 +1901,308 @@ ic::BindName(VMFrame &f, ic::PICInfo *pic) f.regs.sp[0].setObject(*obj); } +bool +BaseIC::isCallOp() +{ + return !!(js_CodeSpec[op].format & JOF_CALLOP); +} + +void +BaseIC::spew(JSContext *cx, const char *event, const char *message) +{ +#ifdef JS_METHODJIT_SPEW + JaegerSpew(JSpew_PICs, "%s %s: %s (%s: %d)\n", + js_CodeName[op], event, message, cx->fp()->script()->filename, + js_FramePCToLineNumber(cx, cx->fp())); +#endif +} + +LookupStatus +BaseIC::disable(JSContext *cx, const char *reason, void *stub) +{ + spew(cx, "disabled", reason); + RepatchBuffer repatcher(slowPathStart.executableAddress(), INLINE_PATH_LENGTH); + repatcher.relink(slowPathCall, FunctionPtr(stub)); + return Lookup_Uncacheable; +} + +bool +BaseIC::shouldUpdate(JSContext *cx) +{ + if (!hit) { + hit = true; + spew(cx, "ignored", "first hit"); + return false; + } + JS_ASSERT(stubsGenerated < MAX_PIC_STUBS); + return true; +} + +static void JS_FASTCALL +DisabledGetElem(VMFrame &f, ic::GetElementIC *ic) +{ + stubs::GetElem(f); +} + +bool +GetElementIC::shouldUpdate(JSContext *cx) +{ + if (!hit) { + hit = true; + spew(cx, "ignored", "first hit"); + return false; + } + JS_ASSERT(stubsGenerated < MAX_GETELEM_IC_STUBS); + return true; +} + +LookupStatus +GetElementIC::disable(JSContext *cx, const char *reason) +{ + slowCallPatched = true; + BaseIC::disable(cx, reason, JS_FUNC_TO_DATA_PTR(void *, DisabledGetElem)); + return Lookup_Uncacheable; +} + +LookupStatus +GetElementIC::error(JSContext *cx) +{ + disable(cx, "error"); + return Lookup_Error; +} + +void +GetElementIC::purge() +{ + if (inlineTypeGuardPatched || inlineClaspGuardPatched) { + RepatchBuffer repatcher(fastPathStart.executableAddress(), INLINE_PATH_LENGTH); + + // Repatch the inline jumps. + if (inlineTypeGuardPatched) + repatcher.relink(fastPathStart.jumpAtOffset(inlineTypeGuard), slowPathStart); + if (inlineClaspGuardPatched) + repatcher.relink(fastPathStart.jumpAtOffset(inlineClaspGuard), slowPathStart); + } + + if (slowCallPatched) { + RepatchBuffer repatcher(slowPathStart.executableAddress(), INLINE_PATH_LENGTH); + repatcher.relink(slowPathCall, FunctionPtr(JS_FUNC_TO_DATA_PTR(void *, ic::GetElement))); + } + + reset(); +} + +LookupStatus +GetElementIC::attachGetProp(JSContext *cx, JSObject *obj, const Value &v, jsid id, Value *vp) +{ + JS_ASSERT(v.isString()); + + GetPropertyHelper getprop(cx, obj, JSID_TO_ATOM(id), *this); + LookupStatus status = getprop.lookupAndTest(); + if (status != Lookup_Cacheable) + return status; + + Assembler masm; + + // Guard on the string's type and identity. + MaybeJump atomTypeGuard; + if (hasInlineTypeGuard() && !inlineTypeGuardPatched) { + // We link all string-key dependent stubs together, and store the + // first set of guards in the IC, separately, from int-key dependent + // stubs. As long as we guarantee that the first string-key dependent + // stub guards on the key type, then all other string-key stubs can + // omit the guard. + JS_ASSERT(!idRemat.isTypeKnown()); + atomTypeGuard = masm.testString(Assembler::NotEqual, typeReg); + } else { + // If there was no inline type guard, then a string type is guaranteed. + // Otherwise, we are guaranteed the type has already been checked, via + // the comment above. + JS_ASSERT_IF(!hasInlineTypeGuard(), idRemat.knownType() == JSVAL_TYPE_STRING); + } + + // Reify the shape before guards that could flow into shape guarding stubs. + if (!obj->isDenseArray() && !typeRegHasBaseShape) { + masm.loadShape(objReg, typeReg); + typeRegHasBaseShape = true; + } + + MaybeJump atomIdGuard; + if (!idRemat.isConstant()) + atomIdGuard = masm.branchPtr(Assembler::NotEqual, idRemat.dataReg(), ImmPtr(v.toString())); + + // Guard on the base shape (or in the dense array case, the clasp). + Jump shapeGuard; + if (obj->isDenseArray()) { + shapeGuard = masm.testObjClass(Assembler::NotEqual, objReg, obj->getClass()); + } else { + shapeGuard = masm.branch32(Assembler::NotEqual, typeReg, Imm32(obj->shape())); + } + + // Guard on the prototype, if applicable. + MaybeJump protoGuard; + JSObject *holder = getprop.holder; + RegisterID holderReg = objReg; + if (obj != holder) { + // Bake in the holder identity. Careful not to clobber |objReg|, since we can't remat it. + holderReg = typeReg; + masm.move(ImmPtr(holder), holderReg); + typeRegHasBaseShape = false; + + // Guard on the holder's shape. + protoGuard = masm.guardShape(holderReg, holder); + } + + // Load the value. + const Shape *shape = getprop.shape; + masm.loadObjProp(holder, holderReg, shape, typeReg, objReg); + + Jump done = masm.jump(); + + PICLinker buffer(cx, *this); + if (!buffer.init(masm)) + return error(cx); + + // Patch all guards. + buffer.maybeLink(atomIdGuard, slowPathStart); + buffer.maybeLink(atomTypeGuard, slowPathStart); + buffer.link(shapeGuard, slowPathStart); + buffer.maybeLink(protoGuard, slowPathStart); + buffer.link(done, fastPathRejoin); + + CodeLocationLabel cs = buffer.finalizeCodeAddendum(); +#if DEBUG + char *chars = js_DeflateString(cx, v.toString()->chars(), v.toString()->length()); + JaegerSpew(JSpew_PICs, "generated getelem stub at %p for atom 0x%x (\"%s\") shape 0x%x (%s: %d)\n", + cs.executableAddress(), id, chars, holder->shape(), cx->fp()->script()->filename, + js_FramePCToLineNumber(cx, cx->fp())); + cx->free(chars); +#endif + + // Update the inline guards, if needed. + if (shouldPatchInlineTypeGuard() || shouldPatchUnconditionalClaspGuard()) { + PICRepatchBuffer repatcher(*this, fastPathStart); + + if (shouldPatchInlineTypeGuard()) { + // A type guard is present in the inline path, and this is the + // first string stub, so patch it now. + JS_ASSERT(!inlineTypeGuardPatched); + JS_ASSERT(atomTypeGuard.isSet()); + + repatcher.relink(inlineTypeGuard, cs); + inlineTypeGuardPatched = true; + } + + if (shouldPatchUnconditionalClaspGuard()) { + // The clasp guard is unconditional, meaning there is no type + // check. This is the first stub, so it has to be patched. Note + // that it is wrong to patch the inline clasp guard otherwise, + // because it follows an integer-id guard. + JS_ASSERT(!hasInlineTypeGuard()); + + repatcher.relink(inlineClaspGuard, cs); + inlineClaspGuardPatched = true; + } + } + + // If there were previous stub guards, patch them now. + if (hasLastStringStub) { + PICRepatchBuffer repatcher(*this, lastStringStub); + if (atomGuard) + repatcher.relink(atomGuard, cs); + repatcher.relink(firstShapeGuard, cs); + if (secondShapeGuard) + repatcher.relink(secondShapeGuard, cs); + } + + // Update state. + hasLastStringStub = true; + lastStringStub = cs; + if (atomIdGuard.isSet()) { + atomGuard = buffer.locationOf(atomIdGuard.get()) - cs; + JS_ASSERT(atomGuard == buffer.locationOf(atomIdGuard.get()) - cs); + JS_ASSERT(atomGuard); + } else { + atomGuard = 0; + } + if (protoGuard.isSet()) { + secondShapeGuard = buffer.locationOf(protoGuard.get()) - cs; + JS_ASSERT(secondShapeGuard == buffer.locationOf(protoGuard.get()) - cs); + JS_ASSERT(secondShapeGuard); + } else { + secondShapeGuard = 0; + } + firstShapeGuard = buffer.locationOf(shapeGuard) - cs; + JS_ASSERT(firstShapeGuard == buffer.locationOf(shapeGuard) - cs); + JS_ASSERT(firstShapeGuard); + + stubsGenerated++; + + if (stubsGenerated == MAX_GETELEM_IC_STUBS) + disable(cx, "max stubs reached"); + + // Finally, fetch the value to avoid redoing the property lookup. + if (shape->isMethod()) + *vp = ObjectValue(shape->methodObject()); + else + *vp = holder->getSlot(shape->slot); + + return Lookup_Cacheable; +} + +LookupStatus +GetElementIC::update(JSContext *cx, JSObject *obj, const Value &v, jsid id, Value *vp) +{ + if (v.isString()) + return attachGetProp(cx, obj, v, id, vp); + return disable(cx, "unhandled object and key type"); +} + +void JS_FASTCALL +ic::GetElement(VMFrame &f, ic::GetElementIC *ic) +{ + JSContext *cx = f.cx; + + // Right now, we don't optimize for strings. + if (!f.regs.sp[-2].isObject()) { + ic->disable(cx, "non-object"); + stubs::GetElem(f); + return; + } + + JSObject *obj = ValueToObject(cx, &f.regs.sp[-2]); + if (!obj) + THROW(); + + Value idval = f.regs.sp[-1]; + + jsid id; + if (idval.isInt32() && INT_FITS_IN_JSID(idval.toInt32())) { + id = INT_TO_JSID(idval.toInt32()); + } else { + if (!js_InternNonIntElementId(cx, obj, idval, &id)) + THROW(); + } + + if (ic->shouldUpdate(cx)) { +#ifdef DEBUG + f.regs.sp[-2] = MagicValue(JS_GENERIC_MAGIC); +#endif + LookupStatus status = ic->update(cx, obj, idval, id, &f.regs.sp[-2]); + if (status != Lookup_Uncacheable) { + if (status == Lookup_Error) + THROW(); + + // If the result can be cached, the value was already retrieved. + JS_ASSERT(!f.regs.sp[-2].isMagic()); + return; + } + } + + if (!obj->getProperty(cx, id, &f.regs.sp[-2])) + THROW(); +} + void JITScript::purgePICs() { @@ -2338,15 +2224,15 @@ JITScript::purgePICs() case ic::PICInfo::GET: GetPropCompiler::reset(pic); break; - case ic::PICInfo::GETELEM: - GetElemCompiler::reset(pic); - break; default: JS_NOT_REACHED("Unhandled PIC kind"); break; } pic.reset(); } + + for (uint32 i = 0; i < nGetElems; i++) + getElems[i].purge(); } void diff --git a/js/src/methodjit/PolyIC.h b/js/src/methodjit/PolyIC.h index 774a771047d..48bc8abc918 100644 --- a/js/src/methodjit/PolyIC.h +++ b/js/src/methodjit/PolyIC.h @@ -37,7 +37,7 @@ * * ***** END LICENSE BLOCK ***** */ -#if !defined jsjaeger_poly_ic_h__ && defined JS_METHODJIT && defined JS_POLYIC +#if !defined jsjaeger_poly_ic_h__ && defined JS_METHODJIT #define jsjaeger_poly_ic_h__ #include "jscntxt.h" @@ -46,7 +46,9 @@ #include "assembler/assembler/MacroAssembler.h" #include "assembler/assembler/CodeLocation.h" #include "methodjit/MethodJIT.h" +#include "BaseAssembler.h" #include "RematInfo.h" +#include "BaseCompiler.h" namespace js { namespace mjit { @@ -54,6 +56,7 @@ namespace ic { /* Maximum number of stubs for a given callsite. */ static const uint32 MAX_PIC_STUBS = 16; +static const uint32 MAX_GETELEM_IC_STUBS = 17; /* SetPropCompiler */ #if defined JS_CPU_X86 @@ -159,26 +162,6 @@ union PICLabels { int32 stubShapeJump : 8; } getprop; - /* GetElemCompiler */ - struct { - /* Offset from storeBack to beginning of 'mov dslots, addr' */ - int32 dslotsLoadOffset : 8; - - /* Offset from shapeGuard to end of shape comparison. */ - int32 inlineShapeOffset : 8; - - /* Offset from shapeGuard to end of atom comparison. */ - int32 inlineAtomOffset : 8; - - /* Offset from storeBack to end of value load. */ - int32 inlineValueOffset : 8; - - /* Offset from lastStubStart to end of shape jump. */ - // TODO: We can redefine the location of lastStubStart to be - // after the jump -- at which point this is always 0. - int32 stubShapeJump : 8; - } getelem; - /* BindNameCompiler */ struct { /* Offset from shapeGuard to end of shape jump. */ @@ -187,21 +170,27 @@ union PICLabels { }; #endif -struct BaseIC { +enum LookupStatus { + Lookup_Error = 0, + Lookup_Uncacheable, + Lookup_Cacheable +}; + +struct BaseIC : public MacroAssemblerTypedefs { // Address of inline fast-path. - JSC::CodeLocationLabel fastPathStart; + CodeLocationLabel fastPathStart; // Address to rejoin to the fast-path. - JSC::CodeLocationLabel fastPathRejoin; + CodeLocationLabel fastPathRejoin; // Start of the slow path. - JSC::CodeLocationLabel slowPathStart; + CodeLocationLabel slowPathStart; // Slow path stub call. - JSC::CodeLocationCall slowPathCall; + CodeLocationCall slowPathCall; // Address of the start of the last generated stub, if any. - JSC::CodeLocationLabel lastStubStart; + CodeLocationLabel lastStubStart; typedef Vector ExecPoolVector; @@ -210,16 +199,25 @@ struct BaseIC { // Return the start address of the last path in this PIC, which is the // inline path if no stubs have been generated yet. - JSC::CodeLocationLabel lastPathStart() { + CodeLocationLabel lastPathStart() { return stubsGenerated > 0 ? lastStubStart : fastPathStart; } // Whether or not the callsite has been hit at least once. bool hit : 1; + bool slowCallPatched : 1; // Number of stubs generated. uint32 stubsGenerated : 5; + // Offset from start of stub to jump target of second shape guard as Nitro + // asm data location. This is 0 if there is only one shape guard in the + // last stub. + int secondShapeGuard : 11; + + // Opcode this was compiled for. + JSOp op : 8; + // Release ExecutablePools referred to by this PIC. void releasePools() { for (JSC::ExecutablePool **pExecPool = execPools.begin(); @@ -229,17 +227,122 @@ struct BaseIC { } } + void init() { + new (&execPools) ExecPoolVector(SystemAllocPolicy()); + } + void finish() { + releasePools(); + this->~BaseIC(); + } + void reset() { hit = false; + slowCallPatched = false; stubsGenerated = 0; + secondShapeGuard = 0; releasePools(); execPools.clear(); } + bool shouldUpdate(JSContext *cx); + void spew(JSContext *cx, const char *event, const char *reason); + LookupStatus disable(JSContext *cx, const char *reason, void *stub); + bool isCallOp(); +}; + +struct GetElementIC : public BaseIC { + // On stub entry: + // If hasInlineTypeCheck() is true, and inlineTypeCheckPatched is false, + // - typeReg contains the type of the |id| parameter. + // If hasInlineTypeCheck() is true, and inlineTypeCheckPatched is true, + // - typeReg contains the shape of |objReg| iff typeRegHasBaseShape + // is true. + // Otherwise, typeReg is garbage. + // + // On stub exit, typeReg must contain the type of the result value. + RegisterID typeReg : 5; + + // On stub entry, objReg contains the object pointer for the |obj| parameter. + // On stub exit, objReg must contain the payload of the result value. + RegisterID objReg : 5; + + // Offset from the fast path to the inline type check. + // This is only set if hasInlineTypeCheck() is true. + unsigned inlineTypeGuard : 8; + + // Offset from the fast path to the inline clasp guard. This is always + // set; if |id| is known to not be int32, then it's an unconditional + // jump to the slow path. + unsigned inlineClaspGuard : 8; + + // This is usable if hasInlineTypeGuard() returns true, which implies + // that a dense array fast path exists. The inline type guard serves as + // the head of the chain of all string-based element stubs. + bool inlineTypeGuardPatched : 1; + + // This is always usable, and specifies whether the inline clasp guard + // has been patched. If hasInlineTypeGuard() is true, it guards against + // a dense array, and guarantees the inline type guard has passed. + // Otherwise, there is no inline type guard, and the clasp guard is just + // an unconditional jump. + bool inlineClaspGuardPatched : 1; + + //////////////////////////////////////////// + // State for string-based property stubs. // + //////////////////////////////////////////// + + // True if typeReg is guaranteed to have the shape of objReg. + bool typeRegHasBaseShape : 1; + + // These offsets are used for string-key dependent stubs, such as named + // property accesses. They are separated from the int-key dependent stubs, + // in order to guarantee that the id type needs only one guard per type. + int atomGuard : 8; // optional, non-zero if present + int firstShapeGuard : 8; // always set + int secondShapeGuard : 8; // optional, non-zero if present + + bool hasLastStringStub : 1; + CodeLocationLabel lastStringStub; + + // A limited ValueRemat instance. It may contains either: + // 1) A constant, or + // 2) A known type and data reg, or + // 3) A data reg. + // The sync bits are not set, and the type reg is never set and should not + // be used, as it is encapsulated more accurately in |typeReg|. Also, note + // carefully that the data reg is immutable. + ValueRemat idRemat; + + bool hasInlineTypeGuard() const { + return !idRemat.isTypeKnown(); + } + bool shouldPatchInlineTypeGuard() { + return hasInlineTypeGuard() && !inlineTypeGuardPatched; + } + bool shouldPatchUnconditionalClaspGuard() { + return !hasInlineTypeGuard() && !inlineClaspGuardPatched; + } + + void init() { + BaseIC::init(); + reset(); + } + void reset() { + BaseIC::reset(); + inlineTypeGuardPatched = false; + inlineClaspGuardPatched = false; + typeRegHasBaseShape = false; + hasLastStringStub = false; + } + void purge(); + LookupStatus update(JSContext *cx, JSObject *obj, const Value &v, jsid id, Value *vp); + LookupStatus attachGetProp(JSContext *cx, JSObject *obj, const Value &v, jsid id, + Value *vp); + LookupStatus disable(JSContext *cx, const char *reason); + LookupStatus error(JSContext *cx); + bool shouldUpdate(JSContext *cx); }; struct PICInfo : public BaseIC { - typedef JSC::MacroAssembler::RegisterID RegisterID; - // Operation this is a PIC for. enum Kind #ifdef _MSC_VER @@ -252,7 +355,6 @@ struct PICInfo : public BaseIC { SETMETHOD, // JSOP_SETMETHOD NAME, // JSOP_NAME BIND, // JSOP_BINDNAME - GETELEM, // JSOP_GETELEM XNAME // JSOP_GETXPROP }; @@ -263,20 +365,10 @@ struct PICInfo : public BaseIC { // Reverse offset from slowPathStart to the type check slow path. int32 typeCheckOffset; - - // Remat info for the object reg. - int32 objRemat : MIN_STATE_REMAT_BITS; - bool objNeedsRemat : 1; - RegisterID idReg : 5; // only used in GETELEM PICs. } get; ValueRemat vr; } u; - // Offset from start of stub to jump target of second shape guard as Nitro - // asm data location. This is 0 if there is only one shape guard in the - // last stub. - int secondShapeGuard : 11; - Kind kind : 3; // True if register R holds the base object shape along exits from the @@ -299,7 +391,7 @@ struct PICInfo : public BaseIC { return kind == SET || kind == SETMETHOD; } inline bool isGet() const { - return kind == GET || kind == CALL || kind == GETELEM; + return kind == GET || kind == CALL; } inline RegisterID typeReg() { JS_ASSERT(isGet()); @@ -309,14 +401,6 @@ struct PICInfo : public BaseIC { JS_ASSERT(isGet()); return u.get.hasTypeCheck; } - inline const StateRemat objRemat() const { - JS_ASSERT(isGet()); - return StateRemat::FromInt32(u.get.objRemat); - } - inline bool objNeedsRemat() { - JS_ASSERT(isGet()); - return u.get.objNeedsRemat; - } inline bool shapeNeedsRemat() { return !shapeRegHasBaseShape; } @@ -325,12 +409,6 @@ struct PICInfo : public BaseIC { return !hasTypeCheck(); } - inline void setObjRemat(const StateRemat &sr) { - JS_ASSERT(isGet()); - u.get.objRemat = sr.toInt32(); - JS_ASSERT(u.get.objRemat == sr.toInt32()); - } - #if defined JS_CPU_X64 // Required labels for platform-specific patching. PICLabels labels; @@ -342,30 +420,30 @@ struct PICInfo : public BaseIC { // Index into the script's atom table. JSAtom *atom; - bool shouldGenerate() { - return stubsGenerated < MAX_PIC_STUBS || !inlinePathPatched; + void init() { + BaseIC::init(); + reset(); } // Reset the data members to the state of a fresh PIC before any patching // or stub generation was done. void reset() { inlinePathPatched = false; - if (kind == GET || kind == CALL || kind == GETELEM) - u.get.objNeedsRemat = false; - secondShapeGuard = 0; shapeRegHasBaseShape = true; BaseIC::reset(); } }; +#ifdef JS_POLYIC void PurgePICs(JSContext *cx, JSScript *script); void JS_FASTCALL GetProp(VMFrame &f, ic::PICInfo *); -void JS_FASTCALL GetElem(VMFrame &f, ic::PICInfo *); void JS_FASTCALL SetProp(VMFrame &f, ic::PICInfo *); void JS_FASTCALL CallProp(VMFrame &f, ic::PICInfo *); void JS_FASTCALL Name(VMFrame &f, ic::PICInfo *); void JS_FASTCALL XName(VMFrame &f, ic::PICInfo *); void JS_FASTCALL BindName(VMFrame &f, ic::PICInfo *); +void JS_FASTCALL GetElement(VMFrame &f, ic::GetElementIC *); +#endif } /* namespace ic */ } /* namespace mjit */ diff --git a/js/src/methodjit/PunboxAssembler.h b/js/src/methodjit/PunboxAssembler.h index eabe3d9f3ff..0e14f5b3b90 100644 --- a/js/src/methodjit/PunboxAssembler.h +++ b/js/src/methodjit/PunboxAssembler.h @@ -93,16 +93,10 @@ class PunboxAssembler : public JSC::MacroAssembler return address; } - void loadSlot(RegisterID obj, RegisterID clobber, uint32 slot, bool inlineAccess, - RegisterID type, RegisterID data) { - JS_ASSERT(type != data); - Address address(obj, JSObject::getFixedSlotOffset(slot)); - if (!inlineAccess) { - loadPtr(Address(obj, offsetof(JSObject, slots)), clobber); - address = Address(clobber, slot * sizeof(Value)); - } - - loadValueAsComponents(address, type, data); + void loadInlineSlot(RegisterID objReg, uint32 slot, + RegisterID typeReg, RegisterID dataReg) { + Address address(objReg, JSObject::getFixedSlotOffset(slot)); + loadValueAsComponents(address, typeReg, dataReg); } template diff --git a/js/src/methodjit/StubCompiler.h b/js/src/methodjit/StubCompiler.h index 9dcf1784a1b..9d3099ac83c 100644 --- a/js/src/methodjit/StubCompiler.h +++ b/js/src/methodjit/StubCompiler.h @@ -127,6 +127,7 @@ class StubCompiler STUB_CALL_TYPE(VoidStubPC); #ifdef JS_POLYIC STUB_CALL_TYPE(VoidStubPIC); + STUB_CALL_TYPE(VoidStubGetElemIC); #endif #ifdef JS_MONOIC STUB_CALL_TYPE(VoidStubMIC);