From 79c55c581914e09813636f786de428a2c5331739 Mon Sep 17 00:00:00 2001 From: Jan de Mooij Date: Thu, 28 Jul 2011 11:53:29 +0200 Subject: [PATCH] [INFER] LICM for typed arrays, bug 671084. r=bhackett --- js/src/methodjit/FastOps.cpp | 82 +++++++++++++++------- js/src/methodjit/LoopState.cpp | 124 +++++++++++++++++++++++++-------- js/src/methodjit/LoopState.h | 25 +++++-- 3 files changed, 171 insertions(+), 60 deletions(-) diff --git a/js/src/methodjit/FastOps.cpp b/js/src/methodjit/FastOps.cpp index e9e0d777b1c..5474f4b6dbb 100644 --- a/js/src/methodjit/FastOps.cpp +++ b/js/src/methodjit/FastOps.cpp @@ -1110,7 +1110,7 @@ mjit::Compiler::jsop_setelem_dense() analyze::CrossSSAValue objv(a->inlineIndex, analysis->poppedValue(PC, 2)); analyze::CrossSSAValue indexv(a->inlineIndex, analysis->poppedValue(PC, 1)); bool hoisted = loop && id->isType(JSVAL_TYPE_INT32) && - loop->hoistArrayLengthCheck(objv, indexv); + loop->hoistArrayLengthCheck(DENSE_ARRAY, objv, indexv); if (hoisted) { FrameEntry *slotsFe = loop->invariantArraySlots(objv); @@ -1375,18 +1375,31 @@ mjit::Compiler::jsop_setelem_typed(int atype) bool pinKey = !key.isConstant() && !frame.haveSameBacking(id, value); if (pinKey) frame.pinReg(key.reg()); - RegisterID objReg = frame.copyDataIntoReg(obj); - // Get the internal typed array. - masm.loadPtr(Address(objReg, offsetof(JSObject, privateData)), objReg); + analyze::CrossSSAValue objv(a->inlineIndex, analysis->poppedValue(PC, 1)); + analyze::CrossSSAValue indexv(a->inlineIndex, analysis->poppedValue(PC, 0)); + bool hoisted = loop && id->isType(JSVAL_TYPE_INT32) && + loop->hoistArrayLengthCheck(TYPED_ARRAY, objv, indexv); - // Bounds check. - Jump lengthGuard = masm.guardArrayExtent(TypedArray::lengthOffset(), - objReg, key, Assembler::BelowOrEqual); - stubcc.linkExit(lengthGuard, Uses(3)); + RegisterID objReg; + if (hoisted) { + FrameEntry *slotsFe = loop->invariantArraySlots(objv); + objReg = frame.tempRegForData(slotsFe); + frame.pinReg(objReg); + } else { + objReg = frame.copyDataIntoReg(obj); - // Load the array's packed data vector. - masm.loadPtr(Address(objReg, js::TypedArray::dataOffset()), objReg); + // Get the internal typed array. + masm.loadPtr(Address(objReg, offsetof(JSObject, privateData)), objReg); + + // Bounds check. + Jump lengthGuard = masm.guardArrayExtent(TypedArray::lengthOffset(), + objReg, key, Assembler::BelowOrEqual); + stubcc.linkExit(lengthGuard, Uses(3)); + + // Load the array's packed data vector. + masm.loadPtr(Address(objReg, js::TypedArray::dataOffset()), objReg); + } // Unpin value so that convertForTypedArray can assign a new data // register using tempRegInMaskForData. @@ -1412,7 +1425,10 @@ mjit::Compiler::jsop_setelem_typed(int atype) } if (pinKey) frame.unpinReg(key.reg()); - frame.freeReg(objReg); + if (hoisted) + frame.unpinReg(objReg); + else + frame.freeReg(objReg); stubcc.leave(); OOL_STUBCALL(STRICT_VARIANT(stubs::SetElem), REJOIN_FALLTHROUGH); @@ -1671,7 +1687,7 @@ mjit::Compiler::jsop_getelem_dense(bool isPacked) analyze::CrossSSAValue objv(a->inlineIndex, analysis->poppedValue(PC, 1)); analyze::CrossSSAValue indexv(a->inlineIndex, analysis->poppedValue(PC, 0)); bool hoisted = loop && id->isType(JSVAL_TYPE_INT32) && - loop->hoistArrayLengthCheck(objv, indexv); + loop->hoistArrayLengthCheck(DENSE_ARRAY, objv, indexv); // Get a register with either the object or its slots, depending on whether // we are hoisting the bounds check. @@ -1861,7 +1877,31 @@ mjit::Compiler::jsop_getelem_typed(int atype) : Int32Key::FromRegister(frame.tempRegForData(id)); if (!key.isConstant()) frame.pinReg(key.reg()); - RegisterID objReg = frame.copyDataIntoReg(obj); + + analyze::CrossSSAValue objv(a->inlineIndex, analysis->poppedValue(PC, 1)); + analyze::CrossSSAValue indexv(a->inlineIndex, analysis->poppedValue(PC, 0)); + bool hoisted = loop && id->isType(JSVAL_TYPE_INT32) && + loop->hoistArrayLengthCheck(TYPED_ARRAY, objv, indexv); + + RegisterID objReg; + if (hoisted) { + FrameEntry *slotsFe = loop->invariantArraySlots(objv); + objReg = frame.tempRegForData(slotsFe); + frame.pinReg(objReg); + } else { + objReg = frame.copyDataIntoReg(obj); + + // Get the internal typed array. + masm.loadPtr(Address(objReg, offsetof(JSObject, privateData)), objReg); + + // Bounds check. + Jump lengthGuard = masm.guardArrayExtent(TypedArray::lengthOffset(), + objReg, key, Assembler::BelowOrEqual); + stubcc.linkExit(lengthGuard, Uses(2)); + + // Load the array's packed data vector. + masm.loadPtr(Address(objReg, js::TypedArray::dataOffset()), objReg); + } // We can load directly into an FP-register if the following conditions // are met: @@ -1890,21 +1930,13 @@ mjit::Compiler::jsop_getelem_typed(int atype) typeReg = frame.allocReg(); } - // Get the internal typed array. - masm.loadPtr(Address(objReg, offsetof(JSObject, privateData)), objReg); - - // Bounds check. - Jump lengthGuard = masm.guardArrayExtent(TypedArray::lengthOffset(), - objReg, key, Assembler::BelowOrEqual); - stubcc.linkExit(lengthGuard, Uses(2)); - - // Load the array's packed data vector. - masm.loadPtr(Address(objReg, js::TypedArray::dataOffset()), objReg); - // Load value from the array. masm.loadFromTypedArray(atype, objReg, key, typeReg, dataReg, tempReg); - frame.freeReg(objReg); + if (hoisted) + frame.unpinReg(objReg); + else + frame.freeReg(objReg); if (!key.isConstant()) frame.unpinReg(key.reg()); if (tempReg.isSet()) diff --git a/js/src/methodjit/LoopState.cpp b/js/src/methodjit/LoopState.cpp index 79b55a5187a..6d0f339e123 100644 --- a/js/src/methodjit/LoopState.cpp +++ b/js/src/methodjit/LoopState.cpp @@ -322,7 +322,7 @@ LoopState::entryRedundant(const InvariantEntry &e0, const InvariantEntry &e1) * initlen(array) - c1 <= c0 * NSLOTS_LIMIT <= c0 + c1 */ - if (e0.kind == InvariantEntry::RANGE_CHECK && e1.kind == InvariantEntry::BOUNDS_CHECK && + if (e0.kind == InvariantEntry::RANGE_CHECK && e1.isBoundsCheck() && value01 == value11 && value02 == value12) { int32 constant; if (c1 >= 0) @@ -334,7 +334,7 @@ LoopState::entryRedundant(const InvariantEntry &e0, const InvariantEntry &e1) /* Look for matching tests that differ only in their constants. */ if (e0.kind == e1.kind && array0 == array1 && value01 == value11 && value02 == value12) { - if (e0.kind == InvariantEntry::BOUNDS_CHECK) { + if (e0.isBoundsCheck()) { /* If e0 is X >= Y + c0 and e1 is X >= Y + c1, e0 is redundant if c0 <= c1 */ return (c0 <= c1); } else { @@ -383,23 +383,27 @@ LoopState::checkRedundantEntry(const InvariantEntry &entry) } bool -LoopState::addHoistedCheck(uint32 arraySlot, uint32 valueSlot1, uint32 valueSlot2, int32 constant) +LoopState::addHoistedCheck(InvariantArrayKind arrayKind, uint32 arraySlot, + uint32 valueSlot1, uint32 valueSlot2, int32 constant) { #ifdef DEBUG JS_ASSERT_IF(valueSlot1 == UNASSIGNED, valueSlot2 == UNASSIGNED); + const char *field = (arrayKind == DENSE_ARRAY) ? "initlen" : "length"; if (valueSlot1 == UNASSIGNED) { - JaegerSpew(JSpew_Analysis, "Hoist initlen > %d\n", constant); + JaegerSpew(JSpew_Analysis, "Hoist %s > %d\n", field, constant); } else if (valueSlot2 == UNASSIGNED) { - JaegerSpew(JSpew_Analysis, "Hoisted as initlen > %s + %d\n", + JaegerSpew(JSpew_Analysis, "Hoisted as %s > %s + %d\n", field, frame.entryName(valueSlot1), constant); } else { - JaegerSpew(JSpew_Analysis, "Hoisted as initlen > %s + %s + %d\n", + JaegerSpew(JSpew_Analysis, "Hoisted as %s > %s + %s + %d\n", field, frame.entryName(valueSlot1), frame.entryName(valueSlot2), constant); } #endif InvariantEntry entry; - entry.kind = InvariantEntry::BOUNDS_CHECK; + entry.kind = (arrayKind == DENSE_ARRAY) + ? InvariantEntry::DENSE_ARRAY_BOUNDS_CHECK + : InvariantEntry::TYPED_ARRAY_BOUNDS_CHECK; entry.u.check.arraySlot = arraySlot; entry.u.check.valueSlot1 = valueSlot1; entry.u.check.valueSlot2 = valueSlot2; @@ -415,12 +419,13 @@ LoopState::addHoistedCheck(uint32 arraySlot, uint32 valueSlot1, uint32 valueSlot * bounds check, so this makes invariantSlots infallible. */ bool hasInvariantSlots = false; + InvariantEntry::EntryKind slotsKind = (arrayKind == DENSE_ARRAY) + ? InvariantEntry::DENSE_ARRAY_SLOTS + : InvariantEntry::TYPED_ARRAY_SLOTS; for (unsigned i = 0; !hasInvariantSlots && i < invariantEntries.length(); i++) { InvariantEntry &entry = invariantEntries[i]; - if (entry.kind == InvariantEntry::INVARIANT_SLOTS && - entry.u.array.arraySlot == arraySlot) { + if (entry.kind == slotsKind && entry.u.array.arraySlot == arraySlot) hasInvariantSlots = true; - } } if (!hasInvariantSlots) { uint32 which = frame.allocTemporary(); @@ -432,7 +437,7 @@ LoopState::addHoistedCheck(uint32 arraySlot, uint32 valueSlot1, uint32 valueSlot frame.entryName(fe), frame.entryName(arraySlot)); InvariantEntry slotsEntry; - slotsEntry.kind = InvariantEntry::INVARIANT_SLOTS; + slotsEntry.kind = slotsKind; slotsEntry.u.array.arraySlot = arraySlot; slotsEntry.u.array.temporary = which; invariantEntries.append(slotsEntry); @@ -511,11 +516,12 @@ LoopState::setLoopReg(AnyRegisterID reg, FrameEntry *fe) } bool -LoopState::hoistArrayLengthCheck(const CrossSSAValue &obj, const CrossSSAValue &index) +LoopState::hoistArrayLengthCheck(InvariantArrayKind arrayKind, const CrossSSAValue &obj, + const CrossSSAValue &index) { /* * Note: this method requires that the index is definitely an integer, and - * that obj is either a dense array or not an object. + * that obj is either a dense array, a typed array or not an object. */ if (skipAnalysis) return false; @@ -540,7 +546,7 @@ LoopState::hoistArrayLengthCheck(const CrossSSAValue &obj, const CrossSSAValue & * bounds check fails. */ TypeSet *objTypes = ssa->getValueTypes(obj); - if (!growArrays.empty()) { + if (arrayKind == DENSE_ARRAY && !growArrays.empty()) { unsigned count = objTypes->getObjectCount(); for (unsigned i = 0; i < count; i++) { if (objTypes->getSingleObject(i) != NULL) { @@ -568,12 +574,12 @@ LoopState::hoistArrayLengthCheck(const CrossSSAValue &obj, const CrossSSAValue & if (indexSlot == UNASSIGNED) { /* Hoist checks on x[n] accesses for constant n. */ - return addHoistedCheck(objSlot, UNASSIGNED, UNASSIGNED, indexConstant); + return addHoistedCheck(arrayKind, objSlot, UNASSIGNED, UNASSIGNED, indexConstant); } if (loopInvariantEntry(indexSlot)) { /* Hoist checks on x[y] accesses when y is loop invariant. */ - return addHoistedCheck(objSlot, indexSlot, UNASSIGNED, indexConstant); + return addHoistedCheck(arrayKind, objSlot, indexSlot, UNASSIGNED, indexConstant); } /* @@ -609,7 +615,7 @@ LoopState::hoistArrayLengthCheck(const CrossSSAValue &obj, const CrossSSAValue & */ addNegativeCheck(indexSlot, indexConstant); - return addHoistedCheck(objSlot, testRHS, UNASSIGNED, constant); + return addHoistedCheck(arrayKind, objSlot, testRHS, UNASSIGNED, constant); } /* @@ -629,7 +635,7 @@ LoopState::hoistArrayLengthCheck(const CrossSSAValue &obj, const CrossSSAValue & return false; addNegativeCheck(indexSlot, indexConstant); - return addHoistedCheck(objSlot, indexSlot, testLHS, constant); + return addHoistedCheck(arrayKind, objSlot, indexSlot, testLHS, constant); } JaegerSpew(JSpew_Analysis, "No match found\n"); @@ -738,9 +744,15 @@ LoopState::invariantArraySlots(const CrossSSAValue &obj) return NULL; } + /* + * Note: we don't have to check arrayKind (dense array or typed array) here, + * because an array cannot have entries for both dense array slots and typed + * array slots. + */ for (unsigned i = 0; i < invariantEntries.length(); i++) { InvariantEntry &entry = invariantEntries[i]; - if (entry.kind == InvariantEntry::INVARIANT_SLOTS && + if ((entry.kind == InvariantEntry::DENSE_ARRAY_SLOTS || + entry.kind == InvariantEntry::TYPED_ARRAY_SLOTS) && entry.u.array.arraySlot == objSlot) { return frame.getTemporary(entry.u.array.temporary); } @@ -815,9 +827,15 @@ LoopState::invariantLength(const CrossSSAValue &obj) return fe; } + /* + * Note: we don't have to check arrayKind (dense array or typed array) here, + * because an array cannot have entries for both dense array length and typed + * array length. + */ for (unsigned i = 0; i < invariantEntries.length(); i++) { InvariantEntry &entry = invariantEntries[i]; - if (entry.kind == InvariantEntry::INVARIANT_LENGTH && + if ((entry.kind == InvariantEntry::DENSE_ARRAY_LENGTH || + entry.kind == InvariantEntry::TYPED_ARRAY_LENGTH) && entry.u.array.arraySlot == objSlot) { return frame.getTemporary(entry.u.array.temporary); } @@ -826,6 +844,28 @@ LoopState::invariantLength(const CrossSSAValue &obj) if (!loopInvariantEntry(objSlot)) return NULL; + /* Hoist 'length' access on typed arrays. */ + if (!objTypes->hasObjectFlags(cx, OBJECT_FLAG_NON_TYPED_ARRAY)) { + /* Recompile if object type changes. */ + objTypes->addFreeze(cx); + + uint32 which = frame.allocTemporary(); + if (which == uint32(-1)) + return NULL; + FrameEntry *fe = frame.getTemporary(which); + + JaegerSpew(JSpew_Analysis, "Using %s for loop invariant typed array length of %s\n", + frame.entryName(fe), frame.entryName(objSlot)); + + InvariantEntry entry; + entry.kind = InvariantEntry::TYPED_ARRAY_LENGTH; + entry.u.array.arraySlot = objSlot; + entry.u.array.temporary = which; + invariantEntries.append(entry); + + return fe; + } + if (objTypes->hasObjectFlags(cx, OBJECT_FLAG_NON_DENSE_ARRAY)) return NULL; @@ -850,11 +890,11 @@ LoopState::invariantLength(const CrossSSAValue &obj) return NULL; FrameEntry *fe = frame.getTemporary(which); - JaegerSpew(JSpew_Analysis, "Using %s for loop invariant length of %s\n", + JaegerSpew(JSpew_Analysis, "Using %s for loop invariant dense array length of %s\n", frame.entryName(fe), frame.entryName(objSlot)); InvariantEntry entry; - entry.kind = InvariantEntry::INVARIANT_LENGTH; + entry.kind = InvariantEntry::DENSE_ARRAY_LENGTH; entry.u.array.arraySlot = objSlot; entry.u.array.temporary = which; invariantEntries.append(entry); @@ -1267,13 +1307,19 @@ LoopState::restoreInvariants(jsbytecode *pc, Assembler &masm, const InvariantEntry &entry = invariantEntries[i]; switch (entry.kind) { - case InvariantEntry::BOUNDS_CHECK: { + case InvariantEntry::DENSE_ARRAY_BOUNDS_CHECK: + case InvariantEntry::TYPED_ARRAY_BOUNDS_CHECK: { /* * Hoisted bounds checks always have preceding invariant slots * in the invariant list, so don't recheck this is an object. */ masm.loadPayload(frame.addressOf(entry.u.check.arraySlot), T0); - masm.load32(Address(T0, offsetof(JSObject, initializedLength)), T0); + if (entry.kind == InvariantEntry::DENSE_ARRAY_BOUNDS_CHECK) { + masm.load32(Address(T0, offsetof(JSObject, initializedLength)), T0); + } else { + masm.loadPtr(Address(T0, offsetof(JSObject, privateData)), T0); + masm.load32(Address(T0, TypedArray::lengthOffset()), T0); + } int32 constant = entry.u.check.constant; @@ -1333,27 +1379,49 @@ LoopState::restoreInvariants(jsbytecode *pc, Assembler &masm, break; } - case InvariantEntry::INVARIANT_SLOTS: - case InvariantEntry::INVARIANT_LENGTH: { + case InvariantEntry::DENSE_ARRAY_SLOTS: + case InvariantEntry::DENSE_ARRAY_LENGTH: { uint32 array = entry.u.array.arraySlot; Jump notObject = masm.testObject(Assembler::NotEqual, frame.addressOf(array)); jumps->append(notObject); masm.loadPayload(frame.addressOf(array), T0); - uint32 offset = (entry.kind == InvariantEntry::INVARIANT_SLOTS) + uint32 offset = (entry.kind == InvariantEntry::DENSE_ARRAY_SLOTS) ? JSObject::offsetOfSlots() : offsetof(JSObject, privateData); Address address = frame.addressOf(frame.getTemporary(entry.u.array.temporary)); masm.loadPtr(Address(T0, offset), T0); - if (entry.kind == InvariantEntry::INVARIANT_LENGTH) + if (entry.kind == InvariantEntry::DENSE_ARRAY_LENGTH) masm.storeValueFromComponents(ImmType(JSVAL_TYPE_INT32), T0, address); else masm.storePtr(T0, address); break; } + case InvariantEntry::TYPED_ARRAY_SLOTS: + case InvariantEntry::TYPED_ARRAY_LENGTH: { + uint32 array = entry.u.array.arraySlot; + Jump notObject = masm.testObject(Assembler::NotEqual, frame.addressOf(array)); + jumps->append(notObject); + masm.loadPayload(frame.addressOf(array), T0); + + Address address = frame.addressOf(frame.getTemporary(entry.u.array.temporary)); + + /* Load the internal typed array. */ + masm.loadPtr(Address(T0, offsetof(JSObject, privateData)), T0); + + if (entry.kind == InvariantEntry::TYPED_ARRAY_LENGTH) { + masm.load32(Address(T0, TypedArray::lengthOffset()), T0); + masm.storeValueFromComponents(ImmType(JSVAL_TYPE_INT32), T0, address); + } else { + masm.loadPtr(Address(T0, js::TypedArray::dataOffset()), T0); + masm.storePtr(T0, address); + } + break; + } + case InvariantEntry::INVARIANT_ARGS_BASE: { Address address = frame.addressOf(frame.getTemporary(entry.u.array.temporary)); masm.loadFrameActuals(outerScript->fun, T0); diff --git a/js/src/methodjit/LoopState.h b/js/src/methodjit/LoopState.h index 55c3ce0e61b..baf1fcde025 100644 --- a/js/src/methodjit/LoopState.h +++ b/js/src/methodjit/LoopState.h @@ -87,6 +87,8 @@ namespace mjit { struct TemporaryCopy; +enum InvariantArrayKind { DENSE_ARRAY, TYPED_ARRAY }; + class LoopState : public MacroAssemblerTypedefs { JSContext *cx; @@ -159,12 +161,13 @@ class LoopState : public MacroAssemblerTypedefs * on each other and we need to emit code restoring them in order. */ struct InvariantEntry { - enum { + enum EntryKind { /* * initializedLength(array) > value1 + value2 + constant. * Unsigned comparison, so will fail if value + constant < 0 */ - BOUNDS_CHECK, + DENSE_ARRAY_BOUNDS_CHECK, + TYPED_ARRAY_BOUNDS_CHECK, /* value1 + constant >= 0 */ NEGATIVE_CHECK, @@ -173,8 +176,12 @@ class LoopState : public MacroAssemblerTypedefs RANGE_CHECK, /* For dense arrays */ - INVARIANT_SLOTS, - INVARIANT_LENGTH, + DENSE_ARRAY_SLOTS, + DENSE_ARRAY_LENGTH, + + /* For typed arrays */ + TYPED_ARRAY_SLOTS, + TYPED_ARRAY_LENGTH, /* For lazy arguments */ INVARIANT_ARGS_BASE, @@ -202,8 +209,11 @@ class LoopState : public MacroAssemblerTypedefs } property; } u; InvariantEntry() { PodZero(this); } + bool isBoundsCheck() const { + return kind == DENSE_ARRAY_BOUNDS_CHECK || kind == TYPED_ARRAY_BOUNDS_CHECK; + } bool isCheck() const { - return kind == BOUNDS_CHECK || kind == NEGATIVE_CHECK || kind == RANGE_CHECK; + return isBoundsCheck() || kind == NEGATIVE_CHECK || kind == RANGE_CHECK; } }; Vector invariantEntries; @@ -212,7 +222,7 @@ class LoopState : public MacroAssemblerTypedefs bool checkRedundantEntry(const InvariantEntry &entry); bool loopInvariantEntry(uint32 slot); - bool addHoistedCheck(uint32 arraySlot, + bool addHoistedCheck(InvariantArrayKind arrayKind, uint32 arraySlot, uint32 valueSlot1, uint32 valueSlot2, int32 constant); void addNegativeCheck(uint32 valueSlot, int32 constant); void addRangeCheck(uint32 valueSlot1, uint32 valueSlot2, int32 constant); @@ -280,7 +290,8 @@ class LoopState : public MacroAssemblerTypedefs * These should only be used for entries which are known to be dense arrays * (if they are objects at all). */ - bool hoistArrayLengthCheck(const analyze::CrossSSAValue &obj, + bool hoistArrayLengthCheck(InvariantArrayKind arrayKind, + const analyze::CrossSSAValue &obj, const analyze::CrossSSAValue &index); FrameEntry *invariantArraySlots(const analyze::CrossSSAValue &obj);