Bug 822077; specialise PushRegs for ARM using STM; r=mjrosenb

--HG--
extra : rebase_source : f5eeb006a70ed9ef09adc0382943d6c0932615dc

js/src/ion/IonMacroAssembler.cpp

@@ -73,40 +73,84 @@ template void MacroAssembler::guardTypeSet(const ValueOperand &value, const type
 void
 MacroAssembler::PushRegsInMask(RegisterSet set)
 {
-    size_t diff = set.gprs().size() * STACK_SLOT_SIZE +
-                  set.fpus().size() * sizeof(double);
-
-    reserveStack(diff);
+    int32_t diffF = set.fpus().size() * sizeof(double);
+    int32_t diffG = set.gprs().size() * STACK_SLOT_SIZE;
 
+    reserveStack(diffG);
+#ifdef JS_CPU_ARM
+    if (set.gprs().size() > 1) {
+        startDataTransferM(IsStore, StackPointer, IA, NoWriteBack);
         for (GeneralRegisterIterator iter(set.gprs()); iter.more(); iter++) {
-        diff -= STACK_SLOT_SIZE;
-        storePtr(*iter, Address(StackPointer, diff));
+            diffG -= STACK_SLOT_SIZE;
+            transferReg(*iter);
         }
+        finishDataTransfer();
+    } else
+#endif
+    {
+        for (GeneralRegisterIterator iter(set.gprs()); iter.more(); iter++) {
+            diffG -= STACK_SLOT_SIZE;
+            storePtr(*iter, Address(StackPointer, diffG));
+        }
+    }
+    JS_ASSERT(diffG == 0);
+
+    reserveStack(diffF);
+#ifdef JS_CPU_ARM
+    diffF -= transferMultipleByRuns(set.fpus(), IsStore, StackPointer, IA);
+#else
     for (FloatRegisterIterator iter(set.fpus()); iter.more(); iter++) {
-        diff -= sizeof(double);
-        storeDouble(*iter, Address(StackPointer, diff));
+        diffF -= sizeof(double);
+        storeDouble(*iter, Address(StackPointer, diffF));
     }
+#endif
+    JS_ASSERT(diffF == 0);
 }
 
 void
 MacroAssembler::PopRegsInMaskIgnore(RegisterSet set, RegisterSet ignore)
 {
-    size_t diff = set.gprs().size() * STACK_SLOT_SIZE +
-                  set.fpus().size() * sizeof(double);
-    size_t reserved = diff;
+    int32_t diffG = set.gprs().size() * STACK_SLOT_SIZE;
+    int32_t diffF = set.fpus().size() * sizeof(double);
+    const int32_t reservedG = diffG;
+    const int32_t reservedF = diffF;
 
-    for (GeneralRegisterIterator iter(set.gprs()); iter.more(); iter++) {
-        diff -= STACK_SLOT_SIZE;
-        if (!ignore.has(*iter))
-            loadPtr(Address(StackPointer, diff), *iter);
-    }
+#ifdef JS_CPU_ARM
+    // ARM can load multiple registers at once, but only if we want back all
+    // the registers we previously saved to the stack.
+    if (ignore.empty(true)) {
+        diffF -= transferMultipleByRuns(set.fpus(), IsLoad, StackPointer, IA);
+    } else
+#endif
+    {
         for (FloatRegisterIterator iter(set.fpus()); iter.more(); iter++) {
-        diff -= sizeof(double);
+            diffF -= sizeof(double);
             if (!ignore.has(*iter))
-            loadDouble(Address(StackPointer, diff), *iter);
+                loadDouble(Address(StackPointer, diffF), *iter);
         }
+    }
+    freeStack(reservedF);
+    JS_ASSERT(diffF == 0);
 
-    freeStack(reserved);
+#ifdef JS_CPU_ARM
+    if (set.gprs().size() > 1 && ignore.empty(false)) {
+        startDataTransferM(IsLoad, StackPointer, IA, NoWriteBack);
+        for (GeneralRegisterIterator iter(set.gprs()); iter.more(); iter++) {
+            diffG -= STACK_SLOT_SIZE;
+            transferReg(*iter);
+        }
+        finishDataTransfer();
+    } else
+#endif
+    {
+        for (GeneralRegisterIterator iter(set.gprs()); iter.more(); iter++) {
+            diffG -= STACK_SLOT_SIZE;
+            if (!ignore.has(*iter))
+                loadPtr(Address(StackPointer, diffG), *iter);
+        }
+    }
+    freeStack(reservedG);
+    JS_ASSERT(diffG == 0);
 }
 
 template<typename T>

js/src/ion/RegisterSets.h

@@ -318,6 +318,11 @@ class TypedRegisterSet
     bool has(T reg) const {
         return !!(bits_ & (1 << reg.code()));
     }
+    bool hasNextRegister(T reg) const {
+        if (reg.code() == sizeof(bits_)*8)
+            return false;
+        return !!(bits_ & (1 << (reg.code()+1)));
+    }
     void addUnchecked(T reg) {
         bits_ |= (1 << reg.code());
     }
@@ -344,12 +349,23 @@ class TypedRegisterSet
         JS_FLOOR_LOG2(ireg, bits_);
         return T::FromCode(ireg);
     }
+    T getFirst() const {
+        JS_ASSERT(!empty());
+        int ireg = js_bitscan_ctz32(bits_);
+        return T::FromCode(ireg);
+    }
     T takeAny() {
         JS_ASSERT(!empty());
         T reg = getAny();
         take(reg);
         return reg;
     }
+    T takeFirst() {
+        JS_ASSERT(!empty());
+        T reg = getFirst();
+        take(reg);
+        return reg;
+    }
     void clear() {
         bits_ = 0;
     }
@@ -533,6 +549,7 @@ class RegisterSet {
     }
 };
 
+// iterates backwards, that is, rn to r0
 template <typename T>
 class TypedRegisterIterator
 {
@@ -552,13 +569,48 @@ class TypedRegisterIterator
         regset_.takeAny();
         return old;
     }
+    TypedRegisterIterator<T>& operator ++() {
+        regset_.takeAny();
+        return *this;
+    }
     T operator *() const {
         return regset_.getAny();
     }
 };
 
+// iterates forwards, that is r0 to rn
+template <typename T>
+class TypedRegisterForwardIterator
+{
+    TypedRegisterSet<T> regset_;
+
+  public:
+    TypedRegisterForwardIterator(TypedRegisterSet<T> regset) : regset_(regset)
+    { }
+    TypedRegisterForwardIterator(const TypedRegisterForwardIterator &other) : regset_(other.regset_)
+    { }
+
+    bool more() const {
+        return !regset_.empty();
+    }
+    TypedRegisterForwardIterator<T> operator ++(int) {
+        TypedRegisterIterator<T> old(*this);
+        regset_.takeFirst();
+        return old;
+    }
+    TypedRegisterForwardIterator<T>& operator ++() {
+        regset_.takeFirst();
+        return *this;
+    }
+    T operator *() const {
+        return regset_.getFirst();
+    }
+};
+
 typedef TypedRegisterIterator<Register> GeneralRegisterIterator;
 typedef TypedRegisterIterator<FloatRegister> FloatRegisterIterator;
+typedef TypedRegisterForwardIterator<Register> GeneralRegisterForwardIterator;
+typedef TypedRegisterForwardIterator<FloatRegister> FloatRegisterForwardIterator;
 
 class AnyRegisterIterator
 {

js/src/ion/arm/MacroAssembler-arm.cpp

@@ -1383,6 +1383,37 @@ MacroAssemblerARM::ma_vstr(VFPRegister src, Register base, Register index, int32
     ma_vstr(src, Operand(ScratchRegister, 0), cc);
 }
 
+
+int32_t
+MacroAssemblerARM::transferMultipleByRuns(FloatRegisterSet set, LoadStore ls,
+                                          Register rm, DTMMode mode)
+{
+    int32_t delta;
+    if (mode == IA) {
+        delta = sizeof(double);
+    } else if (mode == DB) {
+        delta = -sizeof(double);
+    } else {
+        JS_NOT_REACHED("Invalid data transfer addressing mode");
+    }
+
+    int32_t offset = 0;
+    FloatRegisterForwardIterator iter(set);
+    while (iter.more()) {
+        startFloatTransferM(ls, rm, mode, WriteBack);
+        int32_t reg = (*iter).code_;
+        do {
+            offset += delta;
+            transferFloatReg(*iter);
+        } while ((++iter).more() && (*iter).code_ == ++reg);
+        finishFloatTransfer();
+    }
+
+    JS_ASSERT(offset == set.size() * sizeof(double) * (mode == DB ? -1 : 1));
+    ma_sub(Imm32(offset), rm);
+    return offset;
+}
+
 bool
 MacroAssemblerARMCompat::buildFakeExitFrame(const Register &scratch, uint32_t *offset)
 {

js/src/ion/arm/MacroAssembler-arm.h

@@ -330,6 +330,14 @@ class MacroAssemblerARM : public Assembler
     void ma_callIonHalfPush(const Register reg);
 
     void ma_call(void *dest);
+
+    // Float registers can only be loaded/stored in continuous runs
+    // when using vstm/vldm.
+    // This function breaks set into continuous runs and loads/stores
+    // them at [rm]. rm will be modified, but returned to its initial value.
+    // Returns the offset from [dm] for the logical next load/store.
+    int32_t transferMultipleByRuns(FloatRegisterSet set, LoadStore ls,
+                                   Register rm, DTMMode mode);
 };
 
 class MacroAssemblerARMCompat : public MacroAssemblerARM