bug 991153: Fix float32 on arm to handle aliased registers (r=jandem)

js/src/jit/AsmJS.cpp

@@ -5906,7 +5906,7 @@ StackDecrementForCall(MacroAssembler &masm, const VectorT &argTypes, unsigned ex
 static const RegisterSet NonVolatileRegs =
     RegisterSet(GeneralRegisterSet(Registers::NonVolatileMask &
                                    ~(uint32_t(1) << Registers::lr)),
-                FloatRegisterSet(FloatRegisters::NonVolatileMask | (1 << FloatRegisters::d15)));
+                FloatRegisterSet(FloatRegisters::NonVolatileMask | (1ULL << FloatRegisters::d15)));
 #else
 static const RegisterSet NonVolatileRegs =
     RegisterSet(GeneralRegisterSet(Registers::NonVolatileMask),
@@ -5921,7 +5921,7 @@ static const unsigned FramePushedAfterSave = NonVolatileRegs.gprs().size() * siz
                                              sizeof(double);
 #else
 static const unsigned FramePushedAfterSave = NonVolatileRegs.gprs().size() * sizeof(intptr_t) +
-                                             NonVolatileRegs.fpus().size() * sizeof(double);
+                                             NonVolatileRegs.fpus().getPushSizeInBytes();
 #endif
 
 static bool
@@ -6653,7 +6653,7 @@ GenerateStackOverflowExit(ModuleCompiler &m, Label *throwLabel)
 static const RegisterSet AllRegsExceptSP =
     RegisterSet(GeneralRegisterSet(Registers::AllMask &
                                    ~(uint32_t(1) << Registers::StackPointer)),
-                FloatRegisterSet(FloatRegisters::AllMask));
+                FloatRegisterSet(FloatRegisters::AllDoubleMask));
 
 // The operation-callback exit is called from arbitrarily-interrupted asm.js
 // code. That means we must first save *all* registers and restore *all*
@@ -6776,12 +6776,12 @@ GenerateInterruptExit(ModuleCompiler &m, Label *throwLabel)
     // argument 0: cx
     masm.loadPtr(Address(IntArgReg0, AsmJSActivation::offsetOfContext()), IntArgReg0);
 
-    masm.PushRegsInMask(RegisterSet(GeneralRegisterSet(0), FloatRegisterSet(FloatRegisters::AllMask)));   // save all FP registers
+    masm.PushRegsInMask(RegisterSet(GeneralRegisterSet(0), FloatRegisterSet(FloatRegisters::AllDoubleMask)));   // save all FP registers
     masm.call(AsmJSImm_HandleExecutionInterrupt);
     masm.branchIfFalseBool(ReturnReg, throwLabel);
 
     // Restore the machine state to before the interrupt. this will set the pc!
-    masm.PopRegsInMask(RegisterSet(GeneralRegisterSet(0), FloatRegisterSet(FloatRegisters::AllMask)));   // restore all FP registers
+    masm.PopRegsInMask(RegisterSet(GeneralRegisterSet(0), FloatRegisterSet(FloatRegisters::AllDoubleMask)));   // restore all FP registers
     masm.mov(r6,sp);
     masm.as_vmsr(r5);
     masm.as_msr(r4);

js/src/jit/CodeGenerator.cpp

@@ -1537,7 +1537,6 @@ CodeGenerator::visitMoveGroup(LMoveGroup *group)
         // No bogus moves.
         JS_ASSERT(*from != *to);
         JS_ASSERT(!from->isConstant());
-
         MoveOp::Type moveType;
         switch (type) {
           case LDefinition::OBJECT:
@@ -8548,12 +8547,22 @@ CodeGenerator::visitAsmJSCall(LAsmJSCall *ins)
 
 #if defined(JS_CODEGEN_ARM)
     if (!UseHardFpABI() && mir->callee().which() == MAsmJSCall::Callee::Builtin) {
+        // The soft ABI passes floating point arguments in GPRs. Since basically
+        // nothing is set up to handle this, the values are placed in the
+        // corresponding VFP registers, then transferred to GPRs immediately
+        // before the call. The mapping is sN <-> rN, where double registers
+        // can be treated as their two component single registers.
         for (unsigned i = 0, e = ins->numOperands(); i < e; i++) {
             LAllocation *a = ins->getOperand(i);
             if (a->isFloatReg()) {
                 FloatRegister fr = ToFloatRegister(a);
-                int srcId = fr.code() * 2;
+                if (fr.isDouble()) {
-                masm.ma_vxfer(fr, Register::FromCode(srcId), Register::FromCode(srcId+1));
+                    uint32_t srcId = fr.singleOverlay().id();
+                    masm.ma_vxfer(fr, Register::FromCode(srcId), Register::FromCode(srcId + 1));
+                } else {
+                    uint32_t srcId = fr.id();
+                    masm.ma_vxfer(fr, Register::FromCode(srcId));
+                }
             }
         }
     }

js/src/jit/IonFrames.cpp

@@ -353,9 +353,19 @@ JitFrameIterator::machineState() const
 
     uint8_t *spillAlign = alignDoubleSpillWithOffset(reinterpret_cast<uint8_t *>(spill), 0);
 
-    double *floatSpill = reinterpret_cast<double *>(spillAlign);
+    char *floatSpill = reinterpret_cast<char *>(spillAlign);
-    for (FloatRegisterBackwardIterator iter(reader.allFloatSpills()); iter.more(); iter++)
+    FloatRegisterSet fregs = reader.allFloatSpills();
-        machine.setRegisterLocation(*iter, --floatSpill);
+    fregs = fregs.reduceSetForPush();
+    for (FloatRegisterBackwardIterator iter(fregs); iter.more(); iter++) {
+        floatSpill -= (*iter).size();
+        for (uint32_t a = 0; a < (*iter).numAlignedAliased(); a++) {
+            // Only say that registers that actually start here start here.
+            // e.g. d0 should not start at s1, only at s0.
+            FloatRegister ftmp;
+            (*iter).alignedAliased(a, &ftmp);
+            machine.setRegisterLocation(ftmp, (double*)floatSpill);
+        }
+    }
 
     return machine;
 }
@@ -1962,15 +1972,22 @@ InlineFrameIterator::isFunctionFrame() const
 
 MachineState
 MachineState::FromBailout(mozilla::Array<uintptr_t, Registers::Total> &regs,
-                          mozilla::Array<double, FloatRegisters::Total> &fpregs)
+                          mozilla::Array<double, FloatRegisters::TotalPhys> &fpregs)
 {
     MachineState machine;
 
     for (unsigned i = 0; i < Registers::Total; i++)
         machine.setRegisterLocation(Register::FromCode(i), &regs[i]);
+#ifdef JS_CODEGEN_ARM
+    float *fbase = (float*)&fpregs[0];
+    for (unsigned i = 0; i < FloatRegisters::TotalDouble; i++)
+        machine.setRegisterLocation(FloatRegister(i, FloatRegister::Double), &fpregs[i]);
+    for (unsigned i = 0; i < FloatRegisters::TotalSingle; i++)
+        machine.setRegisterLocation(FloatRegister(i, FloatRegister::Single), (double*)&fbase[i]);
+#else
     for (unsigned i = 0; i < FloatRegisters::Total; i++)
         machine.setRegisterLocation(FloatRegister::FromCode(i), &fpregs[i]);
-
+#endif
     return machine;
 }
 

js/src/jit/IonFrames.h

@@ -831,7 +831,7 @@ class IonBaselineStubFrameLayout : public IonCommonFrameLayout
 // An invalidation bailout stack is at the stack pointer for the callee frame.
 class InvalidationBailoutStack
 {
-    mozilla::Array<double, FloatRegisters::Total> fpregs_;
+    mozilla::Array<double, FloatRegisters::TotalPhys> fpregs_;
     mozilla::Array<uintptr_t, Registers::Total> regs_;
     IonScript   *ionScript_;
     uint8_t       *osiPointReturnAddress_;

js/src/jit/IonMacroAssembler.cpp

@@ -1578,9 +1578,14 @@ MacroAssembler::convertValueToFloatingPoint(ValueOperand value, FloatRegister ou
     jump(&done);
 
     bind(&isDouble);
-    unboxDouble(value, output);
+    FloatRegister tmp = output;
+    if (outputType == MIRType_Float32 && hasMultiAlias())
+        tmp = ScratchDoubleReg;
+
+    unboxDouble(value, tmp);
     if (outputType == MIRType_Float32)
-        convertDoubleToFloat32(output, output);
+        convertDoubleToFloat32(tmp, output);
+
     bind(&done);
 }
 

js/src/jit/LIR.h

@@ -205,7 +205,7 @@ class LUse : public LAllocation
     static const uint32_t POLICY_BITS = 3;
     static const uint32_t POLICY_SHIFT = 0;
     static const uint32_t POLICY_MASK = (1 << POLICY_BITS) - 1;
-    static const uint32_t REG_BITS = 5;
+    static const uint32_t REG_BITS = 6;
     static const uint32_t REG_SHIFT = POLICY_SHIFT + POLICY_BITS;
     static const uint32_t REG_MASK = (1 << REG_BITS) - 1;
 
@@ -215,7 +215,7 @@ class LUse : public LAllocation
     static const uint32_t USED_AT_START_MASK = (1 << USED_AT_START_BITS) - 1;
 
   public:
-    // Virtual registers get the remaining 20 bits.
+    // Virtual registers get the remaining 19 bits.
     static const uint32_t VREG_BITS = DATA_BITS - (USED_AT_START_SHIFT + USED_AT_START_BITS);
     static const uint32_t VREG_SHIFT = USED_AT_START_SHIFT + USED_AT_START_BITS;
     static const uint32_t VREG_MASK = (1 << VREG_BITS) - 1;
@@ -484,7 +484,7 @@ class LDefinition
     }
     bool isCompatibleReg(const AnyRegister &r) const {
         if (isFloatReg() && r.isFloat()) {
-#if defined(JS_CODEGEN_ARM) && defined(EVERYONE_KNOWS_ABOUT_ALIASING)
+#if defined(JS_CODEGEN_ARM)
             if (type() == FLOAT32)
                 return r.fpu().isSingle();
             return r.fpu().isDouble();

js/src/jit/Lowering.cpp

@@ -3113,8 +3113,8 @@ LIRGenerator::visitAssertRange(MAssertRange *ins)
         break;
 
       case MIRType_Float32: {
-        LDefinition armtemp = hasMultiAlias() ? tempFloat32() : LDefinition::BogusTemp();
+        LDefinition armtemp = hasMultiAlias() ? tempDouble() : LDefinition::BogusTemp();
-        lir = new(alloc()) LAssertRangeF(useRegister(input), tempFloat32(), armtemp);
+        lir = new(alloc()) LAssertRangeF(useRegister(input), tempDouble(), armtemp);
         break;
       }
       case MIRType_Value:

js/src/jit/MoveResolver.cpp

@@ -43,7 +43,7 @@ MoveResolver::findBlockingMove(const PendingMove *last)
     for (PendingMoveIterator iter = pending_.begin(); iter != pending_.end(); iter++) {
         PendingMove *other = *iter;
 
-        if (other->from() == last->to()) {
+        if (other->from().aliases(last->to())) {
             // We now have pairs in the form (A -> X) (X -> y). The second pair
             // blocks the move in the first pair, so return it.
             return other;

js/src/jit/RegisterAllocator.cpp

@@ -268,6 +268,7 @@ AllocationIntegrityState::checkSafepointAllocation(LInstruction *ins,
         AnyRegister reg = alloc.toRegister();
         if (populateSafepoints)
             safepoint->addLiveRegister(reg);
+
         JS_ASSERT(safepoint->liveRegs().has(reg));
     }
 

js/src/jit/RegisterAllocator.h

@@ -322,7 +322,8 @@ class RegisterAllocator
         if (mir->compilingAsmJS()) {
             allRegisters_.take(AnyRegister(HeapReg));
             allRegisters_.take(AnyRegister(GlobalReg));
-            allRegisters_.take(AnyRegister(NANReg));
+            // Need to remove both NANReg, and its aliases.
+            allRegisters_.takeAllAliasedUnchecked(AnyRegister(NANReg));
         }
 #endif
     }

js/src/jit/RegisterSets.h

@@ -363,16 +363,33 @@ class TypedRegisterSet
         return TypedRegisterSet(T::Codes::AllocatableMask & T::Codes::NonVolatileMask);
     }
     bool has(T reg) const {
+        // When checking to see if a set has a register, we only want that exact
+        // register, not worrying about aliasing.
         return !!(bits_ & (SetType(1) << reg.code()));
     }
     void addUnchecked(T reg) {
         bits_ |= (SetType(1) << reg.code());
     }
+    void addAllAliasedUnchecked(T reg) {
+        for (int a = 0; a < reg.numAliased(); a++) {
+            T tmp;
+            reg.aliased(a, &tmp);
+            bits_ |= (SetType(1) << tmp.code());
+        }
+    }
 
     void add(T reg) {
-        JS_ASSERT(!has(reg));
+        // Make sure we don't add two overlapping registers.
+#ifdef DEBUG
+        for (uint32_t a = 0; a < reg.numAliased(); a++) {
+            T tmp;
+            reg.aliased(a, &tmp);
+            JS_ASSERT(!has(tmp));
+        }
+#endif
         addUnchecked(reg);
     }
+
     void add(ValueOperand value) {
 #if defined(JS_NUNBOX32)
         add(value.payloadReg());
@@ -399,6 +416,13 @@ class TypedRegisterSet
     void takeUnchecked(T reg) {
         bits_ &= ~(SetType(1) << reg.code());
     }
+    void takeAllAliasedUnchecked(T reg) {
+        for (int a = 0; a < reg.numAliased(); a++) {
+            T tmp;
+            reg.aliased(a, &tmp);
+            bits_ &= ~(SetType(1) << tmp.code());
+        }
+    }
     void take(ValueOperand value) {
 #if defined(JS_NUNBOX32)
         take(value.payloadReg());
@@ -449,11 +473,11 @@ class TypedRegisterSet
     }
     T getFirst() const {
         JS_ASSERT(!empty());
-        return T::FromCode(mozilla::CountTrailingZeroes32(bits_));
+        return T::FromCode(T::FirstBit(bits_));
     }
     T getLast() const {
         JS_ASSERT(!empty());
-        int ireg = 31 - mozilla::CountLeadingZeroes32(bits_);
+        int ireg = T::LastBit(bits_);
         return T::FromCode(ireg);
     }
     T takeAny() {
@@ -603,6 +627,12 @@ class RegisterSet {
         else
             addUnchecked(any.gpr());
     }
+    void addAllAliasedUnchecked(const AnyRegister &reg) {
+        if (reg.isFloat())
+            fpu_.addAllAliasedUnchecked(reg.fpu());
+        else
+            gpr_.addAllAliasedUnchecked(reg.gpr());
+    }
 
 
     bool empty(bool floats) const {
@@ -629,6 +659,12 @@ class RegisterSet {
         else
             gpr_.take(reg.gpr());
     }
+    void takeAllAliasedUnchecked(AnyRegister reg) {
+        if (reg.isFloat())
+            fpu_.takeAllAliasedUnchecked(reg.fpu());
+        else
+            gpr_.takeAllAliasedUnchecked(reg.gpr());
+    }
     AnyRegister takeAny(bool isFloat) {
         if (isFloat)
             return AnyRegister(takeFloat());

js/src/jit/Registers.h

@@ -70,20 +70,29 @@ struct Register {
         JS_ASSERT(aliasIdx == 0);
         *ret = *this;
     }
+    static uint32_t SetSize(SetType x) {
+        return Codes::SetSize(x);
+    }
+    static uint32_t FirstBit(SetType x) {
+        return Codes::FirstBit(x);
+    }
+    static uint32_t LastBit(SetType x) {
+        return Codes::LastBit(x);
+    }
 };
 
 class RegisterDump
 {
   protected: // Silence Clang warning.
     mozilla::Array<uintptr_t, Registers::Total> regs_;
-    mozilla::Array<double, FloatRegisters::Total> fpregs_;
+    mozilla::Array<double, FloatRegisters::TotalPhys> fpregs_;
 
   public:
     static size_t offsetOfRegister(Register reg) {
         return offsetof(RegisterDump, regs_) + reg.code() * sizeof(uintptr_t);
     }
     static size_t offsetOfRegister(FloatRegister reg) {
-        return offsetof(RegisterDump, fpregs_) + reg.code() * sizeof(double);
+        return offsetof(RegisterDump, fpregs_) + reg.getRegisterDumpOffsetInBytes();
     }
 };
 
@@ -95,7 +104,7 @@ class MachineState
 
   public:
     static MachineState FromBailout(mozilla::Array<uintptr_t, Registers::Total> &regs,
-                                    mozilla::Array<double, FloatRegisters::Total> &fpregs);
+                                    mozilla::Array<double, FloatRegisters::TotalPhys> &fpregs);
 
     void setRegisterLocation(Register reg, uintptr_t *up) {
         regs_[reg.code()] = up;

js/src/jit/StupidAllocator.cpp

@@ -77,8 +77,10 @@ StupidAllocator::init()
         RegisterSet remainingRegisters(allRegisters_);
         while (!remainingRegisters.empty(/* float = */ false))
             registers[registerCount++].reg = AnyRegister(remainingRegisters.takeGeneral());
+
         while (!remainingRegisters.empty(/* float = */ true))
             registers[registerCount++].reg = AnyRegister(remainingRegisters.takeFloat());
+
         JS_ASSERT(registerCount <= MAX_REGISTERS);
     }
 
@@ -94,7 +96,7 @@ StupidAllocator::allocationRequiresRegister(const LAllocation *alloc, AnyRegiste
         const LUse *use = alloc->toUse();
         if (use->policy() == LUse::FIXED) {
             AnyRegister usedReg = GetFixedRegister(virtualRegisters[use->virtualRegister()], use);
-            if (usedReg == reg)
+            if (usedReg.aliases(reg))
                 return true;
         }
     }
@@ -129,7 +131,7 @@ StupidAllocator::ensureHasRegister(LInstruction *ins, uint32_t vreg)
     RegisterIndex existing = findExistingRegister(vreg);
     if (existing != UINT32_MAX) {
         if (registerIsReserved(ins, registers[existing].reg)) {
-            evictRegister(ins, existing);
+            evictAliasedRegister(ins, existing);
         } else {
             registers[existing].age = ins->id();
             return registers[existing].reg;
@@ -158,7 +160,7 @@ StupidAllocator::allocateRegister(LInstruction *ins, uint32_t vreg)
     for (size_t i = 0; i < registerCount; i++) {
         AnyRegister reg = registers[i].reg;
 
-        if (reg.isFloat() != def->isFloatReg())
+        if (!def->isCompatibleReg(reg))
             continue;
 
         // Skip the register if it is in use for an allocated input or output.
@@ -173,7 +175,7 @@ StupidAllocator::allocateRegister(LInstruction *ins, uint32_t vreg)
         }
     }
 
-    evictRegister(ins, best);
+    evictAliasedRegister(ins, best);
     return best;
 }
 
@@ -199,6 +201,16 @@ StupidAllocator::evictRegister(LInstruction *ins, RegisterIndex index)
     registers[index].set(MISSING_ALLOCATION);
 }
 
+void
+StupidAllocator::evictAliasedRegister(LInstruction *ins, RegisterIndex index)
+{
+    for (int i = 0; i < registers[index].reg.numAliased(); i++) {
+        int aindex = registerIndex(registers[index].reg.aliased(i));
+        syncRegister(ins, aindex);
+        registers[aindex].set(MISSING_ALLOCATION);
+    }
+}
+
 void
 StupidAllocator::loadRegister(LInstruction *ins, uint32_t vreg, RegisterIndex index, LDefinition::Type type)
 {
@@ -334,7 +346,9 @@ StupidAllocator::allocateForInstruction(LInstruction *ins)
             AnyRegister reg = GetFixedRegister(virtualRegisters[vreg], use);
             RegisterIndex index = registerIndex(reg);
             if (registers[index].vreg != vreg) {
-                evictRegister(ins, index);
+                // Need to evict multiple registers
+                evictAliasedRegister(ins, registerIndex(reg));
+                // If this vreg is already assigned to an incorrect register
                 RegisterIndex existing = findExistingRegister(vreg);
                 if (existing != UINT32_MAX)
                     evictRegister(ins, existing);

js/src/jit/StupidAllocator.h

@@ -75,6 +75,7 @@ class StupidAllocator : public RegisterAllocator
 
     void syncRegister(LInstruction *ins, RegisterIndex index);
     void evictRegister(LInstruction *ins, RegisterIndex index);
+    void evictAliasedRegister(LInstruction *ins, RegisterIndex index);
     void loadRegister(LInstruction *ins, uint32_t vreg, RegisterIndex index, LDefinition::Type type);
 
     RegisterIndex findExistingRegister(uint32_t vreg);

js/src/jit/arm/Architecture-arm.cpp

@@ -14,6 +14,7 @@
 #include <unistd.h>
 
 #include "jit/arm/Assembler-arm.h"
+#include "jit/RegisterSets.h"
 
 #define HWCAP_USE_HARDFP_ABI (1 << 27)
 
@@ -355,5 +356,15 @@ VFPRegister::getRegisterDumpOffsetInBytes()
     MOZ_ASSUME_UNREACHABLE();
 }
 
+uint32_t
+FloatRegisters::ActualTotalPhys()
+{
+    if (Has32DP())
+        return 32;
+    return 16;
+}
+
+
 } // namespace jit
 } // namespace js
+

js/src/jit/arm/Architecture-arm.h

@@ -148,15 +148,54 @@ class Registers
         static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
         return mozilla::CountPopulation32(x);
     }
+    static uint32_t FirstBit(SetType x) {
+        return mozilla::CountTrailingZeroes32(x);
+    }
+    static uint32_t LastBit(SetType x) {
+        return 31 - mozilla::CountLeadingZeroes32(x);
+    }
 };
 
 // Smallest integer type that can hold a register bitmask.
 typedef uint16_t PackedRegisterMask;
+typedef uint16_t PackedRegisterMask;
 
 class FloatRegisters
 {
   public:
     enum FPRegisterID {
+        s0,
+        s1,
+        s2,
+        s3,
+        s4,
+        s5,
+        s6,
+        s7,
+        s8,
+        s9,
+        s10,
+        s11,
+        s12,
+        s13,
+        s14,
+        s15,
+        s16,
+        s17,
+        s18,
+        s19,
+        s20,
+        s21,
+        s22,
+        s23,
+        s24,
+        s25,
+        s26,
+        s27,
+        s28,
+        s29,
+        s30,
+        s31,
         d0,
         d1,
         d2,
@@ -188,15 +227,28 @@ class FloatRegisters
         d28,
         d29,
         d30,
+        d31,
         invalid_freg
     };
+
     typedef FPRegisterID Code;
 
-    static const char *GetName(Code code) {
+
+    static const char *GetDoubleName(Code code) {
         static const char * const Names[] = { "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
-                                              "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15"};
+                                              "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15",
+                                              "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23",
+                                              "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31"};
         return Names[code];
     }
+    static const char *GetSingleName(Code code) {
+        static const char * const Names[] = { "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
+                                              "s8", "s9", "s10", "s11", "s12", "s13", "s14", "s15",
+                                              "s16", "s17", "s18", "s19", "s20", "s21", "s22", "s23",
+                                              "s24", "s25", "s26", "s27", "s28", "s29", "s30", "s31"};
+        return Names[code];
+    }
+
     static const char *GetName(uint32_t i) {
         JS_ASSERT(i < Total);
         return GetName(Code(i));
@@ -205,34 +257,60 @@ class FloatRegisters
     static Code FromName(const char *name);
 
     static const Code Invalid = invalid_freg;
-
+    static const uint32_t Total = 48;
-    static const uint32_t Total = 16;
+    static const uint32_t TotalDouble = 16;
-    static const uint32_t Allocatable = 15;
+    static const uint32_t TotalSingle = 32;
-
+    static const uint32_t Allocatable = 45;
-    static const uint32_t AllMask = (1 << Total) - 1;
+    // There are only 32 places that we can put values.
+    static const uint32_t TotalPhys = 32;
+    static uint32_t ActualTotalPhys();
+    static const uint64_t AllDoubleMask = ((1ull << 16) - 1) << 32;
+    static const uint64_t AllMask = ((1ull << 48) - 1);
 
     // d15 is the ScratchFloatReg.
-    static const uint32_t NonVolatileMask =
+    static const uint64_t NonVolatileDoubleMask =
-        (1 << d8) |
+         ((1ULL << d8) |
-        (1 << d9) |
+          (1ULL << d9) |
-        (1 << d10) |
+          (1ULL << d10) |
-        (1 << d11) |
+          (1ULL << d11) |
-        (1 << d12) |
+          (1ULL << d12) |
-        (1 << d13) |
+          (1ULL << d13) |
-        (1 << d14);
+          (1ULL << d14));
+    // s30 and s31 alias d15.
+    static const uint64_t NonVolatileMask =
+        (NonVolatileDoubleMask |
+         ((1 << s16) |
+          (1 << s17) |
+          (1 << s18) |
+          (1 << s19) |
+          (1 << s20) |
+          (1 << s21) |
+          (1 << s22) |
+          (1 << s23) |
+          (1 << s24) |
+          (1 << s25) |
+          (1 << s26) |
+          (1 << s27) |
+          (1 << s28) |
+          (1 << s29) |
+          (1 << s30)));
 
-    static const uint32_t VolatileMask = AllMask & ~NonVolatileMask;
+    static const uint64_t VolatileMask = AllMask & ~NonVolatileMask;
+    static const uint64_t VolatileDoubleMask = AllDoubleMask & ~NonVolatileDoubleMask;
 
-    static const uint32_t WrapperMask = VolatileMask;
+    static const uint64_t WrapperMask = VolatileMask;
 
     // d15 is the ARM scratch float register.
-    static const uint32_t NonAllocatableMask = (1 << d15) | (1 << invalid_freg);
+    // s30 and s31 alias d15.
+    static const uint64_t NonAllocatableMask = ((1ULL << d15)) |
+                                                (1ULL << s30) |
+                                                (1ULL << s31);
 
     // Registers that can be allocated without being saved, generally.
-    static const uint32_t TempMask = VolatileMask & ~NonAllocatableMask;
+    static const uint64_t TempMask = VolatileMask & ~NonAllocatableMask;
 
-    static const uint32_t AllocatableMask = AllMask & ~NonAllocatableMask;
+    static const uint64_t AllocatableMask = AllMask & ~NonAllocatableMask;
-    typedef uint32_t SetType;
+    typedef uint64_t SetType;
 };
 
 template <typename T>
@@ -262,7 +340,7 @@ class VFPRegister
     // do decide to address them seprately (vmov, I'm looking at you), we will
     // likely specify it as a separate field.
   public:
-    Code code_ : 5;
+    uint32_t code_ : 5;
   protected:
     bool _isInvalid : 1;
     bool _isMissing : 1;
@@ -293,7 +371,7 @@ class VFPRegister
     bool isInt() const { return (kind == UInt) || (kind == Int); }
     bool isSInt() const { return kind == Int; }
     bool isUInt() const { return kind == UInt; }
-    bool equiv(VFPRegister other) const { return other.kind == kind; }
+    bool equiv(const VFPRegister &other) const { return other.kind == kind; }
     size_t size() const { return (kind == Double) ? 8 : 4; }
     bool isInvalid() const;
     bool isMissing() const;
@@ -327,14 +405,15 @@ class VFPRegister
         // This should only be used in areas where we only have doubles and
         // singles.
         JS_ASSERT(isFloat());
-        return Code(code_);
+        return Code(code_ | (kind << 5));
     }
     uint32_t id() const {
         return code_;
     }
     static VFPRegister FromCode(uint32_t i) {
         uint32_t code = i & 31;
-        return VFPRegister(code, Double);
+        uint32_t kind = i >> 5;
+        return VFPRegister(code, RegType(kind));
     }
     bool volatile_() const {
         if (isDouble())
@@ -342,7 +421,9 @@ class VFPRegister
         return !!((1 << code_) & FloatRegisters::VolatileMask);
     }
     const char *name() const {
-        return FloatRegisters::GetName(code_);
+        if (isDouble())
+            return FloatRegisters::GetDoubleName(Code(code_));
+        return FloatRegisters::GetSingleName(Code(code_));
     }
     bool operator != (const VFPRegister &other) const {
         return other.kind != kind || code_ != other.code_;
@@ -354,15 +435,12 @@ class VFPRegister
     }
     static const int NumAliasedDoubles = 16;
     uint32_t numAliased() const {
-        return 1;
-#ifdef EVERYONE_KNOWS_ABOUT_ALIASING
         if (isDouble()) {
             if (code_ < NumAliasedDoubles)
                 return 3;
             return 1;
         }
         return 2;
-#endif
     }
 
     // N.B. FloatRegister is an explicit outparam here because msvc-2010
@@ -413,7 +491,7 @@ class VFPRegister
     }
     typedef FloatRegisters::SetType SetType;
     static uint32_t SetSize(SetType x) {
-        static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+        static_assert(sizeof(SetType) == 8, "SetType must be 64 bits");
         return mozilla::CountPopulation32(x);
     }
     static Code FromName(const char *name) {
@@ -423,6 +501,12 @@ class VFPRegister
     static uint32_t GetSizeInBytes(const TypedRegisterSet<VFPRegister> &s);
     static uint32_t GetPushSizeInBytes(const TypedRegisterSet<VFPRegister> &s);
     uint32_t getRegisterDumpOffsetInBytes();
+    static uint32_t FirstBit(SetType x) {
+        return mozilla::CountTrailingZeroes64(x);
+    }
+    static uint32_t LastBit(SetType x) {
+        return 63 - mozilla::CountLeadingZeroes64(x);
+    }
 
 };
 

js/src/jit/arm/Assembler-arm.cpp

@@ -22,6 +22,8 @@ using namespace js::jit;
 
 using mozilla::CountLeadingZeroes32;
 
+void dbg_break() {}
+
 // Note this is used for inter-AsmJS calls and may pass arguments and results in
 // floating point registers even if the system ABI does not.
 ABIArgGenerator::ABIArgGenerator() :
@@ -46,7 +48,6 @@ ABIArgGenerator::next(MIRType type)
         intRegIndex_++;
         break;
       case MIRType_Float32:
-      case MIRType_Double:
         if (floatRegIndex_ == NumFloatArgRegs) {
             static const int align = sizeof(double) - 1;
             stackOffset_ = (stackOffset_ + align) & ~align;
@@ -54,9 +55,22 @@ ABIArgGenerator::next(MIRType type)
             stackOffset_ += sizeof(uint64_t);
             break;
         }
-        current_ = ABIArg(FloatRegister::FromCode(floatRegIndex_));
+        current_ = ABIArg(VFPRegister(floatRegIndex_, VFPRegister::Single));
         floatRegIndex_++;
         break;
+      case MIRType_Double:
+        // Bump the number of used registers up to the next multiple of two.
+        floatRegIndex_ = (floatRegIndex_ + 1) & ~1;
+        if (floatRegIndex_ == NumFloatArgRegs) {
+            static const int align = sizeof(double) - 1;
+            stackOffset_ = (stackOffset_ + align) & ~align;
+            current_ = ABIArg(stackOffset_);
+            stackOffset_ += sizeof(uint64_t);
+            break;
+        }
+        current_ = ABIArg(VFPRegister(floatRegIndex_ >> 1, VFPRegister::Double));
+        floatRegIndex_+=2;
+        break;
       default:
         MOZ_ASSUME_UNREACHABLE("Unexpected argument type");
     }
@@ -1189,6 +1203,7 @@ VFPRegister
 VFPRegister::doubleOverlay(unsigned int which) const
 {
     JS_ASSERT(!_isInvalid);
+    JS_ASSERT(which == 0);
     if (kind != Double)
         return VFPRegister(code_ >> 1, Double);
     return *this;
@@ -1586,7 +1601,7 @@ class PoolHintData {
         JS_ASSERT(cond_ == cond >> 28);
         loadType_ = lt;
         ONES = ExpectedOnes;
-        destReg_ = destReg.isDouble() ? destReg.code() : destReg.doubleOverlay().code();
+        destReg_ = destReg.id();
         destType_ = destReg.isDouble();
     }
     Assembler::Condition getCond() {
@@ -1597,8 +1612,8 @@ class PoolHintData {
         return Register::FromCode(destReg_);
     }
     VFPRegister getVFPReg() {
-        VFPRegister r = VFPRegister(FloatRegister::FromCode(destReg_));
+        VFPRegister r = VFPRegister(destReg_, destType_ ? VFPRegister::Double : VFPRegister::Single);
-        return destType_ ? r : r.singleOverlay();
+        return r;
     }
 
     int32_t getIndex() {
@@ -2319,7 +2334,6 @@ Assembler::retarget(Label *label, Label *target)
 }
 
 
-void dbg_break() {}
 static int stopBKPT = -1;
 void
 Assembler::as_bkpt()

js/src/jit/arm/Assembler-arm.h

@@ -82,19 +82,21 @@ class ABIArgGenerator
 static MOZ_CONSTEXPR_VAR Register PreBarrierReg = r1;
 
 static MOZ_CONSTEXPR_VAR Register InvalidReg = { Registers::invalid_reg };
-static MOZ_CONSTEXPR_VAR FloatRegister InvalidFloatReg(FloatRegisters::invalid_freg);
+static MOZ_CONSTEXPR_VAR FloatRegister InvalidFloatReg;
 
 static MOZ_CONSTEXPR_VAR Register JSReturnReg_Type = r3;
 static MOZ_CONSTEXPR_VAR Register JSReturnReg_Data = r2;
 static MOZ_CONSTEXPR_VAR Register StackPointer = sp;
 static MOZ_CONSTEXPR_VAR Register FramePointer = InvalidReg;
 static MOZ_CONSTEXPR_VAR Register ReturnReg = r0;
-static MOZ_CONSTEXPR_VAR FloatRegister ReturnFloat32Reg(FloatRegisters::d0);
+static MOZ_CONSTEXPR_VAR FloatRegister ReturnFloat32Reg = { FloatRegisters::d0, VFPRegister::Single };
-static MOZ_CONSTEXPR_VAR FloatRegister ReturnDoubleReg(FloatRegisters::d0);
+static MOZ_CONSTEXPR_VAR FloatRegister ReturnDoubleReg = { FloatRegisters::d0, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister ScratchFloat32Reg(FloatRegisters::d15);
+static MOZ_CONSTEXPR_VAR FloatRegister ScratchFloat32Reg = { FloatRegisters::d30, VFPRegister::Single };
-static MOZ_CONSTEXPR_VAR FloatRegister ScratchDoubleReg(FloatRegisters::d15);
+static MOZ_CONSTEXPR_VAR FloatRegister ScratchDoubleReg = { FloatRegisters::d15, VFPRegister::Double };
+static MOZ_CONSTEXPR_VAR FloatRegister ScratchUIntReg = { FloatRegisters::d15, VFPRegister::UInt };
+static MOZ_CONSTEXPR_VAR FloatRegister ScratchIntReg = { FloatRegisters::d15, VFPRegister::Int };
 
-static MOZ_CONSTEXPR_VAR FloatRegister NANReg(FloatRegisters::d14);
+static MOZ_CONSTEXPR_VAR FloatRegister NANReg = { FloatRegisters::d14, VFPRegister::Double };
 
 // Registers used in the GenerateFFIIonExit Enable Activation block.
 static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegCallee = r4;
@@ -112,22 +114,23 @@ static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegD1 = r1;
 static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegD2 = r4;
 
 
-static MOZ_CONSTEXPR_VAR FloatRegister d0(FloatRegisters::d0);
+static MOZ_CONSTEXPR_VAR FloatRegister d0  = {FloatRegisters::d0, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d1(FloatRegisters::d1);
+static MOZ_CONSTEXPR_VAR FloatRegister d1  = {FloatRegisters::d1, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d2(FloatRegisters::d2);
+static MOZ_CONSTEXPR_VAR FloatRegister d2  = {FloatRegisters::d2, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d3(FloatRegisters::d3);
+static MOZ_CONSTEXPR_VAR FloatRegister d3  = {FloatRegisters::d3, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d4(FloatRegisters::d4);
+static MOZ_CONSTEXPR_VAR FloatRegister d4  = {FloatRegisters::d4, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d5(FloatRegisters::d5);
+static MOZ_CONSTEXPR_VAR FloatRegister d5  = {FloatRegisters::d5, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d6(FloatRegisters::d6);
+static MOZ_CONSTEXPR_VAR FloatRegister d6  = {FloatRegisters::d6, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d7(FloatRegisters::d7);
+static MOZ_CONSTEXPR_VAR FloatRegister d7  = {FloatRegisters::d7, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d8(FloatRegisters::d8);
+static MOZ_CONSTEXPR_VAR FloatRegister d8  = {FloatRegisters::d8, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d9(FloatRegisters::d9);
+static MOZ_CONSTEXPR_VAR FloatRegister d9  = {FloatRegisters::d9, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d10(FloatRegisters::d10);
+static MOZ_CONSTEXPR_VAR FloatRegister d10 = {FloatRegisters::d10, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d11(FloatRegisters::d11);
+static MOZ_CONSTEXPR_VAR FloatRegister d11 = {FloatRegisters::d11, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d12(FloatRegisters::d12);
+static MOZ_CONSTEXPR_VAR FloatRegister d12 = {FloatRegisters::d12, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d13(FloatRegisters::d13);
+static MOZ_CONSTEXPR_VAR FloatRegister d13 = {FloatRegisters::d13, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d14(FloatRegisters::d14);
+static MOZ_CONSTEXPR_VAR FloatRegister d14 = {FloatRegisters::d14, VFPRegister::Double};
-static MOZ_CONSTEXPR_VAR FloatRegister d15(FloatRegisters::d15);
+static MOZ_CONSTEXPR_VAR FloatRegister d15 = {FloatRegisters::d15, VFPRegister::Double};
+
 
 // For maximal awesomeness, 8 should be sufficent. ldrd/strd (dual-register
 // load/store) operate in a single cycle when the address they are dealing with
@@ -1618,6 +1621,7 @@ class Assembler : public AssemblerShared
             JS_ASSERT(dtmLastReg >= 0);
             JS_ASSERT(rn.code() == unsigned(dtmLastReg) + dtmDelta);
         }
+
         dtmLastReg = rn.code();
     }
     void finishFloatTransfer() {
@@ -1625,11 +1629,32 @@ class Assembler : public AssemblerShared
         dtmActive = false;
         JS_ASSERT(dtmLastReg != -1);
         dtmDelta = dtmDelta ? dtmDelta : 1;
+        // The operand for the vstr/vldr instruction is the lowest register in the range.
+        int low = Min(dtmLastReg, vdtmFirstReg);
+        int high = Max(dtmLastReg, vdtmFirstReg);
         // Fencepost problem.
-        int len = dtmDelta * (dtmLastReg - vdtmFirstReg) + 1;
+        int len = high - low + 1;
-        as_vdtm(dtmLoadStore, dtmBase,
+        // vdtm can only transfer 16 registers at once.  If we need to transfer more,
-                VFPRegister(FloatRegister::FromCode(Min(vdtmFirstReg, dtmLastReg))),
+        // then either hoops are necessary, or we need to be updating the register.
-                len, dtmCond);
+        JS_ASSERT_IF(len > 16, dtmUpdate == WriteBack);
+
+        int adjustLow = dtmLoadStore == IsStore ? 0 : 1;
+        int adjustHigh = dtmLoadStore == IsStore ? -1 : 0;
+        while (len > 0) {
+            // Limit the instruction to 16 registers.
+            int curLen = Min(len, 16);
+            // If it is a store, we want to start at the high end and move down
+            // (e.g. vpush d16-d31; vpush d0-d15).
+            int curStart = (dtmLoadStore == IsStore) ? high - curLen + 1 : low;
+            as_vdtm(dtmLoadStore, dtmBase,
+                    VFPRegister(FloatRegister::FromCode(curStart)),
+                    curLen, dtmCond);
+            // Update the bounds.
+            low += adjustLow * curLen;
+            high += adjustHigh * curLen;
+            // Update the length parameter.
+            len -= curLen;
+        }
     }
 
   private:
@@ -2000,7 +2025,9 @@ class InstructionIterator {
 };
 
 static const uint32_t NumIntArgRegs = 4;
-static const uint32_t NumFloatArgRegs = 8;
+// There are 16 *float* registers available for arguments
+// If doubles are used, only half the number of registers are available.
+static const uint32_t NumFloatArgRegs = 16;
 
 static inline bool
 GetIntArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs, Register *out)
@@ -2048,12 +2075,22 @@ GetArgStackDisp(uint32_t arg)
 #if defined(JS_CODEGEN_ARM_HARDFP) || defined(JS_ARM_SIMULATOR)
 
 static inline bool
-GetFloatArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs, FloatRegister *out)
+GetFloat32ArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs, FloatRegister *out)
 {
     JS_ASSERT(UseHardFpABI());
     if (usedFloatArgs >= NumFloatArgRegs)
         return false;
-    *out = FloatRegister::FromCode(usedFloatArgs);
+    *out = VFPRegister(usedFloatArgs, VFPRegister::Single);
+    return true;
+}
+static inline bool
+GetDoubleArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs, FloatRegister *out)
+{
+    JS_ASSERT(UseHardFpABI());
+    JS_ASSERT((usedFloatArgs % 2) == 0);
+    if (usedFloatArgs >= NumFloatArgRegs)
+        return false;
+    *out = VFPRegister(usedFloatArgs>>1, VFPRegister::Double);
     return true;
 }
 

js/src/jit/arm/Bailouts-arm.cpp

@@ -30,7 +30,7 @@ class BailoutStack
     };
 
   protected: // Silence Clang warning about unused private fields.
-    mozilla::Array<double, FloatRegisters::Total> fpregs_;
+    mozilla::Array<double, FloatRegisters::TotalPhys> fpregs_;
     mozilla::Array<uintptr_t, Registers::Total> regs_;
 
     uintptr_t snapshotOffset_;

js/src/jit/arm/CodeGenerator-arm.cpp

@@ -284,7 +284,7 @@ CodeGeneratorARM::visitMinMaxD(LMinMaxD *ins)
 
     // Check for zero.
     masm.bind(&equal);
-    masm.compareDouble(first, InvalidFloatReg);
+    masm.compareDouble(first, NoVFPRegister);
     // First wasn't 0 or -0, so just return it.
     masm.ma_b(&done, Assembler::VFP_NotEqualOrUnordered);
     // So now both operands are either -0 or 0.

js/src/jit/arm/MacroAssembler-arm.cpp

@@ -164,6 +164,8 @@ MacroAssemblerARM::convertFloat32ToInt32(FloatRegister src, Register dest,
 
 void
 MacroAssemblerARM::convertFloat32ToDouble(FloatRegister src, FloatRegister dest) {
+    JS_ASSERT(dest.isDouble());
+    JS_ASSERT(src.isSingle());
     as_vcvt(VFPRegister(dest), VFPRegister(src).singleOverlay());
 }
 
@@ -1608,21 +1610,29 @@ MacroAssemblerARM::ma_vcvt_U32_F64(FloatRegister src, FloatRegister dest, Condit
 void
 MacroAssemblerARM::ma_vcvt_F32_I32(FloatRegister src, FloatRegister dest, Condition cc)
 {
+    JS_ASSERT(src.isSingle());
+    JS_ASSERT(dest.isSInt());
     as_vcvt(VFPRegister(dest).sintOverlay(), VFPRegister(src).singleOverlay(), false, cc);
 }
 void
 MacroAssemblerARM::ma_vcvt_F32_U32(FloatRegister src, FloatRegister dest, Condition cc)
 {
+    JS_ASSERT(src.isSingle());
+    JS_ASSERT(dest.isUInt());
     as_vcvt(VFPRegister(dest).uintOverlay(), VFPRegister(src).singleOverlay(), false, cc);
 }
 void
 MacroAssemblerARM::ma_vcvt_I32_F32(FloatRegister src, FloatRegister dest, Condition cc)
 {
+    JS_ASSERT(src.isSInt());
+    JS_ASSERT(dest.isSingle());
     as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src).sintOverlay(), false, cc);
 }
 void
 MacroAssemblerARM::ma_vcvt_U32_F32(FloatRegister src, FloatRegister dest, Condition cc)
 {
+    JS_ASSERT(src.isUInt());
+    JS_ASSERT(dest.isSingle());
     as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src).uintOverlay(), false, cc);
 }
 
@@ -1869,7 +1879,7 @@ MacroAssemblerARMCompat::freeStack(Register amount)
 void
 MacroAssembler::PushRegsInMask(RegisterSet set)
 {
-    int32_t diffF = set.fpus().size() * sizeof(double);
+    int32_t diffF = set.fpus().getPushSizeInBytes();
     int32_t diffG = set.gprs().size() * sizeof(intptr_t);
 
     if (set.gprs().size() > 1) {
@@ -1898,7 +1908,7 @@ void
 MacroAssembler::PopRegsInMaskIgnore(RegisterSet set, RegisterSet ignore)
 {
     int32_t diffG = set.gprs().size() * sizeof(intptr_t);
-    int32_t diffF = set.fpus().size() * sizeof(double);
+    int32_t diffF = set.fpus().getPushSizeInBytes();
     const int32_t reservedG = diffG;
     const int32_t reservedF = diffF;
 
@@ -2671,7 +2681,7 @@ void
 MacroAssemblerARMCompat::compareDouble(FloatRegister lhs, FloatRegister rhs)
 {
     // Compare the doubles, setting vector status flags.
-    if (rhs == InvalidFloatReg)
+    if (rhs.isMissing())
         ma_vcmpz(lhs);
     else
         ma_vcmp(lhs, rhs);
@@ -2708,7 +2718,7 @@ void
 MacroAssemblerARMCompat::compareFloat(FloatRegister lhs, FloatRegister rhs)
 {
     // Compare the doubles, setting vector status flags.
-    if (rhs == InvalidFloatReg)
+    if (rhs.isMissing())
         as_vcmpz(VFPRegister(lhs).singleOverlay());
     else
         as_vcmp(VFPRegister(lhs).singleOverlay(), VFPRegister(rhs).singleOverlay());
@@ -3153,6 +3163,7 @@ MacroAssemblerARMCompat::unboxNonDouble(const Address &src, Register dest)
 void
 MacroAssemblerARMCompat::unboxDouble(const ValueOperand &operand, FloatRegister dest)
 {
+    MOZ_ASSERT(dest.isDouble());
     as_vxfer(operand.payloadReg(), operand.typeReg(),
              VFPRegister(dest), CoreToFloat);
 }
@@ -3160,6 +3171,7 @@ MacroAssemblerARMCompat::unboxDouble(const ValueOperand &operand, FloatRegister
 void
 MacroAssemblerARMCompat::unboxDouble(const Address &src, FloatRegister dest)
 {
+    MOZ_ASSERT(dest.isDouble());
     ma_vldr(Operand(src), dest);
 }
 
@@ -3729,8 +3741,12 @@ MacroAssemblerARMCompat::setupABICall(uint32_t args)
 #endif
     floatArgsInGPR[0] = MoveOperand();
     floatArgsInGPR[1] = MoveOperand();
+    floatArgsInGPR[2] = MoveOperand();
+    floatArgsInGPR[3] = MoveOperand();
     floatArgsInGPRValid[0] = false;
     floatArgsInGPRValid[1] = false;
+    floatArgsInGPRValid[2] = false;
+    floatArgsInGPRValid[3] = false;
 }
 
 void
@@ -3763,52 +3779,54 @@ MacroAssemblerARMCompat::passHardFpABIArg(const MoveOperand &from, MoveOp::Type
     if (!enoughMemory_)
         return;
     switch (type) {
-      case MoveOp::FLOAT32:
+      case MoveOp::FLOAT32: {
-      case MoveOp::DOUBLE: {
-        // N.B. This isn't a limitation of the ABI, it is a limitation of the
-        // compiler right now. There isn't a good way to handle odd numbered
-        // single registers, so everything goes to hell when we try. Current fix
-        // is to never use more than one float in a function call. Fix coming
-        // along with complete float32 support in bug 957504.
-        JS_ASSERT(!usedFloat32_);
-        if (type == MoveOp::FLOAT32)
-            usedFloat32_ = true;
         FloatRegister fr;
-        if (GetFloatArgReg(usedIntSlots_, usedFloatSlots_, &fr)) {
+        passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_Float32;
+        if (GetFloat32ArgReg(usedIntSlots_, usedFloatSlots_, &fr)) {
             if (from.isFloatReg() && from.floatReg() == fr) {
                 // Nothing to do; the value is in the right register already.
                 usedFloatSlots_++;
-                if (type == MoveOp::FLOAT32)
+                passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_Float32;
-                    passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_Float32;
-                else
-                    passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_Double;
                 return;
             }
             to = MoveOperand(fr);
         } else {
             // If (and only if) the integer registers have started spilling, do
             // we need to take the register's alignment into account.
-            uint32_t disp = INT_MAX;
+            uint32_t disp = GetFloat32ArgStackDisp(usedIntSlots_, usedFloatSlots_, &padding_);
-            if (type == MoveOp::FLOAT32)
-                disp = GetFloat32ArgStackDisp(usedIntSlots_, usedFloatSlots_, &padding_);
-            else
-                disp = GetDoubleArgStackDisp(usedIntSlots_, usedFloatSlots_, &padding_);
             to = MoveOperand(sp, disp);
         }
         usedFloatSlots_++;
-        if (type == MoveOp::FLOAT32)
-            passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_Float32;
-        else
-            passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_Double;
         break;
       }
+
+      case MoveOp::DOUBLE: {
+          FloatRegister fr;
+          passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_Double;
+          usedFloatSlots_ = (usedFloatSlots_ + 1) & -2;
+          if (GetDoubleArgReg(usedIntSlots_, usedFloatSlots_, &fr)) {
+              if (from.isFloatReg() && from.floatReg() == fr) {
+                  // Nothing to do; the value is in the right register already.
+                  usedFloatSlots_ += 2;
+                  return;
+              }
+              to = MoveOperand(fr);
+          } else {
+              // If (and only if) the integer registers have started spilling, do we
+              // need to take the register's alignment into account
+              uint32_t disp = GetDoubleArgStackDisp(usedIntSlots_, usedFloatSlots_, &padding_);
+              to = MoveOperand(sp, disp);
+          }
+          usedFloatSlots_+=2;
+          break;
+      }
       case MoveOp::GENERAL: {
         Register r;
+        passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_General;
         if (GetIntArgReg(usedIntSlots_, usedFloatSlots_, &r)) {
             if (from.isGeneralReg() && from.reg() == r) {
                 // Nothing to do; the value is in the right register already.
                 usedIntSlots_++;
-                passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_General;
                 return;
             }
             to = MoveOperand(r);
@@ -3817,7 +3835,6 @@ MacroAssemblerARMCompat::passHardFpABIArg(const MoveOperand &from, MoveOp::Type
             to = MoveOperand(sp, disp);
         }
         usedIntSlots_++;
-        passedArgTypes_ = (passedArgTypes_ << ArgType_Shift) | ArgType_General;
         break;
       }
       default:
@@ -3858,8 +3875,8 @@ MacroAssemblerARMCompat::passSoftFpABIArg(const MoveOperand &from, MoveOp::Type
     MoveOperand dest;
     if (GetIntArgReg(usedIntSlots_, 0, &destReg)) {
         if (type == MoveOp::DOUBLE || type == MoveOp::FLOAT32) {
-            floatArgsInGPR[destReg.code() >> 1] = from;
+            floatArgsInGPR[destReg.code()] = from;
-            floatArgsInGPRValid[destReg.code() >> 1] = true;
+            floatArgsInGPRValid[destReg.code()] = true;
             useResolver = false;
         } else if (from.isGeneralReg() && from.reg() == destReg) {
             // No need to move anything.
@@ -3946,13 +3963,24 @@ MacroAssemblerARMCompat::callWithABIPre(uint32_t *stackAdjust, bool callFromAsmJ
         emitter.emit(moveResolver_);
         emitter.finish();
     }
-    for (int i = 0; i < 2; i++) {
+    for (int i = 0; i < 4; i++) {
         if (floatArgsInGPRValid[i]) {
             MoveOperand from = floatArgsInGPR[i];
-            Register to0 = Register::FromCode(i * 2), to1 = Register::FromCode(i * 2 + 1);
+            Register to0 = Register::FromCode(i);
+            Register to1;
+
+            if (!from.isFloatReg() || from.floatReg().isDouble()) {
+                // Doubles need to be moved into a pair of aligned registers
+                // whether they come from the stack, or VFP registers.
+                to1 = Register::FromCode(i + 1);
+                MOZ_ASSERT(i % 2 == 0);
+            }
 
             if (from.isFloatReg()) {
-                ma_vxfer(VFPRegister(from.floatReg()), to0, to1);
+                if (from.floatReg().isDouble())
+                    ma_vxfer(from.floatReg(), to0, to1);
+                else
+                    ma_vxfer(from.floatReg(), to0);
             } else {
                 JS_ASSERT(from.isMemory());
                 // Note: We can safely use the MoveOperand's displacement here,
@@ -4095,6 +4123,7 @@ MacroAssemblerARMCompat::handleFailureWithHandler(void *handler)
 {
     // Reserve space for exception information.
     int size = (sizeof(ResumeFromException) + 7) & ~7;
+
     ma_sub(Imm32(size), sp);
     ma_mov(sp, r0);
 
@@ -4191,7 +4220,7 @@ MacroAssemblerARMCompat::floor(FloatRegister input, Register output, Label *bail
     Label handleZero;
     Label handleNeg;
     Label fin;
-    compareDouble(input, InvalidFloatReg);
+    compareDouble(input, NoVFPRegister);
     ma_b(&handleZero, Assembler::Equal);
     ma_b(&handleNeg, Assembler::Signed);
     // NaN is always a bail condition, just bail directly.
@@ -4242,7 +4271,7 @@ MacroAssemblerARMCompat::floorf(FloatRegister input, Register output, Label *bai
     Label handleZero;
     Label handleNeg;
     Label fin;
-    compareFloat(input, InvalidFloatReg);
+    compareFloat(input, NoVFPRegister);
     ma_b(&handleZero, Assembler::Equal);
     ma_b(&handleNeg, Assembler::Signed);
     // NaN is always a bail condition, just bail directly.
@@ -4294,7 +4323,7 @@ MacroAssemblerARMCompat::ceil(FloatRegister input, Register output, Label *bail)
     Label handlePos;
     Label fin;
 
-    compareDouble(input, InvalidFloatReg);
+    compareDouble(input, NoVFPRegister);
     // NaN is always a bail condition, just bail directly.
     ma_b(bail, Assembler::Overflow);
     ma_b(&handleZero, Assembler::Equal);
@@ -4347,7 +4376,7 @@ MacroAssemblerARMCompat::ceilf(FloatRegister input, Register output, Label *bail
     Label handlePos;
     Label fin;
 
-    compareFloat(input, InvalidFloatReg);
+    compareFloat(input, NoVFPRegister);
     // NaN is always a bail condition, just bail directly.
     ma_b(bail, Assembler::Overflow);
     ma_b(&handleZero, Assembler::Equal);

js/src/jit/arm/MacroAssembler-arm.h

@@ -433,20 +433,25 @@ private:
     {
         JS_ASSERT(sign == 1 || sign == -1);
 
-        int32_t delta = sign * sizeof(double);
+        int32_t delta = sign * sizeof(float);
         int32_t offset = 0;
-        RegisterIterator iter(set);
+        // Build up a new set, which is the sum of all of the single and double
+        // registers. This set can have up to 48 registers in it total
+        // s0-s31 and d16-d31
+        FloatRegisterSet mod = set.reduceSetForPush();
+
+        RegisterIterator iter(mod);
         while (iter.more()) {
             startFloatTransferM(ls, rm, mode, WriteBack);
-            int32_t reg = (*iter).code_;
+            int32_t reg = (*iter).code();
             do {
                 offset += delta;
+                if ((*iter).isDouble())
+                    offset += delta;
                 transferFloatReg(*iter);
-            } while ((++iter).more() && (*iter).code_ == (reg += sign));
+            } while ((++iter).more() && (*iter).code() == (reg += sign));
             finishFloatTransfer();
         }
-
-        JS_ASSERT(offset == static_cast<int32_t>(set.size() * sizeof(double)) * sign);
         return offset;
     }
 };
@@ -478,8 +483,8 @@ class MacroAssemblerARMCompat : public MacroAssemblerARM
 
     // Used to work around the move resolver's lack of support for moving into
     // register pairs, which the softfp ABI needs.
-    mozilla::Array<MoveOperand, 2> floatArgsInGPR;
+    mozilla::Array<MoveOperand, 4> floatArgsInGPR;
-    mozilla::Array<bool, 2> floatArgsInGPRValid;
+    mozilla::Array<bool, 4> floatArgsInGPRValid;
 
     // Compute space needed for the function call and set the properties of the
     // callee. It returns the space which has to be allocated for calling the

js/src/jit/arm/Simulator-arm.cpp

@@ -759,7 +759,7 @@ ArmDebugger::debug()
                                 printf("\n");
                             }
                         }
-                        for (uint32_t i = 0; i < FloatRegisters::Total; i++) {
+                        for (uint32_t i = 0; i < FloatRegisters::TotalPhys; i++) {
                             dvalue = getVFPDoubleRegisterValue(i);
                             uint64_t as_words = mozilla::BitwiseCast<uint64_t>(dvalue);
                             printf("%3s: %f 0x%08x %08x\n",
@@ -1321,28 +1321,28 @@ Simulator::set_dw_register(int dreg, const int *dbl)
 void
 Simulator::get_d_register(int dreg, uint64_t *value)
 {
-    MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::Total));
+    MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::TotalPhys));
     memcpy(value, vfp_registers_ + dreg * 2, sizeof(*value));
 }
 
 void
 Simulator::set_d_register(int dreg, const uint64_t *value)
 {
-    MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::Total));
+    MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::TotalPhys));
     memcpy(vfp_registers_ + dreg * 2, value, sizeof(*value));
 }
 
 void
 Simulator::get_d_register(int dreg, uint32_t *value)
 {
-    MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::Total));
+    MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::TotalPhys));
     memcpy(value, vfp_registers_ + dreg * 2, sizeof(*value) * 2);
 }
 
 void
 Simulator::set_d_register(int dreg, const uint32_t *value)
 {
-    MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::Total));
+    MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::TotalPhys));
     memcpy(vfp_registers_ + dreg * 2, value, sizeof(*value) * 2);
 }
 
@@ -1414,7 +1414,7 @@ Simulator::setVFPRegister(int reg_index, const InputType &value)
 {
     MOZ_ASSERT(reg_index >= 0);
     MOZ_ASSERT_IF(register_size == 1, reg_index < num_s_registers);
-    MOZ_ASSERT_IF(register_size == 2, reg_index < int(FloatRegisters::Total));
+    MOZ_ASSERT_IF(register_size == 2, reg_index < int(FloatRegisters::TotalPhys));
 
     char buffer[register_size * sizeof(vfp_registers_[0])];
     memcpy(buffer, &value, register_size * sizeof(vfp_registers_[0]));
@@ -1427,7 +1427,7 @@ ReturnType Simulator::getFromVFPRegister(int reg_index)
 {
     MOZ_ASSERT(reg_index >= 0);
     MOZ_ASSERT_IF(register_size == 1, reg_index < num_s_registers);
-    MOZ_ASSERT_IF(register_size == 2, reg_index < int(FloatRegisters::Total));
+    MOZ_ASSERT_IF(register_size == 2, reg_index < int(FloatRegisters::TotalPhys));
 
     ReturnType value = 0;
     char buffer[register_size * sizeof(vfp_registers_[0])];
@@ -2102,7 +2102,7 @@ Simulator::scratchVolatileRegisters(bool scratchFloat)
         uint64_t scratch_value_d = 0x5a5a5a5a5a5a5a5aLU ^ uint64_t(icount_) ^ (uint64_t(icount_) << 30);
         for (uint32_t i = d0; i < d8; i++)
             set_d_register(i, &scratch_value_d);
-        for (uint32_t i = d16; i < FloatRegisters::Total; i++)
+        for (uint32_t i = d16; i < FloatRegisters::TotalPhys; i++)
             set_d_register(i, &scratch_value_d);
     }
 }

js/src/jit/arm/Trampoline-arm.cpp

@@ -25,14 +25,14 @@ using namespace js;
 using namespace js::jit;
 
 static const FloatRegisterSet NonVolatileFloatRegs =
-    FloatRegisterSet((1 << FloatRegisters::d8) |
+    FloatRegisterSet((1ULL << FloatRegisters::d8) |
-                     (1 << FloatRegisters::d9) |
+                     (1ULL << FloatRegisters::d9) |
-                     (1 << FloatRegisters::d10) |
+                     (1ULL << FloatRegisters::d10) |
-                     (1 << FloatRegisters::d11) |
+                     (1ULL << FloatRegisters::d11) |
-                     (1 << FloatRegisters::d12) |
+                     (1ULL << FloatRegisters::d12) |
-                     (1 << FloatRegisters::d13) |
+                     (1ULL << FloatRegisters::d13) |
-                     (1 << FloatRegisters::d14) |
+                     (1ULL << FloatRegisters::d14) |
-                     (1 << FloatRegisters::d15));
+                     (1ULL << FloatRegisters::d15));
 
 static void
 GenerateReturn(MacroAssembler &masm, int returnCode, SPSProfiler *prof)
@@ -350,7 +350,6 @@ JitRuntime::generateInvalidator(JSContext *cx)
 {
     // See large comment in x86's JitRuntime::generateInvalidator.
     MacroAssembler masm(cx);
-    //masm.as_bkpt();
     // At this point, one of two things has happened:
     // 1) Execution has just returned from C code, which left the stack aligned
     // 2) Execution has just returned from Ion code, which left the stack unaligned.
@@ -365,9 +364,17 @@ JitRuntime::generateInvalidator(JSContext *cx)
         masm.transferReg(Register::FromCode(i));
     masm.finishDataTransfer();
 
+    // Since our datastructures for stack inspection are compile-time fixed,
+    // if there are only 16 double registers, then we need to reserve
+    // space on the stack for the missing 16.
+    if (FloatRegisters::ActualTotalPhys() != FloatRegisters::TotalPhys) {
+        int missingRegs = FloatRegisters::TotalPhys - FloatRegisters::ActualTotalPhys();
+        masm.ma_sub(Imm32(missingRegs * sizeof(double)), sp);
+    }
+
     masm.startFloatTransferM(IsStore, sp, DB, WriteBack);
-    for (uint32_t i = 0; i < FloatRegisters::Total; i++)
+    for (uint32_t i = 0; i < FloatRegisters::ActualTotalPhys(); i++)
-        masm.transferFloatReg(FloatRegister::FromCode(i));
+        masm.transferFloatReg(FloatRegister(i, FloatRegister::Double));
     masm.finishFloatTransfer();
 
     masm.ma_mov(sp, r0);
@@ -529,6 +536,7 @@ PushBailoutFrame(MacroAssembler &masm, uint32_t frameClass, Register spArg)
     // STEP 1a: Save our register sets to the stack so Bailout() can read
     // everything.
     // sp % 8 == 0
+
     masm.startDataTransferM(IsStore, sp, DB, WriteBack);
     // We don't have to push everything, but this is likely easier.
     // Setting regs_.
@@ -536,9 +544,16 @@ PushBailoutFrame(MacroAssembler &masm, uint32_t frameClass, Register spArg)
         masm.transferReg(Register::FromCode(i));
     masm.finishDataTransfer();
 
+    // Since our datastructures for stack inspection are compile-time fixed,
+    // if there are only 16 double registers, then we need to reserve
+    // space on the stack for the missing 16.
+    if (FloatRegisters::ActualTotalPhys() != FloatRegisters::TotalPhys) {
+        int missingRegs = FloatRegisters::TotalPhys - FloatRegisters::ActualTotalPhys();
+        masm.ma_sub(Imm32(missingRegs * sizeof(double)), sp);
+    }
     masm.startFloatTransferM(IsStore, sp, DB, WriteBack);
-    for (uint32_t i = 0; i < FloatRegisters::Total; i++)
+    for (uint32_t i = 0; i < FloatRegisters::ActualTotalPhys(); i++)
-        masm.transferFloatReg(FloatRegister::FromCode(i));
+        masm.transferFloatReg(FloatRegister(i, FloatRegister::Double));
     masm.finishFloatTransfer();
 
     // STEP 1b: Push both the "return address" of the function call (the address
@@ -591,7 +606,7 @@ GenerateBailoutThunk(JSContext *cx, MacroAssembler &masm, uint32_t frameClass)
     masm.ma_add(sp, Imm32(sizeOfBailoutInfo), sp);
     // Common size of a bailout frame.
     uint32_t bailoutFrameSize = sizeof(void *) + // frameClass
-                              sizeof(double) * FloatRegisters::Total +
+                              sizeof(double) * FloatRegisters::TotalPhys +
                               sizeof(void *) * Registers::Total;
 
     if (frameClass == NO_FRAME_SIZE_CLASS_ID) {
@@ -893,7 +908,7 @@ JitRuntime::generatePreBarrier(JSContext *cx, MIRType type)
     RegisterSet save;
     if (cx->runtime()->jitSupportsFloatingPoint) {
         save = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
-                           FloatRegisterSet(FloatRegisters::VolatileMask));
+                           FloatRegisterSet(FloatRegisters::VolatileDoubleMask));
     } else {
         save = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
                            FloatRegisterSet());

js/src/jit/shared/CodeGenerator-shared.cpp

@@ -769,12 +769,18 @@ CodeGeneratorShared::visitOutOfLineTruncateSlow(OutOfLineTruncateSlow *ool)
     Register dest = ool->dest();
 
     saveVolatile(dest);
+#ifdef JS_CODEGEN_ARM
+    if (ool->needFloat32Conversion()) {
+        masm.convertFloat32ToDouble(src, ScratchDoubleReg);
+        src = ScratchDoubleReg;
+    }
 
+#else
     if (ool->needFloat32Conversion()) {
         masm.push(src);
         masm.convertFloat32ToDouble(src, src);
     }
-
+#endif
     masm.setupUnalignedABICall(1, dest);
     masm.passABIArg(src, MoveOp::DOUBLE);
     if (gen->compilingAsmJS())
@@ -783,9 +789,10 @@ CodeGeneratorShared::visitOutOfLineTruncateSlow(OutOfLineTruncateSlow *ool)
         masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, js::ToInt32));
     masm.storeCallResult(dest);
 
+#ifndef JS_CODEGEN_ARM
     if (ool->needFloat32Conversion())
         masm.pop(src);
-
+#endif
     restoreVolatile(dest);
 
     masm.jump(ool->rejoin());

js/src/jit/x64/Architecture-x64.h

@@ -32,6 +32,12 @@ class Registers {
         static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
         return mozilla::CountPopulation32(x);
     }
+    static uint32_t FirstBit(SetType x) {
+        return mozilla::CountTrailingZeroes32(x);
+    }
+    static uint32_t LastBit(SetType x) {
+        return 31 - mozilla::CountLeadingZeroes32(x);
+    }
     static const char *GetName(Code code) {
         static const char * const Names[] = { "rax", "rcx", "rdx", "rbx",
                                               "rsp", "rbp", "rsi", "rdi",
@@ -180,7 +186,12 @@ struct FloatRegister {
         static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
         return mozilla::CountPopulation32(x);
     }
-
+    static uint32_t FirstBit(SetType x) {
+        return mozilla::CountTrailingZeroes32(x);
+    }
+    static uint32_t LastBit(SetType x) {
+        return 31 - mozilla::CountLeadingZeroes32(x);
+    }
     Code code_;
 
     static FloatRegister FromCode(uint32_t i) {

js/src/jit/x86/Architecture-x86.h

@@ -41,6 +41,12 @@ class Registers {
         static_assert(sizeof(SetType) == 1, "SetType must be 8 bits");
         return mozilla::CountPopulation32(x);
     }
+    static uint32_t FirstBit(SetType x) {
+        return mozilla::CountTrailingZeroes32(x);
+    }
+    static uint32_t LastBit(SetType x) {
+        return 31 - mozilla::CountLeadingZeroes32(x);
+    }
     static const char *GetName(Code code) {
         static const char * const Names[] = { "eax", "ecx", "edx", "ebx",
                                               "esp", "ebp", "esi", "edi" };
@@ -156,7 +162,12 @@ struct FloatRegister {
         static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
         return mozilla::CountPopulation32(x);
     }
-
+    static uint32_t FirstBit(SetType x) {
+        return mozilla::CountTrailingZeroes32(x);
+    }
+    static uint32_t LastBit(SetType x) {
+        return 31 - mozilla::CountLeadingZeroes32(x);
+    }
     Code code_;
 
     static FloatRegister FromCode(uint32_t i) {

js/src/jsapi-tests/testJitMoveEmitterCycles.cpp

@@ -5,7 +5,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#if defined(JS_ARM_SIMULATOR) && defined(EVERYONE_KNOWS_ABOUT_ALIASING)
+#if defined(JS_ARM_SIMULATOR)
 #include "jit/arm/Assembler-arm.h"
 #include "jit/arm/MoveEmitter-arm.h"
 #include "jit/arm/Simulator-arm.h"