Bug 803730 - Fix boxing of object input in instanceOf checks. r=sstangl,dvander

js/src/ion/CodeGenerator.cpp

@@ -4020,17 +4020,27 @@ CodeGenerator::emitInstanceOf(LInstruction *ins, Register rhs)
     Register rhsFlags = ToRegister(ins->getTemp(0));
     Register lhsTmp = ToRegister(ins->getTemp(0));
 
-    Label callHasInstance;
     Label boundFunctionCheck;
     Label boundFunctionDone;
     Label done;
     Label loopPrototypeChain;
 
+    JS_ASSERT(ins->isInstanceOfO() || ins->isInstanceOfV());
+    bool lhsIsValue = ins->isInstanceOfV();
+
     typedef bool (*pf)(JSContext *, HandleObject, HandleValue, JSBool *);
     static const VMFunction HasInstanceInfo = FunctionInfo<pf>(js::HasInstance);
 
-    OutOfLineCode *call = oolCallVM(HasInstanceInfo, ins, (ArgList(), rhs, ToValue(ins, 0)),
+    // If the lhs is an object, then the ValueOperand that gets sent to
-                                   StoreRegisterTo(output));
+    // HasInstance must be boxed first.  If the lhs is a value, it can
+    // be sent directly.  Hence the choice between ToValue and ToTempValue
+    // below.  Note that the same check is done below in the generated code
+    // and explicit boxing instructions emitted before calling the OOL code
+    // if we're handling a LInstanceOfO.
+
+    OutOfLineCode *call = oolCallVM(HasInstanceInfo, ins,
+        (ArgList(), rhs, lhsIsValue ? ToValue(ins, 0) : ToTempValue(ins, 0)),
+        StoreRegisterTo(output));
     if (!call)
         return false;
 
@@ -4052,7 +4062,18 @@ CodeGenerator::emitInstanceOf(LInstruction *ins, Register rhs)
 
     masm.loadBaseShape(rhsTmp, output);
     masm.cmpPtr(Address(output, BaseShape::offsetOfClass()), ImmWord(&js::FunctionClass));
-    masm.j(Assembler::NotEqual, call->entry());
+    if (lhsIsValue) {
+        // If the input LHS is a value, no boxing necessary.
+        masm.j(Assembler::NotEqual, call->entry());
+    } else {
+        // If the input LHS is raw object pointer, it must be boxed before
+        // calling into js::HasInstance.
+        Label dontCallHasInstance;
+        masm.j(Assembler::Equal, &dontCallHasInstance);
+        masm.boxNonDouble(JSVAL_TYPE_OBJECT, ToRegister(ins->getOperand(0)), ToTempValue(ins, 0));
+        masm.jump(call->entry());
+        masm.bind(&dontCallHasInstance);
+    }
 
     // Check Bound Function
     masm.loadPtr(Address(output, BaseShape::offsetOfFlags()), rhsFlags);
@@ -4086,7 +4107,7 @@ CodeGenerator::emitInstanceOf(LInstruction *ins, Register rhs)
     // When lhs is a value: The HasInstance for function objects always
     // return false when lhs isn't an object. So check if
     // lhs is an object and otherwise return false
-    if (ins->isInstanceOfV()) {
+    if (lhsIsValue) {
         Label isObject;
         ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LHS);
         masm.branchTestObject(Assembler::Equal, lhsValue, &isObject);
@@ -4125,7 +4146,17 @@ CodeGenerator::emitInstanceOf(LInstruction *ins, Register rhs)
     masm.loadPtr(Address(lhsTmp, offsetof(types::TypeObject, proto)), lhsTmp);
 
     // Bail out if we hit a lazy proto
-    masm.branch32(Assembler::Equal, lhsTmp, Imm32(1), call->entry());
+    if (lhsIsValue) {
+        masm.branch32(Assembler::Equal, lhsTmp, Imm32(1), call->entry());
+    } else {
+        // If the input LHS is raw object pointer, it must be boxed before
+        // calling into js::HasInstance.
+        Label dontCallHasInstance;
+        masm.branch32(Assembler::NotEqual, lhsTmp, Imm32(1), &dontCallHasInstance);
+        masm.boxNonDouble(JSVAL_TYPE_OBJECT, ToRegister(ins->getOperand(0)), ToTempValue(ins, 0));
+        masm.jump(call->entry());
+        masm.bind(&dontCallHasInstance);
+    }
 
     masm.testPtr(lhsTmp, lhsTmp);
     masm.j(Assembler::Zero, &done);

js/src/ion/arm/CodeGenerator-arm.cpp

@@ -1029,6 +1029,14 @@ CodeGeneratorARM::ToOutValue(LInstruction *ins)
     return ValueOperand(typeReg, payloadReg);
 }
 
+ValueOperand
+CodeGeneratorARM::ToTempValue(LInstruction *ins, size_t pos)
+{
+    Register typeReg = ToRegister(ins->getTemp(pos + TYPE_INDEX));
+    Register payloadReg = ToRegister(ins->getTemp(pos + PAYLOAD_INDEX));
+    return ValueOperand(typeReg, payloadReg);
+}
+
 bool
 CodeGeneratorARM::visitValue(LValue *value)
 {

js/src/ion/arm/CodeGenerator-arm.h

@@ -110,6 +110,7 @@ class CodeGeneratorARM : public CodeGeneratorShared
   protected:
     ValueOperand ToValue(LInstruction *ins, size_t pos);
     ValueOperand ToOutValue(LInstruction *ins);
+    ValueOperand ToTempValue(LInstruction *ins, size_t pos);
 
     // Functions for LTestVAndBranch.
     Register splitTagForTest(const ValueOperand &value);

js/src/ion/arm/MacroAssembler-arm.cpp

@@ -2228,6 +2228,12 @@ MacroAssemblerARMCompat::boxDouble(const FloatRegister &src, const ValueOperand
     as_vxfer(dest.payloadReg(), dest.typeReg(), VFPRegister(src), FloatToCore);
 }
 
+void
+MacroAssemblerARMCompat::boxNonDouble(JSValueType type, const Register &src, const ValueOperand &dest) {
+    if (src != dest.payloadReg())
+        ma_mov(src, dest.payloadReg());
+    ma_mov(ImmType(type), dest.typeReg());
+}
 
 void
 MacroAssemblerARMCompat::boolValueToDouble(const ValueOperand &operand, const FloatRegister &dest)

js/src/ion/arm/MacroAssembler-arm.h

@@ -551,6 +551,7 @@ class MacroAssemblerARMCompat : public MacroAssemblerARM
 
     // boxing code
     void boxDouble(const FloatRegister &src, const ValueOperand &dest);
+    void boxNonDouble(JSValueType type, const Register &src, const ValueOperand &dest);
 
     // Extended unboxing API. If the payload is already in a register, returns
     // that register. Otherwise, provides a move to the given scratch register,

js/src/ion/x64/CodeGenerator-x64.cpp

@@ -33,6 +33,12 @@ CodeGeneratorX64::ToOutValue(LInstruction *ins)
     return ValueOperand(ToRegister(ins->getDef(0)));
 }
 
+ValueOperand
+CodeGeneratorX64::ToTempValue(LInstruction *ins, size_t pos)
+{
+    return ValueOperand(ToRegister(ins->getTemp(pos)));
+}
+
 bool
 CodeGeneratorX64::visitDouble(LDouble *ins)
 {

js/src/ion/x64/CodeGenerator-x64.h

@@ -23,6 +23,7 @@ class CodeGeneratorX64 : public CodeGeneratorX86Shared
   protected:
     ValueOperand ToValue(LInstruction *ins, size_t pos);
     ValueOperand ToOutValue(LInstruction *ins);
+    ValueOperand ToTempValue(LInstruction *ins, size_t pos);
 
 
     void loadUnboxedValue(Operand source, MIRType type, const LDefinition *dest);

js/src/ion/x64/MacroAssembler-x64.h

@@ -647,6 +647,10 @@ class MacroAssemblerX64 : public MacroAssemblerX86Shared
     void boxDouble(const FloatRegister &src, const ValueOperand &dest) {
         movqsd(src, dest.valueReg());
     }
+    void boxNonDouble(JSValueType type, const Register &src, const ValueOperand &dest) {
+        JS_ASSERT(src != dest.valueReg());
+        boxValue(type, src, dest.valueReg());
+    }
 
     // Note that the |dest| register here may be ScratchReg, so we shouldn't
     // use it.

js/src/ion/x86/CodeGenerator-x86.cpp

@@ -70,6 +70,14 @@ CodeGeneratorX86::ToOutValue(LInstruction *ins)
     return ValueOperand(typeReg, payloadReg);
 }
 
+ValueOperand
+CodeGeneratorX86::ToTempValue(LInstruction *ins, size_t pos)
+{
+    Register typeReg = ToRegister(ins->getTemp(pos + TYPE_INDEX));
+    Register payloadReg = ToRegister(ins->getTemp(pos + PAYLOAD_INDEX));
+    return ValueOperand(typeReg, payloadReg);
+}
+
 bool
 CodeGeneratorX86::visitValue(LValue *value)
 {

js/src/ion/x86/CodeGenerator-x86.h

@@ -43,6 +43,7 @@ class CodeGeneratorX86 : public CodeGeneratorX86Shared
   protected:
     ValueOperand ToValue(LInstruction *ins, size_t pos);
     ValueOperand ToOutValue(LInstruction *ins);
+    ValueOperand ToTempValue(LInstruction *ins, size_t pos);
 
     void storeElementTyped(const LAllocation *value, MIRType valueType, MIRType elementType,
                            const Register &elements, const LAllocation *index);

js/src/ion/x86/MacroAssembler-x86.h

@@ -533,6 +533,11 @@ class MacroAssemblerX86 : public MacroAssemblerX86Shared
         psrldq(Imm32(4), src);
         movd(src, dest.typeReg());
     }
+    void boxNonDouble(JSValueType type, const Register &src, const ValueOperand &dest) {
+        if (src != dest.payloadReg())
+            movl(src, dest.payloadReg());
+        movl(ImmType(type), dest.typeReg());
+    }
     void unboxInt32(const ValueOperand &src, const Register &dest) {
         movl(src.payloadReg(), dest);
     }