Bug 826148 - Part 4: Ion IC (r=dvander)

js/src/ion/CodeGenerator.cpp

@@ -3844,6 +3844,8 @@ class OutOfLineCache : public OutOfLineCodeBase<CodeGenerator>
             return codegen->visitOutOfLineBindNameCache(this);
           case LInstruction::LOp_GetNameCache:
             return codegen->visitOutOfLineGetNameCache(this);
+          case LInstruction::LOp_CallsiteCloneCache:
+            return codegen->visitOutOfLineCallsiteCloneCache(this);
           default:
             JS_NOT_REACHED("Bad instruction");
             return false;
@@ -3883,6 +3885,41 @@ CodeGenerator::visitCache(LInstruction *ins)
     return true;
 }
 
+typedef JSObject *(*CallsiteCloneCacheFn)(JSContext *, size_t, HandleObject);
+static const VMFunction CallsiteCloneCacheInfo =
+    FunctionInfo<CallsiteCloneCacheFn>(CallsiteCloneCache);
+
+bool
+CodeGenerator::visitOutOfLineCallsiteCloneCache(OutOfLineCache *ool)
+{
+    LCallsiteCloneCache *lir = ool->cache()->toCallsiteCloneCache();
+    const MCallsiteCloneCache *mir = lir->mir();
+    Register callee = ToRegister(lir->callee());
+    RegisterSet liveRegs = lir->safepoint()->liveRegs();
+    Register output = ToRegister(lir->output());
+
+    IonCacheCallsiteClone cache(ool->getInlineJump(), ool->getInlineLabel(),
+                                masm.labelForPatch(), liveRegs,
+                                callee, mir->block()->info().script(), mir->callPc(), output);
+
+    JS_ASSERT(!mir->resumePoint());
+
+    size_t cacheIndex = allocateCache(cache);
+
+    saveLive(lir);
+
+    pushArg(callee);
+    pushArg(Imm32(cacheIndex));
+    if (!callVM(CallsiteCloneCacheInfo, lir))
+        return false;
+
+    masm.storeCallResult(output);
+    restoreLive(lir);
+
+    masm.jump(ool->rejoin());
+    return true;
+}
+
 typedef bool (*GetNameCacheFn)(JSContext *, size_t, HandleObject, MutableHandleValue);
 static const VMFunction GetNameCacheInfo =
     FunctionInfo<GetNameCacheFn>(GetNameCache);

js/src/ion/CodeGenerator.h

@@ -201,6 +201,7 @@ class CodeGenerator : public CodeGeneratorSpecific
     bool visitOutOfLineSetPropertyCache(OutOfLineCache *ool);
     bool visitOutOfLineBindNameCache(OutOfLineCache *ool);
     bool visitOutOfLineGetNameCache(OutOfLineCache *ool);
+    bool visitOutOfLineCallsiteCloneCache(OutOfLineCache *ool);
 
     bool visitGetPropertyCacheV(LGetPropertyCacheV *ins) {
         return visitCache(ins);
@@ -223,6 +224,9 @@ class CodeGenerator : public CodeGeneratorSpecific
     bool visitGetNameCache(LGetNameCache *ins) {
         return visitCache(ins);
     }
+    bool visitCallsiteCloneCache(LCallsiteCloneCache *ins) {
+        return visitCache(ins);
+    }
 
   private:
     bool visitCache(LInstruction *load);

js/src/ion/IonBuilder.cpp

@@ -3109,8 +3109,7 @@ IonBuilder::checkInlineableGetPropertyCache(uint32_t argc)
 }
 
 MPolyInlineDispatch *
-IonBuilder::makePolyInlineDispatch(JSContext *cx, AutoObjectVector &targets, int argc,
+IonBuilder::makePolyInlineDispatch(JSContext *cx, int argc, MGetPropertyCache *getPropCache,
-                                   MGetPropertyCache *getPropCache,
                                    types::StackTypeSet *types, types::StackTypeSet *barrier,
                                    MBasicBlock *bottom,
                                    Vector<MDefinition *, 8, IonAllocPolicy> &retvalDefns)
@@ -3207,7 +3206,8 @@ IonBuilder::makePolyInlineDispatch(JSContext *cx, AutoObjectVector &targets, int
     fallbackBlock->end(MGoto::New(fallbackEndBlock));
 
     // Create Call
-    MCall *call = MCall::New(NULL, argc + 1, argc, false);
+    MCall *call = MCall::New(NULL, argc + 1, argc, false,
+                             oracle->getCallTarget(script(), argc, pc));
     if (!call)
         return NULL;
 
@@ -3249,7 +3249,8 @@ IonBuilder::makePolyInlineDispatch(JSContext *cx, AutoObjectVector &targets, int
 }
 
 bool
-IonBuilder::inlineScriptedCall(AutoObjectVector &targets, uint32_t argc, bool constructing,
+IonBuilder::inlineScriptedCall(AutoObjectVector &targets, AutoObjectVector &originals,
+                               uint32_t argc, bool constructing,
                                types::StackTypeSet *types, types::StackTypeSet *barrier)
 {
 #ifdef DEBUG
@@ -3289,7 +3290,7 @@ IonBuilder::inlineScriptedCall(AutoObjectVector &targets, uint32_t argc, bool co
             int numCases = inlinePropTable->numEntries();
             IonSpew(IonSpew_Inlining, "Got inlineable property cache with %d cases", numCases);
 
-            inlinePropTable->trimToTargets(targets);
+            inlinePropTable->trimToAndMaybePatchTargets(targets, originals);
 
             // Trim the cases based on those that match the targets at this call site.
             IonSpew(IonSpew_Inlining, "%d inlineable cases left after trimming to %d targets",
@@ -3339,14 +3340,18 @@ IonBuilder::inlineScriptedCall(AutoObjectVector &targets, uint32_t argc, bool co
         // In the polymorphic case, we end the current block with a MPolyInlineDispatch instruction.
 
         // Create a PolyInlineDispatch instruction for this call site
-        MPolyInlineDispatch *disp = makePolyInlineDispatch(cx, targets, argc, getPropCache,
+        MPolyInlineDispatch *disp = makePolyInlineDispatch(cx, argc, getPropCache, types, barrier,
-                                                           types, barrier, bottom, retvalDefns);
+                                                           bottom, retvalDefns);
         if (!disp)
             return false;
+
+        // It's guaranteed that targets.length() == originals.length()
         for (size_t i = 0; i < targets.length(); i++) {
-            // Create an MConstant for the function
+            // Create an MConstant for the function. Note that we guard on the
-            JSFunction *func = targets[i]->toFunction();
+            // original function pointer, even if we have a clone, as we only
-            RootedFunction target(cx, func);
+            // clone at the callsite, so guarding on the clone would be
+            // guaranteed to fail.
+            JSFunction *func = originals[i]->toFunction();
             MConstant *constFun = MConstant::New(ObjectValue(*func));
 
             // Create new entry block for the inlined callee graph.
@@ -3693,6 +3698,21 @@ IonBuilder::createThis(HandleFunction target, MDefinition *callee)
     return createThisScripted(callee);
 }
 
+bool
+IonBuilder::anyFunctionIsCloneAtCallsite(types::StackTypeSet *funTypes)
+{
+    uint32_t count = funTypes->getObjectCount();
+    if (count < 1)
+        return false;
+
+    for (uint32_t i = 0; i < count; i++) {
+        JSObject *obj = funTypes->getSingleObject(i);
+        if (obj->isFunction() && obj->toFunction()->isCloneAtCallsite())
+            return true;
+    }
+    return false;
+}
+
 bool
 IonBuilder::jsop_funcall(uint32_t argc)
 {
@@ -3706,7 +3726,7 @@ IonBuilder::jsop_funcall(uint32_t argc)
     // If |Function.prototype.call| may be overridden, don't optimize callsite.
     RootedFunction native(cx, getSingleCallTarget(argc, pc));
     if (!native || !native->isNative() || native->native() != &js_fun_call)
-        return makeCall(native, argc, false);
+        return makeCall(native, argc, false, false);
 
     // Extract call target.
     types::StackTypeSet *funTypes = oracle->getCallArg(script(), argc, 0, pc);
@@ -3739,7 +3759,7 @@ IonBuilder::jsop_funcall(uint32_t argc)
     }
 
     // Call without inlining.
-    return makeCall(target, argc, false);
+    return makeCall(target, argc, false, false);
 }
 
 bool
@@ -3747,7 +3767,7 @@ IonBuilder::jsop_funapply(uint32_t argc)
 {
     RootedFunction native(cx, getSingleCallTarget(argc, pc));
     if (argc != 2)
-        return makeCall(native, argc, false);
+        return makeCall(native, argc, false, false);
 
     // Disable compilation if the second argument to |apply| cannot be guaranteed
     // to be either definitely |arguments| or definitely not |arguments|.
@@ -3758,7 +3778,7 @@ IonBuilder::jsop_funapply(uint32_t argc)
 
     // Fallback to regular call if arg 2 is not definitely |arguments|.
     if (isArgObj != DefinitelyArguments)
-        return makeCall(native, argc, false);
+        return makeCall(native, argc, false, false);
 
     if (!native ||
         !native->isNative() ||
@@ -3847,7 +3867,7 @@ IonBuilder::jsop_funapplyarguments(uint32_t argc)
     // Pop apply function.
     current->pop();
 
-    return makeCall(target, false, argFunc, thisArg, args);
+    return makeCall(target, false, false, argFunc, thisArg, args);
 }
 
 bool
@@ -3856,11 +3876,29 @@ IonBuilder::jsop_call(uint32_t argc, bool constructing)
     AssertCanGC();
 
     // Acquire known call target if existent.
-    AutoObjectVector targets(cx);
+    AutoObjectVector originals(cx);
-    uint32_t numTargets = getPolyCallTargets(argc, pc, targets, 4);
+    uint32_t numTargets = getPolyCallTargets(argc, pc, originals, 4);
     types::StackTypeSet *barrier;
     types::StackTypeSet *types = oracle->returnTypeSet(script(), pc, &barrier);
 
+    // If any call targets need to be cloned, clone them. Keep track of the
+    // originals as we need to case on them for poly inline.
+    bool hasClones = false;
+    AutoObjectVector targets(cx);
+    RootedFunction fun(cx);
+    RootedScript scriptRoot(cx, script());
+    for (uint32_t i = 0; i < numTargets; i++) {
+        fun = originals[i]->toFunction();
+        if (fun->isCloneAtCallsite()) {
+            fun = CloneFunctionAtCallsite(cx, fun, scriptRoot, pc);
+            if (!fun)
+                return false;
+            hasClones = true;
+        }
+        if (!targets.append(fun))
+            return false;
+    }
+
     // Attempt to inline native and scripted functions.
     if (inliningEnabled()) {
         // Inline a single native call if possible.
@@ -3877,14 +3915,34 @@ IonBuilder::jsop_call(uint32_t argc, bool constructing)
         }
 
         if (numTargets > 0 && makeInliningDecision(targets, argc))
-            return inlineScriptedCall(targets, argc, constructing, types, barrier);
+            return inlineScriptedCall(targets, originals, argc, constructing, types, barrier);
     }
 
     RootedFunction target(cx, NULL);
     if (numTargets == 1)
         target = targets[0]->toFunction();
 
-    return makeCallBarrier(target, argc, constructing, types, barrier);
+    return makeCallBarrier(target, argc, constructing, hasClones, types, barrier);
+}
+
+MDefinition *
+IonBuilder::makeCallsiteClone(HandleFunction target, MDefinition *fun)
+{
+    // Bake in the clone eagerly if we have a known target. We have arrived here
+    // because TI told us that the known target is a should-clone-at-callsite
+    // function, which means that target already is the clone.
+    if (target) {
+        MConstant *constant = MConstant::New(ObjectValue(*target));
+        current->add(constant);
+        return constant;
+    }
+
+    // Add a callsite clone IC if we have multiple targets. Note that we
+    // should have checked already that at least some targets are marked as
+    // should-clone-at-callsite.
+    MCallsiteCloneCache *clone = MCallsiteCloneCache::New(fun, pc);
+    current->add(clone);
+    return clone;
 }
 
 static bool
@@ -4002,18 +4060,19 @@ IonBuilder::popFormals(uint32_t argc, MDefinition **fun, MPassArg **thisArg,
 }
 
 MCall *
-IonBuilder::makeCallHelper(HandleFunction target, uint32_t argc, bool constructing)
+IonBuilder::makeCallHelper(HandleFunction target, uint32_t argc, bool constructing,
+                           bool cloneAtCallsite)
 {
     Vector<MPassArg *> args(cx);
     MPassArg *thisArg;
     MDefinition *fun;
 
     popFormals(argc, &fun, &thisArg, &args);
-    return makeCallHelper(target, constructing, fun, thisArg, args);
+    return makeCallHelper(target, constructing, cloneAtCallsite, fun, thisArg, args);
 }
 
 MCall *
-IonBuilder::makeCallHelper(HandleFunction target, bool constructing,
+IonBuilder::makeCallHelper(HandleFunction target, bool constructing, bool cloneAtCallsite,
                            MDefinition *fun, MPassArg *thisArg, Vector<MPassArg *> &args)
 {
     // This function may be called with mutated stack.
@@ -4027,7 +4086,8 @@ IonBuilder::makeCallHelper(HandleFunction target, bool constructing,
     if (target && !target->isNative())
         targetArgs = Max<uint32_t>(target->nargs, argc);
 
-    MCall *call = MCall::New(target, targetArgs + 1, argc, constructing);
+    MCall *call = MCall::New(target, targetArgs + 1, argc, constructing,
+                             target ? NULL : oracle->getCallTarget(script(), argc, pc));
     if (!call)
         return NULL;
 
@@ -4071,6 +4131,11 @@ IonBuilder::makeCallHelper(HandleFunction target, bool constructing,
     // Pass |this| and function.
     call->addArg(0, thisArg);
 
+    // Add a callsite clone IC for multiple targets which all should be
+    // callsite cloned, or bake in the clone for a single target.
+    if (cloneAtCallsite)
+        fun = makeCallsiteClone(target, fun);
+
     if (target && JSOp(*pc) == JSOP_CALL) {
         // We know we have a single call target.  Check whether the "this" types
         // are DOM types and our function a DOM function, and if so flag the
@@ -4084,6 +4149,7 @@ IonBuilder::makeCallHelper(HandleFunction target, bool constructing,
             call->setDOMFunction();
         }
     }
+
     call->initFunction(fun);
 
     current->add(call);
@@ -4113,7 +4179,7 @@ AdjustTypeBarrierForDOMCall(const JSJitInfo* jitinfo, types::StackTypeSet *types
 
 bool
 IonBuilder::makeCallBarrier(HandleFunction target, uint32_t argc,
-                            bool constructing,
+                            bool constructing, bool cloneAtCallsite,
                             types::StackTypeSet *types,
                             types::StackTypeSet *barrier)
 {
@@ -4122,17 +4188,18 @@ IonBuilder::makeCallBarrier(HandleFunction target, uint32_t argc,
     MDefinition *fun;
 
     popFormals(argc, &fun, &thisArg, &args);
-    return makeCallBarrier(target, constructing, fun, thisArg, args, types, barrier);
+    return makeCallBarrier(target, constructing, cloneAtCallsite,
+                           fun, thisArg, args, types, barrier);
 }
 
 bool
-IonBuilder::makeCallBarrier(HandleFunction target, bool constructing,
+IonBuilder::makeCallBarrier(HandleFunction target, bool constructing, bool cloneAtCallsite,
                             MDefinition *fun, MPassArg *thisArg,
                             Vector<MPassArg *> &args,
                             types::StackTypeSet *types,
                             types::StackTypeSet *barrier)
 {
-    MCall *call = makeCallHelper(target, constructing, fun, thisArg, args);
+    MCall *call = makeCallHelper(target, constructing, cloneAtCallsite, fun, thisArg, args);
     if (!call)
         return false;
 
@@ -4150,24 +4217,25 @@ IonBuilder::makeCallBarrier(HandleFunction target, bool constructing,
 }
 
 bool
-IonBuilder::makeCall(HandleFunction target, uint32_t argc, bool constructing)
+IonBuilder::makeCall(HandleFunction target, uint32_t argc, bool constructing, bool cloneAtCallsite)
 {
     Vector<MPassArg *> args(cx);
     MPassArg *thisArg;
     MDefinition *fun;
 
     popFormals(argc, &fun, &thisArg, &args);
-    return makeCall(target, constructing, fun, thisArg, args);
+    return makeCall(target, constructing, cloneAtCallsite, fun, thisArg, args);
 }
 
 bool
-IonBuilder::makeCall(HandleFunction target, bool constructing,
+IonBuilder::makeCall(HandleFunction target, bool constructing, bool cloneAtCallsite,
                      MDefinition *fun, MPassArg *thisArg,
                      Vector<MPassArg*> &args)
 {
     types::StackTypeSet *barrier;
     types::StackTypeSet *types = oracle->returnTypeSet(script(), pc, &barrier);
-    return makeCallBarrier(target, constructing, fun, thisArg, args, types, barrier);
+    return makeCallBarrier(target, constructing, cloneAtCallsite,
+                           fun, thisArg, args, types, barrier);
 }
 
 bool
@@ -6284,7 +6352,7 @@ IonBuilder::getPropTryCommonGetter(bool *emitted, HandleId id, types::StackTypeS
     current->add(wrapper);
     current->push(wrapper);
 
-    if (!makeCallBarrier(getter, 0, false, types, barrier))
+    if (!makeCallBarrier(getter, 0, false, false, types, barrier))
         return false;
 
     *emitted = true;
@@ -6450,7 +6518,7 @@ IonBuilder::jsop_setprop(HandlePropertyName name)
 
         // Call the setter. Note that we have to push the original value, not
         // the setter's return value.
-        MCall *call = makeCallHelper(setter, 1, false);
+        MCall *call = makeCallHelper(setter, 1, false, false);
         if (!call)
             return false;
 

js/src/ion/IonBuilder.h

@@ -419,22 +419,27 @@ class IonBuilder : public MIRGenerator
     bool jsop_call_inline(HandleFunction callee, uint32_t argc, bool constructing,
                           MConstant *constFun, MBasicBlock *bottom,
                           Vector<MDefinition *, 8, IonAllocPolicy> &retvalDefns);
-    bool inlineScriptedCall(AutoObjectVector &targets, uint32_t argc, bool constructing,
+    bool inlineScriptedCall(AutoObjectVector &targets, AutoObjectVector &originals,
+                            uint32_t argc, bool constructing,
                             types::StackTypeSet *types, types::StackTypeSet *barrier);
     bool makeInliningDecision(AutoObjectVector &targets, uint32_t argc);
 
+    bool anyFunctionIsCloneAtCallsite(types::StackTypeSet *funTypes);
+    MDefinition *makeCallsiteClone(HandleFunction target, MDefinition *fun);
     void popFormals(uint32_t argc, MDefinition **fun, MPassArg **thisArg,
                     Vector<MPassArg *> *args);
-    MCall *makeCallHelper(HandleFunction target, bool constructing,
+    MCall *makeCallHelper(HandleFunction target, bool constructing, bool cloneAtCallsite,
                           MDefinition *fun, MPassArg *thisArg, Vector<MPassArg *> &args);
-    MCall *makeCallHelper(HandleFunction target, uint32_t argc, bool constructing);
+    MCall *makeCallHelper(HandleFunction target, uint32_t argc, bool constructing,
+                          bool cloneAtCallsite);
     bool makeCallBarrier(HandleFunction target, uint32_t argc, bool constructing,
-                         types::StackTypeSet *types, types::StackTypeSet *barrier);
+                         bool cloneAtCallsite, types::StackTypeSet *types,
-    bool makeCallBarrier(HandleFunction target, bool constructing,
+                         types::StackTypeSet *barrier);
+    bool makeCallBarrier(HandleFunction target, bool constructing, bool cloneAtCallsite,
                          MDefinition *fun, MPassArg *thisArg, Vector<MPassArg *> &args,
                          types::StackTypeSet *types, types::StackTypeSet *barrier);
-    bool makeCall(HandleFunction target, uint32_t argc, bool constructing);
+    bool makeCall(HandleFunction target, uint32_t argc, bool constructing, bool cloneAtCallsite);
-    bool makeCall(HandleFunction target, bool constructing,
+    bool makeCall(HandleFunction target, bool constructing, bool cloneAtCallsite,
                   MDefinition *fun, MPassArg *thisArg, Vector<MPassArg *> &args);
 
     inline bool TestCommonPropFunc(JSContext *cx, types::StackTypeSet *types,
@@ -448,7 +453,7 @@ class IonBuilder : public MIRGenerator
     MGetPropertyCache *checkInlineableGetPropertyCache(uint32_t argc);
 
     MPolyInlineDispatch *
-    makePolyInlineDispatch(JSContext *cx, AutoObjectVector &targets, int argc,
+    makePolyInlineDispatch(JSContext *cx, int argc,
                            MGetPropertyCache *getPropCache,
                            types::StackTypeSet *types, types::StackTypeSet *barrier,
                            MBasicBlock *bottom,

js/src/ion/IonCaches.cpp

@@ -2108,3 +2108,72 @@ js::ion::GetNameCache(JSContext *cx, size_t cacheIndex, HandleObject scopeChain,
 
     return true;
 }
+
+bool
+IonCacheCallsiteClone::attach(JSContext *cx, IonScript *ion, HandleFunction original,
+                              HandleFunction clone)
+{
+    MacroAssembler masm;
+
+    // Guard against object identity on the original.
+    RepatchLabel exit;
+    CodeOffsetJump exitOffset = masm.branchPtrWithPatch(Assembler::NotEqual, calleeReg(),
+                                                        ImmWord(uintptr_t(original.get())), &exit);
+    masm.bind(&exit);
+
+    // Load the clone.
+    masm.movePtr(ImmWord(uintptr_t(clone.get())), outputReg());
+
+    RepatchLabel rejoin;
+    CodeOffsetJump rejoinOffset = masm.jumpWithPatch(&rejoin);
+    masm.bind(&rejoin);
+
+    Linker linker(masm);
+    IonCode *code = linker.newCode(cx);
+    if (!code)
+        return false;
+
+    rejoinOffset.fixup(&masm);
+    exitOffset.fixup(&masm);
+
+    if (ion->invalidated())
+        return true;
+
+    CodeLocationJump rejoinJump(code, rejoinOffset);
+    CodeLocationJump exitJump(code, exitOffset);
+    CodeLocationJump lastJump_ = lastJump();
+    PatchJump(lastJump_, CodeLocationLabel(code));
+    PatchJump(rejoinJump, rejoinLabel());
+    PatchJump(exitJump, cacheLabel());
+    updateLastJump(exitJump);
+
+    IonSpew(IonSpew_InlineCaches, "Generated CALL callee clone stub at %p", code->raw());
+    return true;
+}
+
+JSObject *
+js::ion::CallsiteCloneCache(JSContext *cx, size_t cacheIndex, HandleObject callee)
+{
+    AutoFlushCache afc ("CallsiteCloneCache");
+
+    // Act as the identity for functions that are not clone-at-callsite, as we
+    // generate this cache as long as some callees are clone-at-callsite.
+    RootedFunction fun(cx, callee->toFunction());
+    if (!fun->isCloneAtCallsite())
+        return fun;
+
+    IonScript *ion = GetTopIonJSScript(cx)->ionScript();
+    IonCacheCallsiteClone &cache = ion->getCache(cacheIndex).toCallsiteClone();
+
+    RootedFunction clone(cx, CloneFunctionAtCallsite(cx, fun, cache.callScript(), cache.callPc()));
+    if (!clone)
+        return NULL;
+
+    if (cache.stubCount() < MAX_STUBS) {
+        if (!cache.attach(cx, ion, fun, clone))
+            return NULL;
+        cache.incrementStubCount();
+    }
+
+    return clone;
+}

js/src/ion/IonCaches.h

@@ -23,6 +23,7 @@ class IonCacheSetProperty;
 class IonCacheGetElement;
 class IonCacheBindName;
 class IonCacheName;
+class IonCacheCallsiteClone;
 
 // Common structure encoding the state of a polymorphic inline cache contained
 // in the code for an IonScript. IonCaches are used for polymorphic operations
@@ -87,7 +88,8 @@ class IonCache
         GetElement,
         BindName,
         Name,
-        NameTypeOf
+        NameTypeOf,
+        CallsiteClone
     };
 
   protected:
@@ -139,6 +141,12 @@ class IonCache
             PropertyName *name;
             TypedOrValueRegisterSpace output;
         } name;
+        struct {
+            Register callee;
+            Register output;
+            JSScript *callScript;
+            jsbytecode *callPc;
+        } callsiteclone;
     } u;
 
     // Registers live after the cache, excluding output registers. The initial
@@ -236,6 +244,10 @@ class IonCache
         JS_ASSERT(kind_ == Name || kind_ == NameTypeOf);
         return *(IonCacheName *)this;
     }
+    IonCacheCallsiteClone &toCallsiteClone() {
+        JS_ASSERT(kind_ == CallsiteClone);
+        return *(IonCacheCallsiteClone *)this;
+    }
 
     void setScriptedLocation(UnrootedScript script, jsbytecode *pc) {
         JS_ASSERT(!idempotent_);
@@ -430,6 +442,39 @@ class IonCacheName : public IonCache
                 HandleShape shape);
 };
 
+class IonCacheCallsiteClone : public IonCache
+{
+  public:
+    IonCacheCallsiteClone(CodeOffsetJump initialJump,
+                          CodeOffsetLabel rejoinLabel,
+                          CodeOffsetLabel cacheLabel,
+                          RegisterSet liveRegs,
+                          Register callee, JSScript *callScript, jsbytecode *callPc,
+                          Register output)
+    {
+        init(CallsiteClone, liveRegs, initialJump, rejoinLabel, cacheLabel);
+        u.callsiteclone.callee = callee;
+        u.callsiteclone.callScript = callScript;
+        u.callsiteclone.callPc = callPc;
+        u.callsiteclone.output = output;
+    }
+
+    Register calleeReg() const {
+        return u.callsiteclone.callee;
+    }
+    HandleScript callScript() const {
+        return HandleScript::fromMarkedLocation(&u.callsiteclone.callScript);
+    }
+    jsbytecode *callPc() const {
+        return u.callsiteclone.callPc;
+    }
+    Register outputReg() const {
+        return u.callsiteclone.output;
+    }
+
+    bool attach(JSContext *cx, IonScript *ion, HandleFunction original, HandleFunction clone);
+};
+
 bool
 GetPropertyCache(JSContext *cx, size_t cacheIndex, HandleObject obj, MutableHandleValue vp);
 
@@ -447,6 +492,9 @@ BindNameCache(JSContext *cx, size_t cacheIndex, HandleObject scopeChain);
 bool
 GetNameCache(JSContext *cx, size_t cacheIndex, HandleObject scopeChain, MutableHandleValue vp);
 
+JSObject *
+CallsiteCloneCache(JSContext *cx, size_t cacheIndex, HandleObject callee);
+
 } // namespace ion
 } // namespace js
 

js/src/ion/LIR-Common.h

@@ -2811,6 +2811,22 @@ class LCallGetIntrinsicValue : public LCallInstructionHelper<BOX_PIECES, 0, 0>
     }
 };
 
+class LCallsiteCloneCache : public LInstructionHelper<1, 1, 0>
+{
+  public:
+    LIR_HEADER(CallsiteCloneCache);
+
+    LCallsiteCloneCache(const LAllocation &callee) {
+        setOperand(0, callee);
+    }
+    const LAllocation *callee() {
+        return getOperand(0);
+    }
+    const MCallsiteCloneCache *mir() const {
+        return mir_->toCallsiteCloneCache();
+    }
+};
+
 // Patchable jump to stubs generated for a GetProperty cache, which loads a
 // boxed value.
 class LGetPropertyCacheV : public LInstructionHelper<BOX_PIECES, 1, 0>

js/src/ion/LOpcodes.h

@@ -154,6 +154,7 @@
     _(CallGetProperty)              \
     _(GetNameCache)                 \
     _(CallGetIntrinsicValue)        \
+    _(CallsiteCloneCache)           \
     _(CallGetElement)               \
     _(CallSetElement)               \
     _(CallSetProperty)              \

js/src/ion/Lowering.cpp

@@ -1884,6 +1884,17 @@ LIRGenerator::visitCallGetIntrinsicValue(MCallGetIntrinsicValue *ins)
     return assignSafepoint(lir, ins);
 }
 
+bool
+LIRGenerator::visitCallsiteCloneCache(MCallsiteCloneCache *ins)
+{
+    JS_ASSERT(ins->callee()->type() == MIRType_Object);
+
+    LCallsiteCloneCache *lir = new LCallsiteCloneCache(useRegister(ins->callee()));
+    if (!define(lir, ins))
+        return false;
+    return assignSafepoint(lir, ins);
+}
+
 bool
 LIRGenerator::visitGetPropertyCache(MGetPropertyCache *ins)
 {

js/src/ion/Lowering.h

@@ -175,6 +175,7 @@ class LIRGenerator : public LIRGeneratorSpecific
     bool visitDeleteProperty(MDeleteProperty *ins);
     bool visitGetNameCache(MGetNameCache *ins);
     bool visitCallGetIntrinsicValue(MCallGetIntrinsicValue *ins);
+    bool visitCallsiteCloneCache(MCallsiteCloneCache *ins);
     bool visitCallGetElement(MCallGetElement *ins);
     bool visitCallSetElement(MCallSetElement *ins);
     bool visitSetPropertyCache(MSetPropertyCache *ins);

js/src/ion/MIR.cpp

@@ -395,10 +395,11 @@ MParameter::congruentTo(MDefinition * const &ins) const
 }
 
 MCall *
-MCall::New(JSFunction *target, size_t maxArgc, size_t numActualArgs, bool construct)
+MCall::New(JSFunction *target, size_t maxArgc, size_t numActualArgs, bool construct,
+           types::StackTypeSet *calleeTypes)
 {
     JS_ASSERT(maxArgc >= numActualArgs);
-    MCall *ins = new MCall(target, numActualArgs, construct);
+    MCall *ins = new MCall(target, numActualArgs, construct, calleeTypes);
     if (!ins->init(maxArgc + NumNonArgumentOperands))
         return NULL;
     return ins;

js/src/ion/MIR.h

@@ -1171,18 +1171,23 @@ class MCall
     CompilerRootFunction target_;
     // Original value of argc from the bytecode.
     uint32_t numActualArgs_;
+    // The typeset of the callee, could be NULL.
+    types::StackTypeSet *calleeTypes_;
 
-    MCall(JSFunction *target, uint32_t numActualArgs, bool construct)
+    MCall(JSFunction *target, uint32_t numActualArgs, bool construct,
+          types::StackTypeSet *calleeTypes)
       : construct_(construct),
         target_(target),
-        numActualArgs_(numActualArgs)
+        numActualArgs_(numActualArgs),
+        calleeTypes_(calleeTypes)
     {
         setResultType(MIRType_Value);
     }
 
   public:
     INSTRUCTION_HEADER(Call)
-    static MCall *New(JSFunction *target, size_t maxArgc, size_t numActualArgs, bool construct);
+    static MCall *New(JSFunction *target, size_t maxArgc, size_t numActualArgs, bool construct,
+                      types::StackTypeSet *calleeTypes);
 
     void initPrepareCall(MDefinition *start) {
         JS_ASSERT(start->isPrepareCall());
@@ -1214,6 +1219,9 @@ class MCall
     bool isConstructing() const {
         return construct_;
     }
+    types::StackTypeSet *calleeTypes() const {
+        return calleeTypes_;
+    }
 
     // The number of stack arguments is the max between the number of formal
     // arguments and the number of actual arguments. The number of stack
@@ -4343,12 +4351,16 @@ class InlinePropertyTable : public TempObject
         return entries_[i]->func;
     }
 
-    void trimToTargets(AutoObjectVector &targets) {
+    void trimToAndMaybePatchTargets(AutoObjectVector &targets, AutoObjectVector &originals) {
         size_t i = 0;
         while (i < numEntries()) {
             bool foundFunc = false;
-            for (size_t j = 0; j < targets.length(); j++) {
+            // Compare using originals, but if we find a matching function,
-                if (entries_[i]->func == targets[j]) {
+            // patch it to the target, which might be a clone.
+            for (size_t j = 0; j < originals.length(); j++) {
+                if (entries_[i]->func == originals[j]) {
+                    if (entries_[i]->func != targets[j])
+                        entries_[i] = new Entry(entries_[i]->typeObj, targets[j]->toFunction());
                     foundFunc = true;
                     break;
                 }
@@ -4976,6 +4988,41 @@ class MCallGetIntrinsicValue : public MNullaryInstruction
     }
 };
 
+class MCallsiteCloneCache
+  : public MUnaryInstruction,
+    public SingleObjectPolicy
+{
+    jsbytecode *callPc_;
+
+    MCallsiteCloneCache(MDefinition *callee, jsbytecode *callPc)
+      : MUnaryInstruction(callee),
+        callPc_(callPc)
+    {
+        setResultType(MIRType_Object);
+    }
+
+  public:
+    INSTRUCTION_HEADER(CallsiteCloneCache);
+
+    static MCallsiteCloneCache *New(MDefinition *callee, jsbytecode *callPc) {
+        return new MCallsiteCloneCache(callee, callPc);
+    }
+    TypePolicy *typePolicy() {
+        return this;
+    }
+    MDefinition *callee() const {
+        return getOperand(0);
+    }
+    jsbytecode *callPc() const {
+        return callPc_;
+    }
+
+    // Callsite cloning is idempotent.
+    AliasSet getAliasSet() const {
+        return AliasSet::None();
+    }
+};
+
 class MSetPropertyInstruction : public MBinaryInstruction
 {
     CompilerRootPropertyName name_;

js/src/ion/MOpcodes.h

@@ -123,6 +123,7 @@ namespace ion {
     _(CallGetProperty)                                                      \
     _(GetNameCache)                                                         \
     _(CallGetIntrinsicValue)                                                \
+    _(CallsiteCloneCache)                                                   \
     _(CallGetElement)                                                       \
     _(CallSetElement)                                                       \
     _(CallSetProperty)                                                      \

js/src/ion/VMFunctions.cpp

@@ -47,15 +47,27 @@ ShouldMonitorReturnType(JSFunction *fun)
 }
 
 bool
-InvokeFunction(JSContext *cx, HandleFunction fun, uint32_t argc, Value *argv, Value *rval)
+InvokeFunction(JSContext *cx, HandleFunction fun0, uint32_t argc, Value *argv, Value *rval)
 {
     AssertCanGC();
 
+    RootedFunction fun(cx, fun0);
+
     // In order to prevent massive bouncing between Ion and JM, see if we keep
     // hitting functions that are uncompilable.
     if (fun->isInterpreted()) {
         if (fun->isInterpretedLazy() && !JSFunction::getOrCreateScript(cx, fun))
             return false;
+
+        if (fun->isCloneAtCallsite()) {
+            RootedScript script(cx);
+            jsbytecode *pc;
+            types::TypeScript::GetPcScript(cx, &script, &pc);
+            fun = CloneFunctionAtCallsite(cx, fun0, script, pc);
+            if (!fun)
+                return false;
+        }
+
         if (!fun->nonLazyScript()->canIonCompile()) {
             UnrootedScript script = GetTopIonJSScript(cx);
             if (script->hasIonScript() &&

js/src/ion/VMFunctions.h

@@ -410,7 +410,7 @@ class AutoDetectInvalidation
     }
 };
 
-bool InvokeFunction(JSContext *cx, HandleFunction fun, uint32_t argc, Value *argv, Value *rval);
+bool InvokeFunction(JSContext *cx, HandleFunction fun0, uint32_t argc, Value *argv, Value *rval);
 JSObject *NewGCThing(JSContext *cx, gc::AllocKind allocKind, size_t thingSize);
 
 bool CheckOverRecursed(JSContext *cx);