[INFER] Reserve and check extra space when inlining frames, bug 646004.

js/src/jit-test/tests/jaeger/inline/bug646004.js

@@ -0,0 +1,9 @@
+function reportCompare (expected, actual, description) {}
+function f()
+{
+  f(f, 0x09AA, 0x09B0, f);
+}
+try {
+  reportCompare ("outer", f(),
+                 "Inner function statement should not have been called.");
+} catch (e) {}

js/src/jscntxt.h

@@ -656,6 +656,12 @@ class StackSpace
     static const size_t STACK_QUOTA    = (VALUES_PER_STACK_FRAME + 18) *
                                          JS_MAX_INLINE_CALL_COUNT;
 
+    /*
+     * Extra space to reserve on the stack before invoking the method JIT.
+     * This may be used for inlined stack frames.
+     */
+    static const size_t STACK_EXTRA    = (VALUES_PER_STACK_FRAME + 18) * 10;
+
     /* Kept as a member of JSThreadData; cannot use constructor/destructor. */
     bool init();
     void finish();

js/src/jscntxtinlines.h

@@ -287,8 +287,8 @@ StackSpace::getCallFrame(JSContext *cx, Value *firstUnused, uintN nactual,
 {
     JS_ASSERT(fun->script() == script);
 
-    /* Include an extra sizeof(JSStackFrame) for the method-jit. */
-    uintN nvals = VALUES_PER_STACK_FRAME + script->nslots;
+    /* Include extra space for inlining by the method-jit. */
+    uintN nvals = STACK_EXTRA + script->nslots;
     uintN nformal = fun->nargs;
 
     /* Maintain layout invariant: &formalArgs[0] == ((Value *)fp) - nformal. */
@@ -447,12 +447,12 @@ StackSpace::getStackLimit(JSContext *cx)
         return limit;
     if (ensureSpace(NULL /* don't report error */, sp, STACK_QUOTA))
         return limit;
-    uintN minimum = cx->fp()->numSlots() + VALUES_PER_STACK_FRAME;
+    uintN minimum = cx->fp()->numSlots() + STACK_EXTRA;
     return ensureSpace(cx, sp, minimum) ? sp + minimum : NULL;
 #else
     if (JS_LIKELY(limit <= end))
         return limit;
-    uintN minimum = cx->fp()->numSlots() + VALUES_PER_STACK_FRAME;
+    uintN minimum = cx->fp()->numSlots() + STACK_EXTRA;
     return ensureSpace(cx, sp, minimum) ? sp + minimum : NULL;
 #endif
 }

js/src/methodjit/Compiler.cpp

@@ -538,14 +538,13 @@ mjit::Compiler::generatePrologue()
         }
 
         /*
-         * Guard that there is enough stack space. Note we include the size of
-         * a second frame, to ensure we can create a frame from call sites.
-         * :FIXME: this check does not currently account for space from inlined frames,
-         * nor do checks made when pushing the frame from the interpreter.
+         * Guard that there is enough stack space. Note we reserve space for
+         * any inline frames we end up generating, or a callee's stack frame
+         * we write to before the callee checks the stack.
          */
-        masm.addPtr(Imm32((script->nslots + VALUES_PER_STACK_FRAME * 2) * sizeof(Value)),
-                    JSFrameReg,
-                    Registers::ReturnReg);
+        JS_STATIC_ASSERT(StackSpace::STACK_EXTRA >= VALUES_PER_STACK_FRAME);
+        uint32 nvals = script->nslots + VALUES_PER_STACK_FRAME + StackSpace::STACK_EXTRA;
+        masm.addPtr(Imm32(nvals * sizeof(Value)), JSFrameReg, Registers::ReturnReg);
         Jump stackCheck = masm.branchPtr(Assembler::AboveOrEqual, Registers::ReturnReg,
                                          FrameAddress(offsetof(VMFrame, stackLimit)));
 
@@ -3596,6 +3595,13 @@ mjit::Compiler::inlineScriptedFunction(uint32 argc, bool callingNew)
     if (types->objectCount >= INLINE_SITE_LIMIT)
         return Compile_InlineAbort;
 
+    /*
+     * Compute the maximum height we can grow the stack for inlined frames.
+     * We always reserve space for an extra stack frame pushed when making
+     * a call from the deepest inlined frame.
+     */
+    uint32 stackLimit = outerScript->nslots + StackSpace::STACK_EXTRA - VALUES_PER_STACK_FRAME;
+
     /*
      * Scan each of the possible callees for other conditions precluding
      * inlining. We only inline at a call site if all callees are inlineable.
@@ -3637,6 +3643,10 @@ mjit::Compiler::inlineScriptedFunction(uint32 argc, bool callingNew)
             checka = checka->parent;
         }
 
+        /* Watch for excessively deep nesting of inlined frames. */
+        if (frame.totalDepth() + VALUES_PER_STACK_FRAME + fun->script()->nslots >= stackLimit)
+            return Compile_InlineAbort;
+
         analyze::Script analysis;
         analysis.analyze(cx, script);
 

js/src/methodjit/InvokeHelpers.cpp

@@ -226,8 +226,8 @@ RemovePartialFrame(JSContext *cx, JSStackFrame *fp)
 void JS_FASTCALL
 stubs::HitStackQuota(VMFrame &f)
 {
-    /* Include space to push another frame. */
-    uintN nvals = f.fp()->script()->nslots + VALUES_PER_STACK_FRAME;
+    /* Include space for any inline frames. */
+    uintN nvals = f.fp()->script()->nslots + StackSpace::STACK_EXTRA;
     JS_ASSERT(f.regs.sp == f.fp()->base());
     if (f.cx->stack().bumpCommitAndLimit(f.entryfp, f.regs.sp, nvals, &f.stackLimit))
         return;