Bug 1209585 - Fix possible memory leak if generating stub code fails with OOM r=jandem

js/src/jit-test/tests/baseline/bug1209585.js

@@ -0,0 +1,36 @@
+if (helperThreadCount() == 0)
+    quit();
+
+if (!("oomAtAllocation" in this && "resetOOMFailure" in this))
+    quit();
+
+if ("gczeal" in this)
+    gczeal(0);
+
+eval("g=function() {}")
+var lfGlobal = newGlobal();
+for (lfLocal in this) {
+    if (!(lfLocal in lfGlobal)) {
+        lfGlobal[lfLocal] = this[lfLocal];
+    }
+}
+lfGlobal.offThreadCompileScript(`
+if (!("oomAtAllocation" in this && "resetOOMFailure" in this))
+    gczeal(0);
+function oomTest(f) {
+    var i = 1;
+    do {
+        try {
+            oomAtAllocation(i);
+            f();
+            more = resetOOMFailure();
+        } catch (e) {
+            more = resetOOMFailure();
+        }
+        i++;
+    } while(more);
+}
+var g = newGlobal();
+oomTest(function() { new revocable(); });
+`);
+lfGlobal.runOffThreadScript();

js/src/jit/BaselineIC.cpp

@@ -6210,13 +6210,12 @@ ICGetProp_Fallback::Compiler::generateStubCode(MacroAssembler& masm)
     return true;
 }
 
-bool
+void
 ICGetProp_Fallback::Compiler::postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> code)
 {
     CodeOffsetLabel offset(returnOffset_);
     offset.fixup(&masm);
     cx->compartment()->jitCompartment()->initBaselineGetPropReturnAddr(code->raw() + offset.offset());
-    return true;
 }
 
 bool
@@ -7632,13 +7631,12 @@ ICSetProp_Fallback::Compiler::generateStubCode(MacroAssembler& masm)
     return true;
 }
 
-bool
+void
 ICSetProp_Fallback::Compiler::postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> code)
 {
     CodeOffsetLabel offset(returnOffset_);
     offset.fixup(&masm);
     cx->compartment()->jitCompartment()->initBaselineSetPropReturnAddr(code->raw() + offset.offset());
-    return true;
 }
 
 static void
@@ -9403,17 +9401,16 @@ ICCall_Fallback::Compiler::generateStubCode(MacroAssembler& masm)
     return true;
 }
 
-bool
+void
 ICCall_Fallback::Compiler::postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> code)
 {
     if (MOZ_UNLIKELY(isSpread_))
-        return true;
+        return;
 
     CodeOffsetLabel offset(returnOffset_);
     offset.fixup(&masm);
     cx->compartment()->jitCompartment()->initBaselineCallReturnAddr(code->raw() + offset.offset(),
                                                                     isConstructing_);
-    return true;
 }
 
 typedef bool (*CreateThisFn)(JSContext* cx, HandleObject callee, MutableHandleValue rval);

js/src/jit/BaselineIC.h

@@ -2341,7 +2341,7 @@ class ICGetProp_Fallback : public ICMonitoredFallbackStub
       protected:
         uint32_t returnOffset_;
         bool generateStubCode(MacroAssembler& masm);
-        bool postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> code);
+        void postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> code);
 
       public:
         explicit Compiler(JSContext* cx)
@@ -3334,7 +3334,7 @@ class ICSetProp_Fallback : public ICFallbackStub
       protected:
         uint32_t returnOffset_;
         bool generateStubCode(MacroAssembler& masm);
-        bool postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> code);
+        void postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> code);
 
       public:
         explicit Compiler(JSContext* cx)
@@ -3941,7 +3941,7 @@ class ICCall_Fallback : public ICMonitoredFallbackStub
         bool isSpread_;
         uint32_t returnOffset_;
         bool generateStubCode(MacroAssembler& masm);
-        bool postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> code);
+        void postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> code);
 
         virtual int32_t getKey() const {
             return static_cast<int32_t>(engine_) |

js/src/jit/SharedIC.cpp

@@ -719,10 +719,6 @@ ICStubCompiler::getStubCode()
     if (!newStubCode)
         return nullptr;
 
-    // After generating code, run postGenerateStubCode()
-    if (!postGenerateStubCode(masm, newStubCode))
-        return nullptr;
-
     // All barriers are emitted off-by-default, enable them if needed.
     if (cx->zone()->needsIncrementalBarrier())
         newStubCode->togglePreBarriers(true);
@@ -731,6 +727,10 @@ ICStubCompiler::getStubCode()
     if (!comp->putStubCode(cx, stubKey, newStubCode))
         return nullptr;
 
+    // After generating code, run postGenerateStubCode().  We must not fail
+    // after this point.
+    postGenerateStubCode(masm, newStubCode);
+
     MOZ_ASSERT(entersStubFrame_ == ICStub::CanMakeCalls(kind));
     MOZ_ASSERT(!inStubFrame_);
 

js/src/jit/SharedIC.h

@@ -997,9 +997,8 @@ class ICStubCompiler
     }
 
     virtual bool generateStubCode(MacroAssembler& masm) = 0;
-    virtual bool postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> genCode) {
-        return true;
-    }
+    virtual void postGenerateStubCode(MacroAssembler& masm, Handle<JitCode*> genCode) {}
+
     JitCode* getStubCode();
 
     ICStubCompiler(JSContext* cx, ICStub::Kind kind, Engine engine)