Bug 1136597 - Mark all relevant cross compartment debugger edges when sweeping r=terrence

js/src/gc/RootMarking.cpp

@@ -430,7 +430,7 @@ js::gc::GCRuntime::markRuntime(JSTracer *trc,
             if (!c->zone()->isCollecting())
                 c->markCrossCompartmentWrappers(trc);
         }
-        Debugger::markAllCrossCompartmentEdges(trc);
+        Debugger::markIncomingCrossCompartmentEdges(trc);
     }
 
     {

js/src/jit-test/tests/gc/bug-1136597.js

@@ -0,0 +1,24 @@
+// |jit-test| error:ReferenceError
+var evalInFrame = (function (global) {
+  var dbgGlobal = newGlobal();
+  var dbg = new dbgGlobal.Debugger();
+  return function evalInFrame(upCount, code) {
+    dbg.addDebuggee(global);
+  };
+})(this);
+var gTestcases = new Array();
+var gTc = gTestcases.length;
+function TestCase()
+  gTestcases[gTc++] = this;
+function checkCollation(extensionCoValue, usageValue) {
+    var collator = new Intl.Collator(["de-DE"]);
+    collator.resolvedOptions().collation;
+}
+checkCollation(undefined, "sort");
+checkCollation();
+for ( addpow = 0; addpow < 33; addpow++ ) {
+    new TestCase();
+}
+evalInFrame(0, "i(true)", true);
+gc(3, 'shrinking')
+eval("gc(); h = g1");

js/src/jsgc.cpp

@@ -2632,7 +2632,7 @@ GCRuntime::updatePointersToRelocatedCells()
 
         gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_ROOTS);
         Debugger::markAll(&trc);
-        Debugger::markAllCrossCompartmentEdges(&trc);
+        Debugger::markIncomingCrossCompartmentEdges(&trc);
 
         for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
             WeakMapBase::markAll(c, &trc);

js/src/vm/Debugger.cpp

@@ -2113,15 +2113,24 @@ Debugger::markCrossCompartmentEdges(JSTracer *trc)
  *
  * This happens during the initial mark phase, not iterative marking, because
  * all the edges being reported here are strong references.
+ *
+ * This method is also used during compacting GC to update cross compartment
+ * pointers in zones that are not currently being compacted.
  */
 /* static */ void
-Debugger::markAllCrossCompartmentEdges(JSTracer *trc)
+Debugger::markIncomingCrossCompartmentEdges(JSTracer *trc)
 {
     JSRuntime *rt = trc->runtime();
+    gc::State state = rt->gc.state();
+    MOZ_ASSERT(state == gc::MARK_ROOTS || state == gc::COMPACT);
 
     for (Debugger *dbg = rt->debuggerList.getFirst(); dbg; dbg = dbg->getNext()) {
-        if (!dbg->object->zone()->isCollecting())
+        Zone *zone = dbg->object->zone();
+        if ((state == gc::MARK_ROOTS && !zone->isCollecting()) ||
+            (state == gc::COMPACT && !zone->isGCCompacting()))
+        {
             dbg->markCrossCompartmentEdges(trc);
+        }
     }
 }
 

js/src/vm/Debugger.h

@@ -564,7 +564,7 @@ class Debugger : private mozilla::LinkedListElement<Debugger>
      * Debugger objects that are definitely live but not yet marked, it marks
      * them and returns true. If not, it returns false.
      */
-    static void markAllCrossCompartmentEdges(JSTracer *tracer);
+    static void markIncomingCrossCompartmentEdges(JSTracer *tracer);
     static bool markAllIteratively(GCMarker *trc);
     static void markAll(JSTracer *trc);
     static void sweepAll(FreeOp *fop);