Fix for bug 606705 (Browser crash when using many web workers [@ GCGraphBuilder::NoteXPCOMChild ] or beyond). r=jst, a=beta7+

js/src/xpconnect/src/nsXPConnect.cpp

@@ -987,7 +987,7 @@ xpc_CreateGlobalObject(JSContext *cx, JSClass *clasp,
         xpc::PtrAndPrincipalHashKey *priv_key =
             new xpc::PtrAndPrincipalHashKey(ptr, uri);
         xpc::CompartmentPrivate *priv =
-            new xpc::CompartmentPrivate(priv_key, wantXrays);
+            new xpc::CompartmentPrivate(priv_key, wantXrays, NS_IsMainThread());
         if(!CreateNewCompartment(cx, clasp, principal, priv,
                                  global, compartment))
         {
@@ -1024,7 +1024,7 @@ xpc_CreateMTGlobalObject(JSContext *cx, JSClass *clasp,
         // threadsafety assumptions.
         nsCOMPtr<nsIPrincipal> principal(do_QueryInterface(ptr));
         xpc::CompartmentPrivate *priv =
-            new xpc::CompartmentPrivate(ptr, false);
+            new xpc::CompartmentPrivate(ptr, false, NS_IsMainThread());
         if(!CreateNewCompartment(cx, clasp, principal, priv, global,
                                  compartment))
         {

js/src/xpconnect/src/xpcjsruntime.cpp

@@ -408,16 +408,45 @@ void XPCJSRuntime::TraceXPConnectRoots(JSTracer *trc)
         JS_DHashTableEnumerate(&mJSHolders, TraceJSHolder, trc);
 }
 
+struct Closure
+{
+    JSContext *cx;
+    bool cycleCollectionEnabled;
+    nsCycleCollectionTraversalCallback *cb;
+};
+
+static void
+CheckParticipatesInCycleCollection(PRUint32 aLangID, void *aThing, void *aClosure)
+{
+    Closure *closure = static_cast<Closure*>(aClosure);
+
+    if(aLangID == nsIProgrammingLanguage::JAVASCRIPT && closure->cycleCollectionEnabled)
+    {
+        uint32 kind = js_GetGCThingTraceKind(aThing);
+        if(kind == JSTRACE_OBJECT)
+            closure->cycleCollectionEnabled =
+                xpc::ParticipatesInCycleCollection(closure->cx, static_cast<JSObject*>(aThing));
+        else if(kind == JSTRACE_STRING)
+            closure->cycleCollectionEnabled =
+                xpc::ParticipatesInCycleCollection(closure->cx,
+                                                   static_cast<JSString*>(aThing)->asCell());
+    }
+}
+
 static JSDHashOperator
 NoteJSHolder(JSDHashTable *table, JSDHashEntryHdr *hdr, uint32 number,
              void *arg)
 {
     ObjectHolder* entry = reinterpret_cast<ObjectHolder*>(hdr);
+    Closure *closure = static_cast<Closure*>(arg);
 
-    nsCycleCollectionTraversalCallback* cb =
-        static_cast<nsCycleCollectionTraversalCallback*>(arg);
-    cb->NoteRoot(nsIProgrammingLanguage::CPLUSPLUS, entry->holder,
-                 entry->tracer);
+    closure->cycleCollectionEnabled = PR_TRUE;
+    entry->tracer->Trace(entry->holder, CheckParticipatesInCycleCollection, closure);
+    if(!closure->cycleCollectionEnabled)
+        return JS_DHASH_NEXT;
+
+    closure->cb->NoteRoot(nsIProgrammingLanguage::CPLUSPLUS, entry->holder,
+                          entry->tracer);
 
     return JS_DHASH_NEXT;
 }
@@ -447,6 +476,8 @@ void XPCJSRuntime::AddXPConnectRoots(JSContext* cx,
                     nsXPConnect::JSContextParticipant());
     }
 
+    AutoLockJSGC lock(cx->runtime);
+
     XPCWrappedNativeScope::SuspectAllWrappers(this, cx, cb);
 
     for(XPCRootSetElem *e = mVariantRoots; e ; e = e->GetNextRoot())
@@ -454,12 +485,22 @@ void XPCJSRuntime::AddXPConnectRoots(JSContext* cx,
 
     for(XPCRootSetElem *e = mWrappedJSRoots; e ; e = e->GetNextRoot())
     {
-        nsIXPConnectWrappedJS *wrappedJS = static_cast<nsXPCWrappedJS*>(e);
-        cb.NoteXPCOMRoot(wrappedJS);
+        nsXPCWrappedJS *wrappedJS = static_cast<nsXPCWrappedJS*>(e);
+        JSObject *obj = wrappedJS->GetJSObject();
+
+        // Only suspect wrappedJSObjects that are in a compartment that
+        // participates in cycle collection.
+        if(!xpc::ParticipatesInCycleCollection(cx, obj))
+            continue;
+
+        cb.NoteXPCOMRoot(static_cast<nsIXPConnectWrappedJS *>(wrappedJS));
     }
 
     if(mJSHolders.ops)
-        JS_DHashTableEnumerate(&mJSHolders, NoteJSHolder, &cb);
+    {
+        Closure closure = { cx, PR_TRUE, &cb };
+        JS_DHashTableEnumerate(&mJSHolders, NoteJSHolder, &closure);
+    }
 }
 
 void
@@ -781,20 +822,6 @@ JSBool XPCJSRuntime::GCCallback(JSContext *cx, JSGCStatus status)
     return JS_TRUE;
 }
 
-// Auto JS GC lock helper.
-class AutoLockJSGC
-{
-public:
-    AutoLockJSGC(JSRuntime* rt) : mJSRuntime(rt) { JS_LOCK_GC(mJSRuntime); }
-    ~AutoLockJSGC() { JS_UNLOCK_GC(mJSRuntime); }
-private:
-    JSRuntime* mJSRuntime;
-
-    // Disable copy or assignment semantics.
-    AutoLockJSGC(const AutoLockJSGC&);
-    void operator=(const AutoLockJSGC&);
-};
-
 //static
 void
 XPCJSRuntime::WatchdogMain(void *arg)

js/src/xpconnect/src/xpcprivate.h

@@ -472,6 +472,21 @@ private:
     static void operator delete(void* /*memory*/) {}
 };
 
+/************************************************/
+
+class AutoLockJSGC
+{
+public:
+    AutoLockJSGC(JSRuntime* rt) : mJSRuntime(rt) { JS_LOCK_GC(mJSRuntime); }
+    ~AutoLockJSGC() { JS_UNLOCK_GC(mJSRuntime); }
+private:
+    JSRuntime* mJSRuntime;
+
+    // Disable copy or assignment semantics.
+    AutoLockJSGC(const AutoLockJSGC&);
+    void operator=(const AutoLockJSGC&);
+};
+
 /***************************************************************************
 ****************************************************************************
 *
@@ -4508,16 +4523,18 @@ namespace xpc {
 
 struct CompartmentPrivate
 {
-  CompartmentPrivate(PtrAndPrincipalHashKey *key, bool wantXrays)
+  CompartmentPrivate(PtrAndPrincipalHashKey *key, bool wantXrays, bool cycleCollectionEnabled)
     : key(key),
       ptr(nsnull),
-      wantXrays(wantXrays)
+      wantXrays(wantXrays),
+      cycleCollectionEnabled(cycleCollectionEnabled)
   {
   }
-  CompartmentPrivate(nsISupports *ptr, bool wantXrays)
+  CompartmentPrivate(nsISupports *ptr, bool wantXrays, bool cycleCollectionEnabled)
     : key(nsnull),
       ptr(ptr),
-      wantXrays(wantXrays)
+      wantXrays(wantXrays),
+      cycleCollectionEnabled(cycleCollectionEnabled)
   {
   }
 
@@ -4525,8 +4542,25 @@ struct CompartmentPrivate
   nsAutoPtr<PtrAndPrincipalHashKey> key;
   nsCOMPtr<nsISupports> ptr;
   bool wantXrays;
+  bool cycleCollectionEnabled;
 };
 
+inline bool
+CompartmentParticipatesInCycleCollection(JSContext *cx, JSCompartment *compartment)
+{
+   CompartmentPrivate *priv =
+       static_cast<CompartmentPrivate *>(JS_GetCompartmentPrivate(cx, compartment));
+   NS_ASSERTION(priv, "This should never be null!");
+
+   return priv->cycleCollectionEnabled;
+}
+
+inline bool
+ParticipatesInCycleCollection(JSContext *cx, js::gc::Cell *cell)
+{
+   return CompartmentParticipatesInCycleCollection(cx, cell->compartment());
+}
+
 }
 
 #ifdef XPC_IDISPATCH_SUPPORT

js/src/xpconnect/src/xpcwrappednativescope.cpp

@@ -415,17 +415,22 @@ WrappedNativeSuspecter(JSDHashTable *table, JSDHashEntryHdr *hdr,
     {
         NS_ASSERTION(NS_IsMainThread(), 
                      "Suspecting wrapped natives from non-main thread");
-        NS_ASSERTION(!JS_IsAboutToBeFinalized(closure->cx, wrapper->GetFlatJSObject()),
+
+        // Only suspect wrappedJSObjects that are in a compartment that
+        // participates in cycle collection.
+        JSObject* obj = wrapper->GetFlatJSObject();
+        if(!xpc::ParticipatesInCycleCollection(closure->cx, obj))
+            return JS_DHASH_NEXT;
+
+        NS_ASSERTION(!JS_IsAboutToBeFinalized(closure->cx, obj),
                      "WrappedNativeSuspecter attempting to touch dead object");
 
         // Only record objects that might be part of a cycle as roots, unless
         // the callback wants all traces (a debug feature).
-        if(!(closure->cb.WantAllTraces()) &&
-           !nsXPConnect::IsGray(wrapper->GetFlatJSObject()))
+        if(!(closure->cb.WantAllTraces()) && !nsXPConnect::IsGray(obj))
             return JS_DHASH_NEXT;
 
-        closure->cb.NoteRoot(nsIProgrammingLanguage::JAVASCRIPT,
-                             wrapper->GetFlatJSObject(),
+        closure->cb.NoteRoot(nsIProgrammingLanguage::JAVASCRIPT, obj,
                              nsXPConnect::GetXPConnect());
     }