Bug 777919 - Free LifoAlloc chunks on background thread (r=luke)

js/src/ds/LifoAlloc.cpp

@@ -60,41 +60,7 @@ LifoAlloc::freeAll()
         first = first->next();
         BumpChunk::delete_(victim);
     }
-    first = latest = NULL;
-}
-
-void
-LifoAlloc::freeUnused()
-{
-    /* Don't free anything if we have outstanding marks. */
-    if (markCount || !first)
-        return; 
-
-    JS_ASSERT(first && latest);
-
-    /* Rewind through any unused chunks. */
-    if (!latest->used()) {
-        BumpChunk *lastUsed = NULL;
-        for (BumpChunk *it = first; it != latest; it = it->next()) {
-            if (it->used())
-                lastUsed = it;
-        }
-        if (!lastUsed) {
-            freeAll();
-            return;
-        }
-        latest = lastUsed;
-    }
-
-    /* Free all chunks after |latest|. */
-    BumpChunk *it = latest->next();
-    while (it) {
-        BumpChunk *victim = it;
-        it = it->next();
-        BumpChunk::delete_(victim);
-    }
-
-    latest->setNext(NULL);
+    first = latest = last = NULL;
 }
 
 LifoAlloc::BumpChunk *
@@ -131,11 +97,48 @@ LifoAlloc::getOrCreateChunk(size_t n)
     if (!newChunk)
         return NULL;
     if (!first) {
-        latest = first = newChunk;
+        latest = first = last = newChunk;
     } else {
         JS_ASSERT(latest && !latest->next());
         latest->setNext(newChunk);
-        latest = newChunk;
+        latest = last = newChunk;
     }
     return newChunk;
 }
+
+void
+LifoAlloc::transferFrom(LifoAlloc *other)
+{
+    JS_ASSERT(!markCount);
+    JS_ASSERT(latest == first);
+    JS_ASSERT(!other->markCount);
+
+    if (!other->first)
+        return;
+
+    append(other->first, other->last);
+    other->first = other->last = other->latest = NULL;
+}
+
+void
+LifoAlloc::transferUnusedFrom(LifoAlloc *other)
+{
+    JS_ASSERT(!markCount);
+    JS_ASSERT(latest == first);
+
+    if (other->markCount || !other->first)
+        return;
+
+    /*
+     * Because of how getOrCreateChunk works, there may be unused chunks before
+     * |last|. We do not transfer those chunks. In most cases it is expected
+     * that last == first, so this should be a rare situation that, when it
+     * happens, should not recur.
+     */
+
+    if (other->latest->next()) {
+        append(other->latest->next(), other->last);
+        other->latest->setNext(NULL);
+        other->last = other->latest;
+    }
+}

js/src/ds/LifoAlloc.h

@@ -144,6 +144,7 @@ class LifoAlloc
 
     BumpChunk   *first;
     BumpChunk   *latest;
+    BumpChunk   *last;
     size_t      markCount;
     size_t      defaultChunkSize_;
 
@@ -161,11 +162,20 @@ class LifoAlloc
 
     void reset(size_t defaultChunkSize) {
         JS_ASSERT(RoundUpPow2(defaultChunkSize) == defaultChunkSize);
-        first = latest = NULL;
+        first = latest = last = NULL;
         defaultChunkSize_ = defaultChunkSize;
         markCount = 0;
     }
 
+    void append(BumpChunk *start, BumpChunk *end) {
+        JS_ASSERT(start && end);
+        if (last)
+            last->setNext(start);
+        else
+            first = latest = start;
+        last = end;
+    }
+
   public:
     explicit LifoAlloc(size_t defaultChunkSize) { reset(defaultChunkSize); }
 
@@ -176,6 +186,12 @@ class LifoAlloc
         other->reset(defaultChunkSize_);
     }
 
+    /* Append allocated chunks from |other|. They are removed from |other|. */
+    void transferFrom(LifoAlloc *other);
+
+    /* Append unused chunks from |other|. They are removed from |other|. */
+    void transferUnusedFrom(LifoAlloc *other);
+
     ~LifoAlloc() { freeAll(); }
 
     size_t defaultChunkSize() const { return defaultChunkSize_; }
@@ -183,9 +199,6 @@ class LifoAlloc
     /* Frees all held memory. */
     void freeAll();
 
-    /* Should be called on GC in order to release any held chunks. */
-    void freeUnused();
-
     JS_ALWAYS_INLINE
     void *alloc(size_t n) {
         JS_OOM_POSSIBLY_FAIL();

js/src/jsapi.cpp

@@ -721,6 +721,7 @@ JSRuntime::JSRuntime()
     ownerThread_(NULL),
 #endif
     tempLifoAlloc(TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
+    freeLifoAlloc(TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
     execAlloc_(NULL),
     bumpAlloc_(NULL),
 #ifdef JS_METHODJIT

js/src/jscntxt.h

@@ -387,9 +387,15 @@ struct JSRuntime : js::RuntimeFriendFields
     js::StackSpace stackSpace;
 
     /* Temporary arena pool used while compiling and decompiling. */
-    static const size_t TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE = 1 << 12;
+    static const size_t TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE = 4 * 1024;
     js::LifoAlloc tempLifoAlloc;
 
+    /*
+     * Free LIFO blocks are transferred to this allocator before being freed on
+     * the background GC thread.
+     */
+    js::LifoAlloc freeLifoAlloc;
+
   private:
     /*
      * Both of these allocators are used for regular expression code which is shared at the

js/src/jscompartment.cpp

@@ -571,7 +571,7 @@ JSCompartment::sweep(FreeOp *fop, bool releaseTypes)
 
         {
             gcstats::AutoPhase ap2(rt->gcStats, gcstats::PHASE_FREE_TI_ARENA);
-            oldAlloc.freeAll();
+            rt->freeLifoAlloc.transferFrom(&oldAlloc);
             if (types.constrainedOutputs) {
                 fop->delete_(types.constrainedOutputs);
                 types.constrainedOutputs = NULL;

js/src/jsgc.cpp

@@ -3071,6 +3071,8 @@ GCHelperThread::doSweep()
             freeElementsAndArray(array, array + FREE_ARRAY_LENGTH);
         }
         freeVector.resize(0);
+
+        rt->freeLifoAlloc.freeAll();
     }
 
     bool shrinking = shrinkFlag;
@@ -3141,17 +3143,12 @@ SweepCompartments(FreeOp *fop, gcreason::Reason gcReason)
 }
 
 static void
-PurgeRuntime(JSTracer *trc)
+PurgeRuntime(JSRuntime *rt)
 {
-    JSRuntime *rt = trc->runtime;
+    for (GCCompartmentsIter c(rt); !c.done(); c.next())
+        c->purge();
 
-    for (CompartmentsIter c(rt); !c.done(); c.next()) {
-        /* We can be called from StartVerifyBarriers with a non-GC marker. */
-        if (c->isCollecting() || !IS_GC_MARKING_TRACER(trc))
-            c->purge();
-    }
-
-    rt->tempLifoAlloc.freeUnused();
+    rt->freeLifoAlloc.transferUnusedFrom(&rt->tempLifoAlloc);
 
     rt->gsnCache.purge();
     rt->propertyCache.purge(rt);
@@ -3242,7 +3239,7 @@ BeginMarkPhase(JSRuntime *rt, bool isIncremental)
      */
     {
         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_PURGE);
-        PurgeRuntime(gcmarker);
+        PurgeRuntime(rt);
     }
 
     /*
@@ -3608,6 +3605,9 @@ EndSweepPhase(JSRuntime *rt, JSGCInvocationKind gckind, gcreason::Reason gcReaso
     {
         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_DESTROY);
 
+        if (!rt->gcSweepOnBackgroundThread)
+            rt->freeLifoAlloc.freeAll();
+
         /*
          * Sweep script filenames after sweeping functions in the generic loop
          * above. In this way when a scripted function's finalizer destroys the
@@ -4834,8 +4834,6 @@ StartVerifyPreBarriers(JSRuntime *rt)
 
     JS_TracerInit(trc, rt, AccumulateEdge);
 
-    PurgeRuntime(trc);
-
     const size_t size = 64 * 1024 * 1024;
     trc->root = (VerifyNode *)js_malloc(size);
     JS_ASSERT(trc->root);