bug 673795 - part2, using lists of avaiulable chunks for faster chunk selection. r=wmccloskey

--HG--
extra : rebase_source : ae4f5a82bc4042e341fdb5c08e3f0fe4b4ae8935

js/src/jsapi.cpp

@@ -2720,7 +2720,7 @@ JS_GetGCParameter(JSRuntime *rt, JSGCParamKey key)
       case JSGC_UNUSED_CHUNKS:
         return uint32(rt->gcEmptyChunkCount);
       case JSGC_TOTAL_CHUNKS:
-        return uint32(rt->gcUserChunkSet.count() + rt->gcSystemChunkSet.count() + rt->gcEmptyChunkCount);
+        return uint32(rt->gcChunkSet.count() + rt->gcEmptyChunkCount);
       default:
         JS_ASSERT(key == JSGC_NUMBER);
         return rt->gcNumber;

js/src/jscntxt.h

@@ -390,8 +390,29 @@ struct JSRuntime
     uint32              protoHazardShape;
 
     /* Garbage collector state, used by jsgc.c. */
-    js::GCChunkSet      gcUserChunkSet;
+
-    js::GCChunkSet      gcSystemChunkSet;
+    /*
+     * Set of all GC chunks with at least one allocated thing. The
+     * conservative GC uses it to quickly check if a possible GC thing points
+     * into an allocated chunk.
+     */
+    js::GCChunkSet      gcChunkSet;
+
+    /*
+     * Doubly-linked lists of chunks from user and system compartments. The GC
+     * allocates its arenas from the corresponding list and when all arenas
+     * in the list head are taken, then the chunk is removed from the list.
+     * During the GC when all arenas in a chunk become free, that chunk is
+     * removed from the list and scheduled for release.
+     */
+    js::gc::Chunk       *gcSystemAvailableChunkListHead;
+    js::gc::Chunk       *gcUserAvailableChunkListHead;
+
+    /*
+     * Singly-linked list of empty chunks and its length. We use the list not
+     * to release empty chunks immediately so they can be used for future
+     * allocations. This avoids very high overhead of chunk release/allocation.
+     */
     js::gc::Chunk       *gcEmptyChunkListHead;
     size_t              gcEmptyChunkCount;
 

js/src/jscompartment.cpp

@@ -127,7 +127,6 @@ JSCompartment::~JSCompartment()
 bool
 JSCompartment::init()
 {
-    chunk = NULL;
     for (unsigned i = 0; i < FINALIZE_LIMIT; i++)
         arenas[i].init();
     freeLists.init();
@@ -467,8 +466,6 @@ JSCompartment::markCrossCompartmentWrappers(JSTracer *trc)
 void
 JSCompartment::sweep(JSContext *cx, uint32 releaseInterval)
 {
-    chunk = NULL;
-
     /* Remove dead wrappers from the table. */
     for (WrapperMap::Enum e(crossCompartmentWrappers); !e.empty(); e.popFront()) {
         JS_ASSERT_IF(IsAboutToBeFinalized(cx, e.front().key.toGCThing()) &&

js/src/jscompartment.h

@@ -390,7 +390,6 @@ typedef HashSet<ScriptFilenameEntry *,
 struct JS_FRIEND_API(JSCompartment) {
     JSRuntime                    *rt;
     JSPrincipals                 *principals;
-    js::gc::Chunk                *chunk;
 
     js::gc::ArenaList            arenas[js::gc::FINALIZE_LIMIT];
     js::gc::FreeLists            freeLists;

js/src/jsgc.cpp

@@ -337,27 +337,49 @@ Chunk::init(JSRuntime *rt)
 
     for (size_t i = 0; i != JS_ARRAY_LENGTH(markingDelay); ++i)
         markingDelay[i].init();
+
+    /*
+     * The rest of info fields is initailzied in PickChunk. We do not clear
+     * the mark bitmap as that is done at the start of the next GC.
+     */
 }
 
-bool
+inline Chunk **
-Chunk::unused()
+GetAvailableChunkList(JSCompartment *comp)
 {
-    return info.numFree == ArenasPerChunk;
+    JSRuntime *rt = comp->rt;
+    return comp->isSystemCompartment
+           ? &rt->gcSystemAvailableChunkListHead
+           : &rt->gcUserAvailableChunkListHead;
 }
 
-bool
+inline void
-Chunk::hasAvailableArenas()
+Chunk::addToAvailableList(JSCompartment *comp)
 {
-    return info.numFree > 0;
+    Chunk **listHeadp = GetAvailableChunkList(comp);
+    JS_ASSERT(!info.prevp);
+    JS_ASSERT(!info.next);
+    info.prevp = listHeadp;
+    Chunk *head = *listHeadp;
+    if (head) {
+        JS_ASSERT(head->info.prevp == listHeadp);
+        head->info.prevp = &info.next;
+    }
+    info.next = head;
+    *listHeadp = this;
 }
 
-bool
+inline void
-Chunk::withinArenasRange(Cell *cell)
+Chunk::removeFromAvailableList()
 {
-    uintptr_t addr = uintptr_t(cell);
+    JS_ASSERT(info.prevp);
-    if (addr >= uintptr_t(&arenas[0]) && addr < uintptr_t(&arenas[ArenasPerChunk]))
+    *info.prevp = info.next;
-        return true;
+    if (info.next) {
-    return false;
+        JS_ASSERT(info.next->info.prevp == &info.next);
+        info.next->info.prevp = info.prevp;
+    }
+    info.prevp = NULL;
+    info.next = NULL;
 }
 
 template <size_t thingSize>
@@ -371,6 +393,9 @@ Chunk::allocateArena(JSContext *cx, unsigned thingKind)
     aheader->init(comp, thingKind, thingSize);
     --info.numFree;
 
+    if (!hasAvailableArenas())
+        removeFromAvailableList();
+
     JSRuntime *rt = info.runtime;
     Probes::resizeHeap(comp, rt->gcBytes, rt->gcBytes + ArenaSize);
     JS_ATOMIC_ADD(&rt->gcBytes, ArenaSize);
@@ -409,9 +434,15 @@ Chunk::releaseArena(ArenaHeader *aheader)
     aheader->next = info.emptyArenaListHead;
     info.emptyArenaListHead = aheader;
     ++info.numFree;
-    if (unused()) {
+    if (info.numFree == 1) {
-        rt->gcUserChunkSet.remove(this);
+        JS_ASSERT(!info.prevp);
-        rt->gcSystemChunkSet.remove(this);
+        JS_ASSERT(!info.next);
+        addToAvailableList(aheader->compartment);
+    } else if (!unused()) {
+        JS_ASSERT(info.prevp);
+    } else {
+        rt->gcChunkSet.remove(this);
+        removeFromAvailableList();
 
         /*
          * We keep empty chunks until we are done with finalization to allow
@@ -420,7 +451,7 @@ Chunk::releaseArena(ArenaHeader *aheader)
          * GC_SHRINK.
          */
         info.age = 0;
-        info.link = rt->gcEmptyChunkListHead;
+        info.next = rt->gcEmptyChunkListHead;
         rt->gcEmptyChunkListHead = this;
         rt->gcEmptyChunkCount++;
     }
@@ -450,34 +481,23 @@ ReleaseGCChunk(JSRuntime *rt, Chunk *p)
 inline Chunk *
 PickChunk(JSContext *cx)
 {
-    Chunk *chunk = cx->compartment->chunk;
+    JSCompartment *comp = cx->compartment;
-    if (chunk && chunk->hasAvailableArenas())
+    JSRuntime *rt = comp->rt;
+    Chunk **listHeadp = GetAvailableChunkList(comp);
+    Chunk *chunk = *listHeadp;
+    if (chunk)
         return chunk;
 
-    JSRuntime *rt = cx->runtime;
-    bool isSystemCompartment = cx->compartment->isSystemCompartment;
-
     /*
-     * The chunk used for the last allocation is full, search all chunks for
+     * We do not have available chunks, either get one from the empty set or
-     * free arenas.
+     * allocate one.
      */
-    GCChunkSet *chunkSet = isSystemCompartment ? &rt->gcSystemChunkSet : &rt->gcUserChunkSet;
-    for (GCChunkSet::Range r(chunkSet->all()); !r.empty(); r.popFront()) {
-        chunk = r.front();
-        if (chunk->hasAvailableArenas()) {
-            cx->compartment->chunk = chunk;
-            return chunk;
-        }
-    }
-
-    /* Use an empty chunk when available or allocate a new one. */
     chunk = rt->gcEmptyChunkListHead;
     if (chunk) {
         JS_ASSERT(chunk->unused());
-        JS_ASSERT(!rt->gcUserChunkSet.has(chunk));
+        JS_ASSERT(!rt->gcChunkSet.has(chunk));
-        JS_ASSERT(!rt->gcSystemChunkSet.has(chunk));
         JS_ASSERT(rt->gcEmptyChunkCount >= 1);
-        rt->gcEmptyChunkListHead = chunk->info.link;
+        rt->gcEmptyChunkListHead = chunk->info.next;
         rt->gcEmptyChunkCount--;
     } else {
         chunk = AllocateGCChunk(rt);
@@ -492,27 +512,18 @@ PickChunk(JSContext *cx)
      * FIXME bug 583732 - chunk is newly allocated and cannot be present in
      * the table so using ordinary lookupForAdd is suboptimal here.
      */
-    GCChunkSet::AddPtr p = chunkSet->lookupForAdd(chunk);
+    GCChunkSet::AddPtr p = rt->gcChunkSet.lookupForAdd(chunk);
     JS_ASSERT(!p);
-    if (!chunkSet->add(p, chunk)) {
+    if (!rt->gcChunkSet.add(p, chunk)) {
         ReleaseGCChunk(rt, chunk);
         return NULL;
     }
 
-    cx->compartment->chunk = chunk;
+    chunk->info.prevp = NULL;
-    return chunk;
+    chunk->info.next = NULL;
-}
+    chunk->addToAvailableList(comp);
 
-static void
+    return chunk;
-ReleaseEmptyGCChunks(JSRuntime *rt)
-{
-    for (Chunk *chunk = rt->gcEmptyChunkListHead; chunk; ) {
-        Chunk *next = chunk->info.link;
-        ReleaseGCChunk(rt, chunk);
-        chunk = next;
-    }
-    rt->gcEmptyChunkListHead = NULL;
-    rt->gcEmptyChunkCount = 0;
 }
 
 static void
@@ -520,23 +531,21 @@ ExpireGCChunks(JSRuntime *rt, JSGCInvocationKind gckind)
 {
     AutoLockGC lock(rt);
 
-    if (gckind == GC_SHRINK) {
+    /* Return old empty chunks to the system. */
-        ReleaseEmptyGCChunks(rt);
+    for (Chunk **chunkp = &rt->gcEmptyChunkListHead; *chunkp; ) {
-    } else {
+        JS_ASSERT(rt->gcEmptyChunkCount);
-        /* Return old empty chunks to the system. */
+        Chunk *chunk = *chunkp;
-        for (Chunk **chunkp = &rt->gcEmptyChunkListHead; *chunkp; ) {
+        JS_ASSERT(chunk->unused());
-            JS_ASSERT(rt->gcEmptyChunkCount);
+        JS_ASSERT(!rt->gcChunkSet.has(chunk));
-            Chunk *chunk = *chunkp;
+        JS_ASSERT(chunk->info.age <= MAX_EMPTY_CHUNK_AGE);
-            JS_ASSERT(chunk->info.age <= MAX_EMPTY_CHUNK_AGE);
+        if (gckind == GC_SHRINK || chunk->info.age == MAX_EMPTY_CHUNK_AGE) {
-            if (chunk->info.age == MAX_EMPTY_CHUNK_AGE) {
+            *chunkp = chunk->info.next;
-                *chunkp = chunk->info.link;
+            --rt->gcEmptyChunkCount;
-                --rt->gcEmptyChunkCount;
+            ReleaseGCChunk(rt, chunk);
-                ReleaseGCChunk(rt, chunk);
+        } else {
-            } else {
+            /* Keep the chunk but increase its age. */
-                /* Keep the chunk but increase its age. */
+            ++chunk->info.age;
-                ++chunk->info.age;
+            chunkp = &chunk->info.next;
-                chunkp = &chunk->info.link;
-            }
         }
     }
 }
@@ -576,14 +585,7 @@ static const int64 JIT_SCRIPT_EIGHTH_LIFETIME = 120 * 1000 * 1000;
 JSBool
 js_InitGC(JSRuntime *rt, uint32 maxbytes)
 {
-    /*
+    if (!rt->gcChunkSet.init(INITIAL_CHUNK_CAPACITY))
-     * Make room for at least 16 chunks so the table would not grow before
-     * the browser starts up.
-     */
-    if (!rt->gcUserChunkSet.init(INITIAL_CHUNK_CAPACITY))
-        return false;
-
-    if (!rt->gcSystemChunkSet.init(INITIAL_CHUNK_CAPACITY))
         return false;
 
     if (!rt->gcRootsHash.init(256))
@@ -725,8 +727,7 @@ MarkIfGCThingWord(JSTracer *trc, jsuword w)
 
     Chunk *chunk = Chunk::fromAddress(addr);
 
-    if (!trc->context->runtime->gcUserChunkSet.has(chunk) &&
+    if (!trc->context->runtime->gcChunkSet.has(chunk))
-        !trc->context->runtime->gcSystemChunkSet.has(chunk))
         return CGCT_NOTCHUNK;
 
     /*
@@ -933,13 +934,18 @@ js_FinishGC(JSRuntime *rt)
     rt->compartments.clear();
     rt->atomsCompartment = NULL;
 
-    for (GCChunkSet::Range r(rt->gcUserChunkSet.all()); !r.empty(); r.popFront())
+    rt->gcSystemAvailableChunkListHead = NULL;
+    rt->gcUserAvailableChunkListHead = NULL;
+    for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront())
         ReleaseGCChunk(rt, r.front());
-    for (GCChunkSet::Range r(rt->gcSystemChunkSet.all()); !r.empty(); r.popFront())
+    rt->gcChunkSet.clear();
-        ReleaseGCChunk(rt, r.front());
+    for (Chunk *chunk = rt->gcEmptyChunkListHead; chunk; ) {
-    rt->gcUserChunkSet.clear();
+        Chunk *next = chunk->info.next;
-    rt->gcSystemChunkSet.clear();
+        ReleaseGCChunk(rt, chunk);
-    ReleaseEmptyGCChunks(rt);
+        chunk = next;
+    }
+    rt->gcEmptyChunkListHead = NULL;
+    rt->gcEmptyChunkCount = 0;
 
 #ifdef JS_THREADSAFE
     rt->gcHelperThread.finish(rt);
@@ -2265,10 +2271,7 @@ MarkAndSweep(JSContext *cx, JSCompartment *comp, JSGCInvocationKind gckind GCTIM
     JS_ASSERT(gcmarker.getMarkColor() == BLACK);
     rt->gcMarkingTracer = &gcmarker;
 
-    for (GCChunkSet::Range r(rt->gcUserChunkSet.all()); !r.empty(); r.popFront())
+    for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront())
-        r.front()->bitmap.clear();
-
-    for (GCChunkSet::Range r(rt->gcSystemChunkSet.all()); !r.empty(); r.popFront())
         r.front()->bitmap.clear();
 
     if (comp) {

js/src/jsgc.h

@@ -514,8 +514,9 @@ struct MarkingDelay {
 
 /* The chunk header (located at the end of the chunk to preserve arena alignment). */
 struct ChunkInfo {
-    Chunk           *link;
     JSRuntime       *runtime;
+    Chunk           *next;
+    Chunk           **prevp;
     ArenaHeader     *emptyArenaListHead;
     size_t          age;
     size_t          numFree;
@@ -620,9 +621,17 @@ struct Chunk {
     }
 
     void init(JSRuntime *rt);
-    bool unused();
+
-    bool hasAvailableArenas();
+    bool unused() const {
-    bool withinArenasRange(Cell *cell);
+        return info.numFree == ArenasPerChunk;
+    }
+
+    bool hasAvailableArenas() const {
+        return info.numFree > 0;
+    }
+
+    inline void addToAvailableList(JSCompartment *compartment);
+    inline void removeFromAvailableList();
 
     template <size_t thingSize>
     ArenaHeader *allocateArena(JSContext *cx, unsigned thingKind);