diff --git a/js/src/jsgc.h b/js/src/jsgc.h index a2acda183e0..c009a285498 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -1604,12 +1604,13 @@ struct MarkStack { return true; } - bool push(T item1, T item2) { - T *nextTos = tos + 2; + bool push(T item1, T item2, T item3) { + T *nextTos = tos + 3; if (nextTos > limit) return false; tos[0] = item1; tos[1] = item2; + tos[2] = item3; tos = nextTos; return true; } @@ -1639,8 +1640,7 @@ struct GCMarker : public JSTracer { * the context of push or pop operation. * * Currently we need only 4 tags. However that can be extended to 8 if - * necessary. We tag either pointers to GC things or pointers to Value - * arrays. So the pointers are always at least 8-byte aligned. + * necessary as we tag only GC things. */ enum StackTag { ValueArrayTag, @@ -1654,6 +1654,7 @@ struct GCMarker : public JSTracer { static void staticAsserts() { JS_STATIC_ASSERT(StackTagMask >= uintptr_t(LastTag)); + JS_STATIC_ASSERT(StackTagMask <= gc::Cell::CellMask); } private: @@ -1730,16 +1731,6 @@ struct GCMarker : public JSTracer { if (!stack.push(addr | uintptr_t(tag))) delayMarkingChildren(ptr); } - - bool pushValueArray(void *start, void *end) { - JS_STATIC_ASSERT(ValueArrayTag == 0); - JS_ASSERT(start < end); - uintptr_t startAddr = reinterpret_cast(start); - uintptr_t endAddr = reinterpret_cast(end); - JS_ASSERT(!(startAddr & StackTagMask)); - JS_ASSERT(!(endAddr & StackTagMask)); - return stack.push(endAddr, startAddr); - } }; void diff --git a/js/src/jsgcmark.cpp b/js/src/jsgcmark.cpp index f06d602a0ea..242fa0141ec 100644 --- a/js/src/jsgcmark.cpp +++ b/js/src/jsgcmark.cpp @@ -51,16 +51,17 @@ /* * There are two mostly separate mark paths. The first is a fast path used * internally in the GC. The second is a slow path used for root marking and - * for API consumers like the cycle collector. + * for API consumers like the cycle collector or Class::trace implementations. * * The fast path uses explicit stacks. The basic marking process during a GC is * that all roots are pushed on to a mark stack, and then each item on the * stack is scanned (possibly pushing more stuff) until the stack is empty. * * PushMarkStack pushes a GC thing onto the mark stack. In some cases (shapes - * or strings) it eagerly marks the object rather than pushing it. Popping is - * done by the drainMarkStack method. For each thing it pops, drainMarkStack - * calls ScanObject (or a related function). + * or strings) it eagerly marks the object rather than pushing it. Popping and + * scanning is done by the processMarkStackTop method. For efficiency reasons + * like tail recursion elimination that method also implements the scanning of + * objects. For other GC things it uses helper methods. * * Most of the marking code outside jsgcmark uses functions like MarkObject, * MarkString, etc. These functions check if an object is in the compartment @@ -74,7 +75,7 @@ * mark stacks are not involved. These callbacks may ask for the outgoing * pointers to be visited. Eventually, this leads to the MarkChildren functions * being called. These functions duplicate much of the functionality of - * ScanObject, but they don't push onto an explicit stack. + * scanning functions, but they don't push onto an explicit stack. */ using namespace js; @@ -810,62 +811,24 @@ PushMarkStack(GCMarker *gcmarker, JSString *str) ScanString(gcmarker, str); } -static JS_NEVER_INLINE void -DelayMarkingValueArray(GCMarker *gcmarker, HeapValue *begin, HeapValue *end) -{ - for (HeapValue *vp = begin; vp != end; ++vp) { - const Value &v = *vp; - Cell *cell; - uint32 color; - if (v.isString()) { - cell = v.toString(); - color = BLACK; - } else if (v.isObject()) { - cell = &v.toObject(); - color = gcmarker->getMarkColor(); - } else { - continue; - } - if (cell->markIfUnmarked(color)) - gcmarker->delayMarkingChildren(cell); - } -} - static inline void -PushValueArray(GCMarker *gcmarker, HeapValue *array, size_t size) +PushValueArray(GCMarker *gcmarker, JSObject* obj, HeapValue *start, HeapValue *end) { - if (size != 0) { - JS_ASSERT(array); - HeapValue *end = array + size; - if (!gcmarker->pushValueArray(array, end)) - DelayMarkingValueArray(gcmarker, array, end); + JS_ASSERT(start <= end); + uintptr_t tagged = reinterpret_cast(obj) | GCMarker::ValueArrayTag; + uintptr_t startAddr = reinterpret_cast(start); + uintptr_t endAddr = reinterpret_cast(end); + + /* Push in the reverse order so obj will be on top. */ + if (!gcmarker->stack.push(endAddr, startAddr, tagged)) { + /* + * If we cannot push the array, we trigger delay marking for the whole + * object. + */ + gcmarker->delayMarkingChildren(obj); } } -static JS_ALWAYS_INLINE bool -ScanObjectWithoutSlots(GCMarker *gcmarker, JSObject *obj) -{ - types::TypeObject *type = obj->typeFromGC(); - PushMarkStack(gcmarker, type); - - js::Shape *shape = obj->lastProperty(); - PushMarkStack(gcmarker, shape); - - /* Call the trace hook if necessary. */ - Class *clasp = shape->getObjectClass(); - if (clasp->trace) { - if (clasp == &ArrayClass) { - PushValueArray(gcmarker, - obj->getDenseArrayElements(), - obj->getDenseArrayInitializedLength()); - } else { - clasp->trace(gcmarker, obj); - } - } - - return shape->isNative(); -} - void MarkChildren(JSTracer *trc, JSObject *obj) { @@ -1067,35 +1030,36 @@ inline void GCMarker::processMarkStackTop() { /* - * The code uses explicit goto to eliminate the tail recursion that - * compilers cannot optimize on their own. + * The function uses explicit goto and implements the scanning of the + * object directly. It allows to eliminate the tail recursion and + * significantly improve the marking performance, see bug 641025. */ HeapValue *vp, *end; JSObject *obj; uintptr_t addr = stack.pop(); uintptr_t tag = addr & StackTagMask; + addr &= ~StackTagMask; + if (tag == ValueArrayTag) { - /* - * We set ValueArrayTag to zero to avoid bit setting and clearing when - * pushing and poping tagged value array pointers. This is the most - * common stack operation as we push the array on the stack again when - * we find the next unmarked object in the array. - */ JS_STATIC_ASSERT(ValueArrayTag == 0); + JS_ASSERT(!(addr & Cell::CellMask)); + obj = reinterpret_cast(addr); uintptr_t addr2 = stack.pop(); - JS_ASSERT(addr <= addr2); - JS_ASSERT((addr2 - addr) % sizeof(Value) == 0); - vp = reinterpret_cast(addr); - end = reinterpret_cast(addr2); + uintptr_t addr3 = stack.pop(); + JS_ASSERT(addr2 <= addr3); + JS_ASSERT((addr3 - addr2) % sizeof(Value) == 0); + vp = reinterpret_cast(addr2); + end = reinterpret_cast(addr3); goto scan_value_array; } - addr &= ~StackTagMask; if (tag == ObjectTag) { obj = reinterpret_cast(addr); goto scan_obj; - } else if (tag == TypeTag) { + } + + if (tag == TypeTag) { ScanTypeObject(this, reinterpret_cast(addr)); } else { JS_ASSERT(tag == XmlTag); @@ -1104,43 +1068,62 @@ GCMarker::processMarkStackTop() return; scan_value_array: - JS_ASSERT(vp < end); - do { + JS_ASSERT(vp <= end); + while (vp != end) { const Value &v = *vp++; if (v.isString()) { JSString *str = v.toString(); if (str->markIfUnmarked()) ScanString(this, str); } else if (v.isObject()) { - obj = &v.toObject(); - if (obj->markIfUnmarked(getMarkColor())) { - if (vp != end && !pushValueArray(vp, end)) - DelayMarkingValueArray(this, vp, end); + JSObject *obj2 = &v.toObject(); + if (obj2->markIfUnmarked(getMarkColor())) { + PushValueArray(this, obj, vp, end); + obj = obj2; goto scan_obj; } } - } while (vp != end); + } return; scan_obj: - if (ScanObjectWithoutSlots(this, obj)) { + { + types::TypeObject *type = obj->typeFromGC(); + PushMarkStack(this, type); + + js::Shape *shape = obj->lastProperty(); + PushMarkStack(this, shape); + + /* Call the trace hook if necessary. */ + Class *clasp = shape->getObjectClass(); + if (clasp->trace) { + if (clasp == &ArrayClass) { + JS_ASSERT(!shape->isNative()); + vp = obj->getDenseArrayElements(); + end = vp + obj->getDenseArrayInitializedLength(); + goto scan_value_array; + } + clasp->trace(this, obj); + } + + if (!shape->isNative()) + return; + unsigned nslots = obj->slotSpan(); vp = obj->fixedSlots(); if (obj->slots) { unsigned nfixed = obj->numFixedSlots(); if (nslots > nfixed) { - PushValueArray(this, vp, nfixed); + PushValueArray(this, obj, vp, vp + nfixed); vp = obj->slots; end = vp + (nslots - nfixed); goto scan_value_array; } } - if (nslots) { - end = vp + nslots; - goto scan_value_array; - } + JS_ASSERT(nslots <= obj->numFixedSlots()); + end = vp + nslots; + goto scan_value_array; } - return; } void