diff --git a/js/src/Makefile.in b/js/src/Makefile.in index 46a213b5c53..9e8d426ebf0 100644 --- a/js/src/Makefile.in +++ b/js/src/Makefile.in @@ -216,8 +216,6 @@ VPATH += $(srcdir)/nanojit INSTALLED_HEADERS += \ jsbuiltins.h \ Assembler.h \ - Allocator.h \ - CodeAlloc.h \ LIR.h \ avmplus.h \ Fragmento.h \ @@ -230,8 +228,6 @@ INSTALLED_HEADERS += \ CPPSRCS += \ jstracer.cpp \ Assembler.cpp \ - Allocator.cpp \ - CodeAlloc.cpp \ Fragmento.cpp \ LIR.cpp \ RegAlloc.cpp \ diff --git a/js/src/jscntxt.h b/js/src/jscntxt.h index 5ed25c5113c..c3d2465472e 100644 --- a/js/src/jscntxt.h +++ b/js/src/jscntxt.h @@ -95,14 +95,11 @@ typedef struct VMSideExit VMSideExit; #ifdef __cplusplus namespace nanojit { - class Assembler; - class CodeAlloc; class Fragment; class Fragmento; class LirBuffer; } class TraceRecorder; -class VMAllocator; extern "C++" { template class Queue; } typedef Queue SlotList; @@ -143,9 +140,6 @@ struct JSTraceMonitor { CLS(nanojit::LirBuffer) lirbuf; CLS(nanojit::Fragmento) fragmento; - CLS(VMAllocator) allocator; // A chunk allocator for LIR. - CLS(nanojit::CodeAlloc) codeAlloc; // A general allocator for native code. - CLS(nanojit::Assembler) assembler; CLS(TraceRecorder) recorder; jsval *reservedDoublePool; jsval *reservedDoublePoolPtr; @@ -177,11 +171,9 @@ struct JSTraceMonitor { JSPackedBool useReservedObjects; JSObject *reservedObjects; - /* Parts for the regular expression compiler. This is logically + /* Fragmento for the regular expression compiler. This is logically * a distinct compiler but needs to be managed in exactly the same * way as the real tracing Fragmento. */ - CLS(VMAllocator) reAllocator; - CLS(nanojit::CodeAlloc) reCodeAlloc; CLS(nanojit::LirBuffer) reLirBuf; CLS(nanojit::Fragmento) reFragmento; diff --git a/js/src/jsregexp.cpp b/js/src/jsregexp.cpp index aa87b4e254c..8a098ee4265 100644 --- a/js/src/jsregexp.cpp +++ b/js/src/jsregexp.cpp @@ -2468,7 +2468,6 @@ class RegExpNativeCompiler { LIns* compileFlat(RENode *&node, LIns* pos, LInsList& fails) { - VMAllocator *alloc = JS_TRACE_MONITOR(cx).reAllocator; #ifdef USE_DOUBLE_CHAR_MATCH if (node->u.flat.length == 1) { if (node->next && node->next->op == REOP_FLAT && @@ -2484,7 +2483,7 @@ class RegExpNativeCompiler { } else { size_t i; for (i = 0; i < node->u.flat.length - 1; i += 2) { - if (alloc->outOfMemory()) + if (fragment->lirbuf->outOMem()) return 0; pos = compileFlatDoubleChar(((jschar*) node->kid)[i], ((jschar*) node->kid)[i+1], @@ -2502,7 +2501,7 @@ class RegExpNativeCompiler { return compileFlatSingleChar(node->u.flat.chr, pos, fails); } else { for (size_t i = 0; i < node->u.flat.length; i++) { - if (alloc->outOfMemory()) + if (fragment->lirbuf->outOMem()) return 0; pos = compileFlatSingleChar(((jschar*) node->kid)[i], pos, fails); if (!pos) @@ -2531,7 +2530,7 @@ class RegExpNativeCompiler { if (!charSet->converted && !ProcessCharSet(cx, re, charSet)) return NULL; LIns* skip = lirBufWriter->insSkip(bitmapLen); - if (JS_TRACE_MONITOR(cx).reAllocator->outOfMemory()) + if (fragment->lirbuf->outOMem()) return NULL; void* bitmapData = skip->payload(); memcpy(bitmapData, charSet->u.bits, bitmapLen); @@ -2929,9 +2928,8 @@ class RegExpNativeCompiler { */ LIns *compileNode(RENode *node, LIns *pos, bool atEnd, LInsList &fails) { - VMAllocator *alloc = JS_TRACE_MONITOR(cx).reAllocator; for (; pos && node; node = node->next) { - if (alloc->outOfMemory()) + if (fragment->lirbuf->outOMem()) return NULL; bool childNextIsEnd = atEnd && !node->next; @@ -3004,7 +3002,7 @@ class RegExpNativeCompiler { /* Failed to match on first character, so fail whole match. */ lir->ins1(LIR_ret, lir->insImm(0)); - return !JS_TRACE_MONITOR(cx).reAllocator->outOfMemory(); + return !fragment->lirbuf->outOMem(); } /* Compile normal regular expressions that can match starting at any char. */ @@ -3020,7 +3018,7 @@ class RegExpNativeCompiler { lir->insStorei(lir->ins2(LIR_piadd, start, lir->insImm(2)), state, offsetof(REGlobalData, skipped)); - return !JS_TRACE_MONITOR(cx).reAllocator->outOfMemory(); + return !fragment->lirbuf->outOMem(); } inline LIns* @@ -3062,12 +3060,10 @@ class RegExpNativeCompiler { { GuardRecord* guard = NULL; LIns* pos; - Assembler *assm; bool oom = false; const jschar* re_chars; size_t re_length; Fragmento* fragmento = JS_TRACE_MONITOR(cx).reFragmento; - VMAllocator *alloc = JS_TRACE_MONITOR(cx).reAllocator; re->source->getCharsAndLength(re_chars, re_length); /* @@ -3082,7 +3078,7 @@ class RegExpNativeCompiler { this->cx = cx; /* At this point we have an empty fragment. */ LirBuffer* lirbuf = fragment->lirbuf; - if (alloc->outOfMemory()) + if (lirbuf->outOMem()) goto fail; /* FIXME Use bug 463260 smart pointer when available. */ lir = lirBufWriter = new (&gc) LirBufWriter(lirbuf); @@ -3120,12 +3116,11 @@ class RegExpNativeCompiler { guard = insertGuard(re_chars, re_length); - if (alloc->outOfMemory()) + if (lirbuf->outOMem()) goto fail; - assm = JS_TRACE_MONITOR(cx).assembler; - ::compile(JS_TRACE_MONITOR(cx).reFragmento, assm, fragment); - if (assm->error() != nanojit::None) { - oom = assm->error() == nanojit::OutOMem; + ::compile(fragmento->assm(), fragment); + if (fragmento->assm()->error() != nanojit::None) { + oom = fragmento->assm()->error() == nanojit::OutOMem; goto fail; } @@ -3136,11 +3131,10 @@ class RegExpNativeCompiler { #endif return JS_TRUE; fail: - if (alloc->outOfMemory() || oom || + if (lirbuf->outOMem() || oom || js_OverfullFragmento(&JS_TRACE_MONITOR(cx), fragmento)) { fragmento->clearFrags(); - alloc->reset(); - lirbuf->clear(); + lirbuf->rewind(); } else { if (!guard) insertGuard(re_chars, re_length); re->flags |= JSREG_NOCOMPILE; diff --git a/js/src/jstracer.cpp b/js/src/jstracer.cpp index ed129fb3a76..67ac87e8645 100644 --- a/js/src/jstracer.cpp +++ b/js/src/jstracer.cpp @@ -261,38 +261,6 @@ static GC gc = GC(); static avmplus::AvmCore s_core = avmplus::AvmCore(); static avmplus::AvmCore* core = &s_core; -/* Allocator SPI implementation. */ - -void* -nanojit::Allocator::allocChunk(size_t nbytes) -{ - VMAllocator *vma = (VMAllocator*)this; - JS_ASSERT(!vma->outOfMemory()); - void *p = malloc(nbytes); - if (!p) { - JS_ASSERT(nbytes < sizeof(vma->mReserve)); - vma->mOutOfMemory = true; - p = (void*) &vma->mReserve[0]; - } - vma->mSize += nbytes; - return p; -} - -void -nanojit::Allocator::freeChunk(void *p) { - VMAllocator *vma = (VMAllocator*)this; - if (p != &vma->mReserve[0]) - free(p); -} - -void -nanojit::Allocator::postReset() { - VMAllocator *vma = (VMAllocator*)this; - vma->mOutOfMemory = false; - vma->mSize = 0; -} - - #ifdef JS_JIT_SPEW static void DumpPeerStability(JSTraceMonitor* tm, const void* ip, JSObject* globalObj, uint32 globalShape, uint32 argc); @@ -3164,8 +3132,7 @@ TraceRecorder::snapshot(ExitType exitType) } } - if (sizeof(VMSideExit) + (stackSlots + ngslots) * sizeof(JSTraceType) > - LirBuffer::MAX_SKIP_PAYLOAD_SZB) { + if (sizeof(VMSideExit) + (stackSlots + ngslots) * sizeof(JSTraceType) >= NJ_MAX_SKIP_PAYLOAD_SZB) { /* * ::snapshot() is infallible in the sense that callers don't * expect errors; but this is a trace-aborting error condition. So @@ -3237,7 +3204,7 @@ TraceRecorder::guard(bool expected, LIns* cond, VMSideExit* exit) /* * BIG FAT WARNING: If compilation fails we don't reset the lirbuf, so it's * safe to keep references to the side exits here. If we ever start - * clearing those lirbufs, we have to make sure we purge the side exits + * rewinding those lirbufs, we have to make sure we purge the side exits * that then no longer will be in valid memory. */ if (exit->exitType == LOOP_EXIT) @@ -3272,7 +3239,7 @@ TraceRecorder::copy(VMSideExit* copy) /* * BIG FAT WARNING: If compilation fails we don't reset the lirbuf, so it's * safe to keep references to the side exits here. If we ever start - * clearing those lirbufs, we have to make sure we purge the side exits + * rewinding those lirbufs, we have to make sure we purge the side exits * that then no longer will be in valid memory. */ if (exit->exitType == LOOP_EXIT) @@ -3704,7 +3671,7 @@ FlushJITCache(JSContext* cx) JS_ASSERT(fragmento->labels); fragmento->labels->clear(); #endif - + tm->lirbuf->rewind(); for (size_t i = 0; i < FRAGMENT_TABLE_SIZE; ++i) { VMFragment* f = tm->vmfragments[i]; while (f) { @@ -3719,8 +3686,6 @@ FlushJITCache(JSContext* cx) tm->globalStates[i].globalSlots->clear(); } } - tm->allocator->reset(); - tm->lirbuf->clear(); tm->needFlush = JS_FALSE; } @@ -3744,15 +3709,14 @@ TraceRecorder::compile(JSTraceMonitor* tm) } if (anchor && anchor->exitType != CASE_EXIT) ++treeInfo->branchCount; - if (tm->allocator->outOfMemory()) + if (lirbuf->outOMem()) { + fragmento->assm()->setError(nanojit::OutOMem); return; - - Assembler *assm = JS_TRACE_MONITOR(cx).assembler; - ::compile(fragmento, assm, fragment); - if (assm->error() == nanojit::OutOMem) + } + ::compile(fragmento->assm(), fragment); + if (fragmento->assm()->error() == nanojit::OutOMem) return; - - if (assm->error() != nanojit::None) { + if (fragmento->assm()->error() != nanojit::None) { debug_only_print0(LC_TMTracer, "Blacklisted: error during compilation\n"); Blacklist((jsbytecode*) fragment->root->ip); return; @@ -3762,10 +3726,10 @@ TraceRecorder::compile(JSTraceMonitor* tm) if (anchor) { #ifdef NANOJIT_IA32 if (anchor->exitType == CASE_EXIT) - assm->patch(anchor, anchor->switchInfo); + fragmento->assm()->patch(anchor, anchor->switchInfo); else #endif - assm->patch(anchor); + fragmento->assm()->patch(anchor); } JS_ASSERT(fragment->code()); JS_ASSERT(!fragment->vmprivate); @@ -3785,7 +3749,7 @@ TraceRecorder::compile(JSTraceMonitor* tm) } static bool -JoinPeersIfCompatible(Assembler* assm, Fragment* stableFrag, TreeInfo* stableTree, +JoinPeersIfCompatible(Fragmento* frago, Fragment* stableFrag, TreeInfo* stableTree, VMSideExit* exit) { JS_ASSERT(exit->numStackSlots == stableTree->nStackTypes); @@ -3797,7 +3761,7 @@ JoinPeersIfCompatible(Assembler* assm, Fragment* stableFrag, TreeInfo* stableTre } exit->target = stableFrag; - assm->patch(exit); + frago->assm()->patch(exit); stableTree->dependentTrees.addUnique(exit->from->root); ((TreeInfo*)exit->from->root->vmprivate)->linkedTrees.addUnique(stableFrag); @@ -3890,8 +3854,7 @@ TraceRecorder::closeLoop(JSTraceMonitor* tm, bool& demote) } compile(tm); - Assembler *assm = JS_TRACE_MONITOR(cx).assembler; - if (assm->error() != nanojit::None) + if (fragmento->assm()->error() != nanojit::None) return; joinEdgesToEntry(fragmento, peer_root); @@ -3936,8 +3899,7 @@ TraceRecorder::joinEdgesToEntry(Fragmento* fragmento, VMFragment* peer_root) uexit = ti->unstableExits; unext = &ti->unstableExits; while (uexit != NULL) { - Assembler *assm = JS_TRACE_MONITOR(cx).assembler; - bool remove = JoinPeersIfCompatible(assm, fragment, treeInfo, uexit->exit); + bool remove = JoinPeersIfCompatible(fragmento, fragment, treeInfo, uexit->exit); JS_ASSERT(!remove || fragment != peer); debug_only_stmt( if (remove) { @@ -4021,8 +3983,7 @@ TraceRecorder::endLoop(JSTraceMonitor* tm) lir->insGuard(LIR_x, NULL, createGuardRecord(snapshot(LOOP_EXIT))); compile(tm); - Assembler *assm = JS_TRACE_MONITOR(cx).assembler; - if (assm->error() != nanojit::None) + if (tm->fragmento->assm()->error() != nanojit::None) return; VMFragment* root = (VMFragment*)fragment->root; @@ -4355,8 +4316,7 @@ DeleteRecorder(JSContext* cx) tm->recorder = NULL; /* If we ran out of memory, flush the code cache. */ - Assembler *assm = JS_TRACE_MONITOR(cx).assembler; - if (assm->error() == OutOMem || + if (JS_TRACE_MONITOR(cx).fragmento->assm()->error() == OutOMem || js_OverfullFragmento(tm, tm->fragmento)) { FlushJITCache(cx); return false; @@ -4455,8 +4415,7 @@ StartRecorder(JSContext* cx, VMSideExit* anchor, Fragment* f, TreeInfo* ti, } /* Clear any leftover error state. */ - Assembler *assm = JS_TRACE_MONITOR(cx).assembler; - assm->setError(None); + tm->fragmento->assm()->setError(None); return true; } @@ -4469,9 +4428,10 @@ TrashTree(JSContext* cx, Fragment* f) return; AUDIT(treesTrashed); debug_only_print0(LC_TMTracer, "Trashing tree info.\n"); + Fragmento* fragmento = JS_TRACE_MONITOR(cx).fragmento; TreeInfo* ti = (TreeInfo*)f->vmprivate; f->vmprivate = NULL; - f->releaseCode(JS_TRACE_MONITOR(cx).codeAlloc); + f->releaseCode(fragmento); Fragment** data = ti->dependentTrees.data(); unsigned length = ti->dependentTrees.length(); for (unsigned n = 0; n < length; ++n) @@ -4723,7 +4683,7 @@ RecordTree(JSContext* cx, JSTraceMonitor* tm, Fragment* f, jsbytecode* outer, f->root = f; f->lirbuf = tm->lirbuf; - if (tm->allocator->outOfMemory() || js_OverfullFragmento(tm, tm->fragmento)) { + if (f->lirbuf->outOMem() || js_OverfullFragmento(tm, tm->fragmento)) { Backoff(cx, (jsbytecode*) f->root->ip); FlushJITCache(cx); debug_only_print0(LC_TMTracer, @@ -4879,8 +4839,7 @@ AttemptToStabilizeTree(JSContext* cx, JSObject* globalObj, VMSideExit* exit, jsb if (ti->nGlobalTypes() < ti->globalSlots->length()) SpecializeTreesToMissingGlobals(cx, globalObj, ti); exit->target = f; - Assembler *assm = JS_TRACE_MONITOR(cx).assembler; - assm->patch(exit); + tm->fragmento->assm()->patch(exit); /* Now erase this exit from the unstable exit list. */ UnstableExit** tail = &from_ti->unstableExits; @@ -6037,8 +5996,6 @@ js_MonitorLoopEdge(JSContext* cx, uintN& inlineCallCount) JS_REQUIRES_STACK JSRecordingStatus TraceRecorder::monitorRecording(JSContext* cx, TraceRecorder* tr, JSOp op) { - Assembler *assm = JS_TRACE_MONITOR(cx).assembler; - /* Process needFlush and deepAbort() requests now. */ if (JS_TRACE_MONITOR(cx).needFlush) { FlushJITCache(cx); @@ -6100,15 +6057,14 @@ TraceRecorder::monitorRecording(JSContext* cx, TraceRecorder* tr, JSOp op) return JSRS_STOP; } - if (assm->error()) { + if (JS_TRACE_MONITOR(cx).fragmento->assm()->error()) { js_AbortRecording(cx, "error during recording"); return JSRS_STOP; } - if (tr->traceMonitor->allocator->outOfMemory() || - js_OverfullFragmento(&JS_TRACE_MONITOR(cx), - JS_TRACE_MONITOR(cx).fragmento)) { - js_AbortRecording(cx, "no more memory"); + if (tr->lirbuf->outOMem() || + js_OverfullFragmento(&JS_TRACE_MONITOR(cx), JS_TRACE_MONITOR(cx).fragmento)) { + js_AbortRecording(cx, "no more LIR memory"); FlushJITCache(cx); return JSRS_STOP; } @@ -6468,23 +6424,14 @@ js_InitJIT(JSTraceMonitor *tm) JS_DHASH_DEFAULT_CAPACITY(PC_HASH_COUNT)); } - if (!tm->allocator) - tm->allocator = new VMAllocator(); - - if (!tm->codeAlloc) - tm->codeAlloc = new (&gc) CodeAlloc(); - - if (!tm->assembler) - tm->assembler = new (&gc) Assembler(tm->codeAlloc, core, &js_LogController); - if (!tm->fragmento) { JS_ASSERT(!tm->reservedDoublePool); - Fragmento* fragmento = new (&gc) Fragmento(core, &js_LogController, 32, tm->codeAlloc); - verbose_only(fragmento->labels = new (&gc) LabelMap(core, *tm->allocator);) + Fragmento* fragmento = new (&gc) Fragmento(core, &js_LogController, 32); + verbose_only(fragmento->labels = new (&gc) LabelMap(core);) tm->fragmento = fragmento; - tm->lirbuf = new (&gc) LirBuffer(*tm->allocator); + tm->lirbuf = new (&gc) LirBuffer(fragmento); #ifdef DEBUG - tm->lirbuf->names = new (&gc) LirNameMap(&gc, *tm->allocator, tm->fragmento->labels); + tm->lirbuf->names = new (&gc) LirNameMap(&gc, tm->fragmento->labels); #endif for (size_t i = 0; i < MONITOR_N_GLOBAL_STATES; ++i) { tm->globalStates[i].globalShape = -1; @@ -6494,20 +6441,13 @@ js_InitJIT(JSTraceMonitor *tm) tm->reservedDoublePoolPtr = tm->reservedDoublePool = new jsval[MAX_NATIVE_STACK_SLOTS]; memset(tm->vmfragments, 0, sizeof(tm->vmfragments)); } - - if (!tm->reAllocator) - tm->reAllocator = new VMAllocator(); - - if (!tm->reCodeAlloc) - tm->reCodeAlloc = new (&gc) CodeAlloc(); - if (!tm->reFragmento) { - Fragmento* fragmento = new (&gc) Fragmento(core, &js_LogController, 32, tm->reCodeAlloc); - verbose_only(fragmento->labels = new (&gc) LabelMap(core, *tm->reAllocator);) + Fragmento* fragmento = new (&gc) Fragmento(core, &js_LogController, 32); + verbose_only(fragmento->labels = new (&gc) LabelMap(core);) tm->reFragmento = fragmento; - tm->reLirBuf = new (&gc) LirBuffer(*tm->reAllocator); + tm->reLirBuf = new (&gc) LirBuffer(fragmento); #ifdef DEBUG - tm->reLirBuf->names = new (&gc) LirNameMap(&gc, *tm->reAllocator, fragmento->labels); + tm->reLirBuf->names = new (&gc) LirNameMap(&gc, fragmento->labels); #endif } #if !defined XP_WIN @@ -6569,15 +6509,7 @@ js_FinishJIT(JSTraceMonitor *tm) delete tm->reLirBuf; verbose_only(delete tm->reFragmento->labels;) delete tm->reFragmento; - delete tm->reAllocator; - delete tm->reCodeAlloc; } - if (tm->assembler) - delete tm->assembler; - if (tm->codeAlloc) - delete tm->codeAlloc; - if (tm->allocator) - delete tm->allocator; } void @@ -6687,7 +6619,7 @@ bool js_OverfullFragmento(JSTraceMonitor* tm, Fragmento *fragmento) { /* - * You might imagine the outOfMemory flag on the allocator is sufficient + * You might imagine the outOMem flag on the lirbuf is sufficient * to model the notion of "running out of memory", but there are actually * two separate issues involved: * @@ -6701,28 +6633,26 @@ js_OverfullFragmento(JSTraceMonitor* tm, Fragmento *fragmento) * safely shut down and signal the rest of spidermonkey when it * does. Condition 2 happens quite regularly. * - * Presently, the code in this file doesn't check the outOfMemory condition + * Presently, the code in this file doesn't check the outOMem condition * often enough, and frequently misuses the unchecked results of * lirbuffer insertions on the asssumption that it will notice the - * outOfMemory flag "soon enough" when it returns to the monitorRecording - * function. This turns out to be a false assumption if we use outOfMemory + * outOMem flag "soon enough" when it returns to the monitorRecording + * function. This turns out to be a false assumption if we use outOMem * to signal condition 2: we regularly provoke "passing our intended * size" and regularly fail to notice it in time to prevent writing * over the end of an artificially self-limited LIR buffer. * * To mitigate, though not completely solve, this problem, we're * modeling the two forms of memory exhaustion *separately* for the - * time being: condition 1 is handled by the outOfMemory flag inside + * time being: condition 1 is handled by the outOMem flag inside * nanojit, and condition 2 is being handled independently *here*. So * we construct our fragmentos to use all available memory they like, - * and only report outOfMemory to us when there is literally no OS memory + * and only report outOMem to us when there is literally no OS memory * left. Merely purging our cache when we hit our highwater mark is * handled by the (few) callers of this function. * */ jsuint maxsz = tm->maxCodeCacheBytes; - VMAllocator *allocator = tm->allocator; - CodeAlloc *codeAlloc = tm->codeAlloc; if (fragmento == tm->fragmento) { if (tm->prohibitFlush) return false; @@ -6734,10 +6664,8 @@ js_OverfullFragmento(JSTraceMonitor* tm, Fragmento *fragmento) * code caches. */ maxsz /= 16; - allocator = tm->reAllocator; - codeAlloc = tm->reCodeAlloc; } - return (codeAlloc->size() + allocator->size() > maxsz); + return (fragmento->cacheUsed() > maxsz); } JS_FORCES_STACK JS_FRIEND_API(void) @@ -9011,8 +8939,7 @@ TraceRecorder::newArray(JSObject* ctor, uint32 argc, jsval* argv, jsval* rval) // arr->dslots[i] = box_jsval(vp[i]); for i in 0..argc LIns *dslots_ins = NULL; - VMAllocator *alloc = traceMonitor->allocator; - for (uint32 i = 0; i < argc && !alloc->outOfMemory(); i++) { + for (uint32 i = 0; i < argc && !lirbuf->outOMem(); i++) { LIns *elt_ins = get(argv + i); box_jsval(argv[i], elt_ins); stobj_set_dslot(arr_ins, i, dslots_ins, elt_ins, "set_array_elt"); @@ -9365,7 +9292,6 @@ TraceRecorder::callNative(uintN argc, JSOp mode) } lir->insStorei(this_ins, invokevp_ins, 1 * sizeof(jsval)); - VMAllocator *alloc = traceMonitor->allocator; // Populate argv. for (uintN n = 2; n < 2 + argc; n++) { LIns* i = get(&vp[n]); @@ -9374,7 +9300,7 @@ TraceRecorder::callNative(uintN argc, JSOp mode) // For a very long argument list we might run out of LIR space, so // check inside the loop. - if (alloc->outOfMemory()) + if (lirbuf->outOMem()) ABORT_TRACE("out of memory in argument list"); } @@ -9384,7 +9310,7 @@ TraceRecorder::callNative(uintN argc, JSOp mode) for (uintN n = 2 + argc; n < vplen; n++) { lir->insStorei(undef_ins, invokevp_ins, n * sizeof(jsval)); - if (alloc->outOfMemory()) + if (lirbuf->outOMem()) ABORT_TRACE("out of memory in extra slots"); } } @@ -9999,7 +9925,7 @@ TraceRecorder::record_JSOP_GETELEM() // The entry type map is not necessarily up-to-date, so we capture a new type map // for this point in the code. unsigned stackSlots = NativeStackSlots(cx, 0 /* callDepth */); - if (stackSlots * sizeof(JSTraceType) > LirBuffer::MAX_SKIP_PAYLOAD_SZB) + if (stackSlots * sizeof(JSTraceType) > NJ_MAX_SKIP_PAYLOAD_SZB) ABORT_TRACE("|arguments| requires saving too much stack"); JSTraceType* typemap = (JSTraceType*) lir->insSkip(stackSlots * sizeof(JSTraceType))->payload(); DetermineTypesVisitor detVisitor(*this, typemap); @@ -10423,7 +10349,7 @@ TraceRecorder::interpretedFunctionCall(jsval& fval, JSFunction* fun, uintN argc, // Generate a type map for the outgoing frame and stash it in the LIR unsigned stackSlots = NativeStackSlots(cx, 0 /* callDepth */); - if (sizeof(FrameInfo) + stackSlots * sizeof(JSTraceType) > LirBuffer::MAX_SKIP_PAYLOAD_SZB) + if (sizeof(FrameInfo) + stackSlots * sizeof(JSTraceType) > NJ_MAX_SKIP_PAYLOAD_SZB) ABORT_TRACE("interpreted function call requires saving too much stack"); LIns* data = lir->insSkip(sizeof(FrameInfo) + stackSlots * sizeof(JSTraceType)); FrameInfo* fi = (FrameInfo*)data->payload(); diff --git a/js/src/jstracer.h b/js/src/jstracer.h index 019c4f12e2d..fe193d0b230 100644 --- a/js/src/jstracer.h +++ b/js/src/jstracer.h @@ -383,33 +383,6 @@ struct VMSideExit : public nanojit::SideExit } }; -struct VMAllocator : public nanojit::Allocator -{ - -public: - VMAllocator() : mOutOfMemory(false), mSize(0) - {} - - size_t size() { - return mSize; - } - - bool outOfMemory() { - return mOutOfMemory; - } - - bool mOutOfMemory; - size_t mSize; - /* - * FIXME: Area the LIR spills into if we encounter an OOM mid-way - * through compilation; we must check mOutOfMemory before we run out - * of mReserve, otherwise we're in undefined territory. This area - * used to be one page, now 16 to be "safer". This is a temporary - * and quite unsatisfactory approach to handling OOM in Nanojit. - */ - uintptr_t mReserve[0x10000]; -}; - struct FrameInfo { JSObject* callee; // callee function object JSObject* block; // caller block chain head diff --git a/js/src/nanojit/Allocator.cpp b/js/src/nanojit/Allocator.cpp deleted file mode 100644 index 372226d8b8b..00000000000 --- a/js/src/nanojit/Allocator.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */ -/* vi: set ts=4 sw=4 expandtab: (add to ~/.vimrc: set modeline modelines=5) */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is [Open Source Virtual Machine]. - * - * The Initial Developer of the Original Code is - * Adobe System Incorporated. - * Portions created by the Initial Developer are Copyright (C) 2004-2007 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Adobe AS3 Team - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#include "nanojit.h" - -namespace nanojit -{ - Allocator::Allocator() - : current_chunk(NULL) - , current_top(NULL) - , current_limit(NULL) - { } - - Allocator::~Allocator() - { - reset(); - } - - void Allocator::reset() - { - Chunk *c = current_chunk; - while (c) { - Chunk *prev = c->prev; - this->freeChunk(c); - c = prev; - } - current_chunk = NULL; - current_top = NULL; - current_limit = NULL; - postReset(); - } - - void* Allocator::allocSlow(size_t nbytes) - { - NanoAssert((nbytes & 7) == 0); - fill(nbytes); - NanoAssert(current_top + nbytes <= current_limit); - void* p = current_top; - current_top += nbytes; - return p; - } - - void Allocator::fill(size_t nbytes) - { - const size_t minChunk = 2000; - if (nbytes < minChunk) - nbytes = minChunk; - size_t chunkbytes = sizeof(Chunk) + nbytes - sizeof(int64_t); - void* mem = allocChunk(chunkbytes); - Chunk* chunk = (Chunk*) mem; - chunk->prev = current_chunk; - current_chunk = chunk; - current_top = (char*)chunk->data; - current_limit = (char*)mem + chunkbytes; - } -} diff --git a/js/src/nanojit/Allocator.h b/js/src/nanojit/Allocator.h deleted file mode 100644 index 54237d7f9a5..00000000000 --- a/js/src/nanojit/Allocator.h +++ /dev/null @@ -1,107 +0,0 @@ -/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */ -/* vi: set ts=4 sw=4 expandtab: (add to ~/.vimrc: set modeline modelines=5) */ -/* ***** BEGIN LICENSE BLOCK ***** - * Version: MPL 1.1/GPL 2.0/LGPL 2.1 - * - * The contents of this file are subject to the Mozilla Public License Version - * 1.1 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is [Open Source Virtual Machine]. - * - * The Initial Developer of the Original Code is - * Adobe System Incorporated. - * Portions created by the Initial Developer are Copyright (C) 2009 - * the Initial Developer. All Rights Reserved. - * - * Contributor(s): - * Adobe AS3 Team - * - * Alternatively, the contents of this file may be used under the terms of - * either the GNU General Public License Version 2 or later (the "GPL"), or - * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), - * in which case the provisions of the GPL or the LGPL are applicable instead - * of those above. If you wish to allow use of your version of this file only - * under the terms of either the GPL or the LGPL, and not to allow others to - * use your version of this file under the terms of the MPL, indicate your - * decision by deleting the provisions above and replace them with the notice - * and other provisions required by the GPL or the LGPL. If you do not delete - * the provisions above, a recipient may use your version of this file under - * the terms of any one of the MPL, the GPL or the LGPL. - * - * ***** END LICENSE BLOCK ***** */ - -#ifndef __nanojit_Allocator__ -#define __nanojit_Allocator__ - -namespace nanojit -{ - /** - * Allocator is a bump-pointer allocator with an SPI for getting more - * memory from embedder-implemented allocator, such as malloc()/free(). - * - * allocations never return NULL. The implementation of allocChunk() - * is expected to perform a longjmp or exception when an allocation can't - * proceed. - */ - class Allocator { - public: - Allocator(); - virtual ~Allocator(); - void reset(); - - /** alloc memory, never return null. */ - void* alloc(size_t nbytes) { - nbytes = (nbytes + 7) & ~7; // round up - if (current_top + nbytes <= current_limit) { - void *p = current_top; - current_top += nbytes; - return p; - } - return allocSlow(nbytes); - } - - private: - void* allocSlow(size_t nbytes); - void fill(size_t minbytes); - - class Chunk { - public: - Chunk* prev; - int64_t data[1]; // int64_t forces 8-byte alignment. - }; - - Chunk* current_chunk; - char* current_top; - char* current_limit; - - // allocator SPI - private: - /** allocate another block from a host provided allocator */ - void* allocChunk(size_t nbytes); - - /** free back to the same allocator */ - void freeChunk(void*); - - /** hook for post-reset action. */ - void postReset(); - }; -} - -/** global new overload enabling this pattern: new (allocator) T(...) */ -inline void* operator new(size_t size, nanojit::Allocator &a) { - return a.alloc(size); -} - -/** global new[] overload enabling this pattern: new (allocator) T[] */ -inline void* operator new[](size_t size, nanojit::Allocator& a) { - return a.alloc(size); -} - -#endif // __nanojit_Allocator__ diff --git a/js/src/nanojit/Assembler.cpp b/js/src/nanojit/Assembler.cpp index 516d962b028..d949b24276c 100644 --- a/js/src/nanojit/Assembler.cpp +++ b/js/src/nanojit/Assembler.cpp @@ -45,6 +45,15 @@ #include "portapi_nanojit.h" #endif +#if defined(AVMPLUS_UNIX) && defined(AVMPLUS_ARM) +#include +extern "C" void __clear_cache(void *BEG, void *END); +#endif + +#ifdef AVMPLUS_SPARC +extern "C" void sync_instruction_memory(caddr_t v, u_int len); +#endif + namespace nanojit { int UseSoftfloat = 0; @@ -159,19 +168,20 @@ namespace nanojit * * - merging paths ( build a graph? ), possibly use external rep to drive codegen */ - Assembler::Assembler(CodeAlloc* codeAlloc, AvmCore *core, LogControl* logc) + Assembler::Assembler(Fragmento* frago, LogControl* logc) : hasLoop(0) - , codeList(0) - , core(core) - , _codeAlloc(codeAlloc) - , config(core->config) + , _frago(frago) + , _gc(frago->core()->gc) + , config(frago->core()->config) { + AvmCore *core = frago->core(); nInit(core); verbose_only( _logc = logc; ) verbose_only( _outputCache = 0; ) verbose_only( outlineEOL[0] = '\0'; ) - reset(); + internalReset(); + pageReset(); } void Assembler::arReset() @@ -248,53 +258,132 @@ namespace nanojit return i->isconst() || i->isconstq() || i->isop(LIR_ialloc); } - void Assembler::codeAlloc(NIns *&start, NIns *&end, NIns *&eip) - { - // save the block we just filled - if (start) - CodeAlloc::add(codeList, start, end); - - // CodeAlloc contract: allocations never fail - _codeAlloc->alloc(start, end); - VALGRIND_DISCARD_TRANSLATIONS(start, uintptr_t(end) - uintptr_t(start)); - NanoAssert(uintptr_t(end) - uintptr_t(start) >= (size_t)LARGEST_UNDERRUN_PROT); - eip = end; - } - - void Assembler::reset() + void Assembler::internalReset() { // readies for a brand spanking new code generation pass. - _nIns = 0; - _nExitIns = 0; - _startingIns = 0; - codeStart = codeEnd = 0; - exitStart = exitEnd = 0; - _stats.pages = 0; - codeList = 0; - - nativePageReset(); registerResetAll(); arReset(); } -#ifdef _DEBUG + NIns* Assembler::pageAlloc(bool exitPage) + { + Page*& list = (exitPage) ? _nativeExitPages : _nativePages; + Page* page = _frago->pageAlloc(); + if (page) + { + page->next = list; + list = page; + nMarkExecute(page, PAGE_READ|PAGE_WRITE|PAGE_EXEC); + _stats.pages++; + } + else + { + // return a location that is 'safe' to write to while we are out of mem + setError(OutOMem); + return _startingIns; + } + return &page->code[sizeof(page->code)/sizeof(NIns)]; // just past the end + } + + void Assembler::pageReset() + { + pagesFree(_nativePages); + pagesFree(_nativeExitPages); + + _nIns = 0; + _nExitIns = 0; + _startingIns = 0; + _stats.pages = 0; + + nativePageReset(); + } + + void Assembler::pagesFree(Page*& page) + { + while(page) + { + Page *next = page->next; // pull next ptr prior to free + _frago->pageFree(page); + page = next; + } + } + + #define bytesFromTop(x) ( (size_t)(x) - (size_t)pageTop(x) ) + #define bytesToBottom(x) ( (size_t)pageBottom(x) - (size_t)(x) ) + #define bytesBetween(x,y) ( (size_t)(x) - (size_t)(y) ) + + int32_t Assembler::codeBytes() + { + // start and end on same page? + size_t exit = 0; + int32_t pages = _stats.pages; + if (_nExitIns-1 == _stats.codeExitStart) + ; + else if (samepage(_nExitIns,_stats.codeExitStart)) + exit = bytesBetween(_stats.codeExitStart, _nExitIns); + else + { + pages--; + exit = ((intptr_t)_stats.codeExitStart & (NJ_PAGE_SIZE-1)) ? bytesFromTop(_stats.codeExitStart)+1 : 0; + exit += bytesToBottom(_nExitIns)+1; + } + + size_t main = 0; + if (_nIns-1 == _stats.codeStart) + ; + else if (samepage(_nIns,_stats.codeStart)) + main = bytesBetween(_stats.codeStart, _nIns); + else + { + pages--; + main = ((intptr_t)_stats.codeStart & (NJ_PAGE_SIZE-1)) ? bytesFromTop(_stats.codeStart)+1 : 0; + main += bytesToBottom(_nIns)+1; + } + //nj_dprintf("size %d, exit is %d, main is %d, page count %d, sizeof %d\n", (int)((pages) * NJ_PAGE_SIZE + main + exit),(int)exit, (int)main, (int)_stats.pages, (int)sizeof(Page)); + return (pages) * NJ_PAGE_SIZE + main + exit; + } + + #undef bytesFromTop + #undef bytesToBottom + #undef byteBetween + + Page* Assembler::handoverPages(bool exitPages) + { + Page*& list = (exitPages) ? _nativeExitPages : _nativePages; + NIns*& ins = (exitPages) ? _nExitIns : _nIns; + Page* start = list; + list = 0; + ins = 0; + return start; + } + + #ifdef _DEBUG + bool Assembler::onPage(NIns* where, bool exitPages) + { + Page* page = (exitPages) ? _nativeExitPages : _nativePages; + bool on = false; + while(page) + { + if (samepage(where-1,page)) + on = true; + page = page->next; + } + return on; + } + void Assembler::pageValidate() { - if (error()) return; - // _nIns needs to be at least on one of these pages - NanoAssertMsg(_inExit ? containsPtr(exitStart, exitEnd, _nIns) : containsPtr(codeStart, codeEnd, _nIns), - "Native instruction pointer overstep paging bounds; check overrideProtect for last instruction"); + NanoAssert(!error()); + // _nIns and _nExitIns need to be at least on one of these pages + NanoAssertMsg( onPage(_nIns)&& onPage(_nExitIns,true), "Native instruction pointer overstep paging bounds; check overrideProtect for last instruction"); } -#endif - -#endif - + #endif #ifdef _DEBUG void Assembler::resourceConsistencyCheck() { - if (error()) return; + NanoAssert(!error()); #ifdef NANOJIT_IA32 NanoAssert((_allocator.active[FST0] && _fpuStkDepth == -1) || @@ -589,6 +678,11 @@ namespace nanojit { RegAlloc* captured = _branchStateMap->get(exit); intersectRegisterState(*captured); + verbose_only( + verbose_outputf("## merging trunk with %s", + _frago->labels->format(exit->target)); + verbose_outputf("%010lx:", (unsigned long)_nIns); + ) at = exit->target->fragEntry; NanoAssert(at != 0); _branchStateMap->remove(exit); @@ -611,7 +705,7 @@ namespace nanojit swapptrs(); _inExit = true; - // verbose_only( verbose_outputf(" LIR_xend swapptrs, _nIns is now %08X(%08X), _nExitIns is now %08X(%08X)",_nIns, *_nIns,_nExitIns,*_nExitIns) ); + //verbose_only( verbose_outputf(" LIR_xend swapptrs, _nIns is now %08X(%08X), _nExitIns is now %08X(%08X)",_nIns, *_nIns,_nExitIns,*_nExitIns) ); debug_only( _sv_fpuStkDepth = _fpuStkDepth; _fpuStkDepth = 0; ) nFragExit(guard); @@ -633,7 +727,7 @@ namespace nanojit swapptrs(); _inExit = false; - // verbose_only( verbose_outputf(" LIR_xt/xf swapptrs, _nIns is now %08X(%08X), _nExitIns is now %08X(%08X)",_nIns, *_nIns,_nExitIns,*_nExitIns) ); + //verbose_only( verbose_outputf(" LIR_xt/xf swapptrs, _nIns is now %08X(%08X), _nExitIns is now %08X(%08X)",_nIns, *_nIns,_nExitIns,*_nExitIns) ); verbose_only( verbose_outputf("%010lx:", (unsigned long)jmpTarget);) verbose_only( verbose_outputf("----------------------------------- ## BEGIN exit block (LIR_xt|LIR_xf)") ); @@ -649,15 +743,7 @@ namespace nanojit void Assembler::beginAssembly(Fragment *frag, RegAllocMap* branchStateMap) { - reset(); - - NanoAssert(codeList == 0); - NanoAssert(codeStart == 0); - NanoAssert(codeEnd == 0); - NanoAssert(exitStart == 0); - NanoAssert(exitEnd == 0); - NanoAssert(_nIns == 0); - NanoAssert(_nExitIns == 0); + internalReset(); _thisfrag = frag; _activation.lowwatermark = 1; @@ -703,6 +789,7 @@ namespace nanojit void Assembler::assemble(Fragment* frag, NInsList& loopJumps) { if (error()) return; + AvmCore *core = _frago->core(); _thisfrag = frag; // Used for debug printing, if needed @@ -757,13 +844,15 @@ namespace nanojit ) verbose_only(_thisfrag->compileNbr++; ) + verbose_only(_frago->_stats.compiles++; ) + verbose_only(_frago->_stats.totalCompiles++; ) _inExit = false; LabelStateMap labels(_gc); NInsMap patches(_gc); gen(prev, loopJumps, labels, patches); frag->loopEntry = _nIns; - //frag->outbound = config.tree_opt? _latestGuard : 0; + //frag->outbound = core->config.tree_opt? _latestGuard : 0; //nj_dprintf("assemble frag %X entry %X\n", (int)frag, (int)frag->fragEntry); if (!error()) { @@ -840,39 +929,74 @@ namespace nanojit } ) - // save used parts of current block on fragment's code list, free the rest -#ifdef NANOJIT_ARM - _codeAlloc->addRemainder(codeList, exitStart, exitEnd, _nExitSlot, _nExitIns); - _codeAlloc->addRemainder(codeList, codeStart, codeEnd, _nSlot, _nIns); -#else - _codeAlloc->addRemainder(codeList, exitStart, exitEnd, exitStart, _nExitIns); - _codeAlloc->addRemainder(codeList, codeStart, codeEnd, codeStart, _nIns); -#endif - - debug_only( pageValidate(); ) - - // at this point all our new code is in the d-cache and not the i-cache, - // so flush the i-cache on cpu's that need it. - _codeAlloc->flushICache(codeList); - - // save entry point pointers frag->fragEntry = fragEntry; - frag->setCode(_nIns); + NIns* code = _nIns; +#ifdef PERFM + _nvprof("code", codeBytes()); // requires that all pages are released between begin/endAssembly()otherwise we double count +#endif + // let the fragment manage the pages if we're using trees and there are branches + Page* manage = (_frago->core()->config.tree_opt) ? handoverPages() : 0; + frag->setCode(code, manage); // root of tree should manage all pages + //nj_dprintf("endAssembly frag %X entry %X\n", (int)frag, (int)frag->fragEntry); } else { - // something went wrong, release all allocated memory - _codeAlloc->freeAll(codeList); - _codeAlloc->free(exitStart, exitEnd); - _codeAlloc->free(codeStart, codeEnd); + // In case of failure, reset _nIns ready for the next assembly run. + resetInstructionPointer(); } NanoAssertMsgf(error() || _fpuStkDepth == 0,"_fpuStkDepth %d",_fpuStkDepth); - debug_only( pageValidate(); ) - reset(); + internalReset(); // clear the reservation tables and regalloc NanoAssert( !_branchStateMap || _branchStateMap->isEmpty()); _branchStateMap = 0; + + // Tell Valgrind that new code has been generated, and it must flush + // any translations it has for the memory range generated into. + VALGRIND_DISCARD_TRANSLATIONS(pageTop(_nIns-1), NJ_PAGE_SIZE); + VALGRIND_DISCARD_TRANSLATIONS(pageTop(_nExitIns-1), NJ_PAGE_SIZE); + +#ifdef AVMPLUS_ARM + // If we've modified the code, we need to flush so we don't end up trying + // to execute junk +# if defined(UNDER_CE) + FlushInstructionCache(GetCurrentProcess(), NULL, NULL); +# elif defined(AVMPLUS_UNIX) + for (int i = 0; i < 2; i++) { + Page *p = (i == 0) ? _nativePages : _nativeExitPages; + + Page *first = p; + while (p) { + if (!p->next || p->next != p+1) { + __clear_cache((char*)first, (char*)(p+1)); + first = p->next; + } + p = p->next; + } + } +# endif +#endif + +#ifdef AVMPLUS_SPARC + // Clear Instruction Cache + for (int i = 0; i < 2; i++) { + Page *p = (i == 0) ? _nativePages : _nativeExitPages; + + Page *first = p; + while (p) { + if (!p->next || p->next != p+1) { + sync_instruction_memory((char *)first, NJ_PAGE_SIZE); + first = p->next; + } + p = p->next; + } + } +#endif + +# ifdef AVMPLUS_PORTING_API + NanoJIT_PortAPI_FlushInstructionCache(_nIns, _startingIns); + NanoJIT_PortAPI_FlushInstructionCache(_nExitIns, _endJit2Addr); +# endif } void Assembler::copyRegisters(RegAlloc* copyTo) @@ -1886,6 +2010,8 @@ namespace nanojit } #endif // verbose + #endif /* FEATURE_NANOJIT */ + #if defined(FEATURE_NANOJIT) || defined(NJ_VERBOSE) uint32_t CallInfo::_count_args(uint32_t mask) const { diff --git a/js/src/nanojit/Assembler.h b/js/src/nanojit/Assembler.h index 62b984ca811..4fb3dfe8e69 100644 --- a/js/src/nanojit/Assembler.h +++ b/js/src/nanojit/Assembler.h @@ -97,6 +97,8 @@ namespace nanojit #endif }; + class Fragmento; + // error codes enum AssmError { @@ -169,7 +171,7 @@ namespace nanojit LogControl* _logc; #endif - Assembler(CodeAlloc* codeAlloc, AvmCore* core, LogControl* logc); + Assembler(Fragmento* frago, LogControl* logc); ~Assembler() {} void assemble(Fragment* frag, NInsList& loopJumps); @@ -184,9 +186,12 @@ namespace nanojit #endif AssmError error() { return _err; } void setError(AssmError e) { _err = e; } - void reset(); + void pageReset(); + int32_t codeBytes(); + Page* handoverPages(bool exitPages=false); debug_only ( void pageValidate(); ) + debug_only ( bool onPage(NIns* where, bool exitPages=false); ) // support calling out from a fragment ; used to debug the jit debug_only( void resourceConsistencyCheck(); ) @@ -194,7 +199,6 @@ namespace nanojit Stats _stats; int hasLoop; - CodeList* codeList; // finished blocks of code. private: @@ -229,7 +233,9 @@ namespace nanojit void resetInstructionPointer(); void recordStartingInstructionPointer(); - void codeAlloc(NIns *&start, NIns *&end, NIns *&eip); + NIns* pageAlloc(bool exitPage=false); + void pagesFree(Page*& list); + void internalReset(); bool canRemat(LIns*); Reservation* getresv(LIns *x) { @@ -237,19 +243,17 @@ namespace nanojit return r->used ? r : 0; } - AvmCore *core; - DWB(CodeAlloc*) _codeAlloc; + DWB(Fragmento*) _frago; avmplus::GC* _gc; DWB(Fragment*) _thisfrag; RegAllocMap* _branchStateMap; - NIns *codeStart, *codeEnd; // current block we're adding code to - NIns *exitStart, *exitEnd; // current block for exit stubs - NIns* _nIns; // current native instruction NIns* _nExitIns; // current instruction in exit fragment page NIns* _startingIns; // starting location of code compilation for error handling NIns* _epilogue; + Page* _nativePages; // list of NJ_PAGE_SIZE pages that have been alloc'd + Page* _nativeExitPages; // list of pages that have been allocated for exit code AssmError _err; // 0 = means assemble() appears ok, otherwise it failed AR _activation; @@ -303,10 +307,19 @@ namespace nanojit void assignParamRegs(); void handleLoopCarriedExprs(InsList& pending_lives); + // flag values for nMarkExecute + enum + { + PAGE_READ = 0x0, // here only for clarity: all permissions include READ + PAGE_WRITE = 0x01, + PAGE_EXEC = 0x02 + }; + // platform specific implementation (see NativeXXX.cpp file) void nInit(AvmCore *); Register nRegisterAllocFromSet(int32_t set); void nRegisterResetAll(RegAlloc& a); + void nMarkExecute(Page* page, int flags); NIns* nPatchBranch(NIns* branch, NIns* location); void nFragExit(LIns* guard); diff --git a/js/src/nanojit/CodeAlloc.cpp b/js/src/nanojit/CodeAlloc.cpp index 4927c52f1a5..3541197e516 100644 --- a/js/src/nanojit/CodeAlloc.cpp +++ b/js/src/nanojit/CodeAlloc.cpp @@ -47,20 +47,11 @@ namespace nanojit { static const bool verbose = false; -#if defined(NANOJIT_ARM) - // ARM requires single-page allocations, due to the constant pool that - // lives on each page that must be reachable by a 4kb pcrel load. static const int pagesPerAlloc = 1; -#else - static const int pagesPerAlloc = 256; -#endif - static const int bytesPerPage = 4096; - static const int bytesPerAlloc = pagesPerAlloc * bytesPerPage; + static const int bytesPerAlloc = pagesPerAlloc * GCHeap::kBlockSize; - CodeAlloc::CodeAlloc() - : heapblocks(0) - , availblocks(0) - , totalAllocated(0) + CodeAlloc::CodeAlloc(GCHeap* heap) + : heap(heap), heapblocks(0) {} CodeAlloc::~CodeAlloc() { @@ -71,15 +62,14 @@ namespace nanojit CodeList* next = b->next; void *mem = firstBlock(b); VMPI_setPageProtection(mem, bytesPerAlloc, false /* executable */, true /* writable */); - freeCodeChunk(mem, bytesPerAlloc); - totalAllocated -= bytesPerAlloc; + heap->Free(mem); b = next; } } CodeList* CodeAlloc::firstBlock(CodeList* term) { - char* end = (char*)alignUp(term, bytesPerPage); - return (CodeList*) (end - bytesPerAlloc); + // fragile but correct as long as we allocate one block at a time. + return (CodeList*) alignTo(term, bytesPerAlloc); } int round(size_t x) { @@ -106,20 +96,23 @@ namespace nanojit } void CodeAlloc::alloc(NIns* &start, NIns* &end) { - // Reuse a block if possible. - if (availblocks) { - CodeList* b = removeBlock(availblocks); - b->isFree = false; - start = b->start(); - end = b->end; - if (verbose) - avmplus::AvmLog("alloc %p-%p %d\n", start, end, int(end-start)); - return; + // search each heap block for a free block + for (CodeList* hb = heapblocks; hb != 0; hb = hb->next) { + // check each block to see if it's free and big enough + for (CodeList* b = hb->lower; b != 0; b = b->lower) { + if (b->isFree && b->size() >= minAllocSize) { + // good block + b->isFree = false; + start = b->start(); + end = b->end; + if (verbose) + avmplus::AvmLog("alloc %p-%p %d\n", start, end, int(end-start)); + return; + } + } } // no suitable block found, get more memory - void *mem = allocCodeChunk(bytesPerAlloc); // allocations never fail - totalAllocated += bytesPerAlloc; - NanoAssert(mem != NULL); // see allocCodeChunk contract in CodeAlloc.h + void *mem = heap->Alloc(pagesPerAlloc); // allocations never fail _nvprof("alloc page", uintptr_t(mem)>>12); VMPI_setPageProtection(mem, bytesPerAlloc, true/*executable*/, true/*writable*/); CodeList* b = addMem(mem, bytesPerAlloc); @@ -137,53 +130,28 @@ namespace nanojit AvmAssert(!blk->isFree); - // coalesce adjacent blocks. - bool already_on_avail_list; - + // coalesce if (blk->lower && blk->lower->isFree) { // combine blk into blk->lower (destroy blk) CodeList* lower = blk->lower; CodeList* higher = blk->higher; - already_on_avail_list = lower->size() >= minAllocSize; lower->higher = higher; higher->lower = lower; debug_only( sanity_check();) blk = lower; } - else - already_on_avail_list = false; - - // the last block in each heapblock is a terminator block, - // which is never free, therefore blk->higher != null + // the last block in each heapblock is never free, therefore blk->higher != null if (blk->higher->isFree) { // combine blk->higher into blk (destroy blk->higher) CodeList *higher = blk->higher->higher; blk->higher = higher; higher->lower = blk; debug_only(sanity_check();) - - if ( higher->size() >= minAllocSize ) { - // Unlink higher from the available block chain. - if ( availblocks == higher ) { - removeBlock(availblocks); - } - else { - CodeList* free_block = availblocks; - while ( free_block && free_block->next != higher) { - free_block = free_block->next; - } - NanoAssert(free_block && free_block->next == higher); - if (free_block && free_block->next == higher) - free_block->next = higher->next; - } - } } blk->isFree = true; NanoAssert(!blk->lower || !blk->lower->isFree); NanoAssert(blk->higher && !blk->higher->isFree); //memset(blk->start(), 0xCC, blk->size()); // INT 3 instruction - if ( !already_on_avail_list && blk->size() >= minAllocSize ) - addBlock(availblocks, blk); } void CodeAlloc::sweep() { @@ -197,8 +165,7 @@ namespace nanojit *prev = hb->next; _nvprof("free page",1); VMPI_setPageProtection(mem, bytesPerAlloc, false /* executable */, true /* writable */); - freeCodeChunk(mem, bytesPerAlloc); - totalAllocated -= bytesPerAlloc; + heap->Free(mem); } else { prev = &hb->next; } @@ -218,17 +185,13 @@ extern "C" void __clear_cache(char *BEG, char *END); #endif #ifdef AVMPLUS_SPARC -extern "C" void sync_instruction_memory(caddr_t v, u_int len); +extern "C" void sync_instruction_memory(caddr_t v, u_int len); #endif #if defined NANOJIT_IA32 || defined NANOJIT_X64 // intel chips have dcache/icache interlock - void CodeAlloc::flushICache(CodeList* &blocks) { - // Tell Valgrind that new code has been generated, and it must flush - // any translations it has for the memory range generated into. - for (CodeList *b = blocks; b != 0; b = b->next) - VALGRIND_DISCARD_TRANSLATIONS(b->start(), b->size()); - } + void CodeAlloc::flushICache(CodeList* &) + {} #elif defined NANOJIT_ARM && defined UNDER_CE // on arm/winmo, just flush the whole icache @@ -312,7 +275,6 @@ extern "C" void sync_instruction_memory(caddr_t v, u_int len); CodeList* CodeAlloc::removeBlock(CodeList* &blocks) { CodeList* b = blocks; - NanoAssert(b); blocks = b->next; b->next = 0; return b; @@ -388,10 +350,6 @@ extern "C" void sync_instruction_memory(caddr_t v, u_int len); return size; } - size_t CodeAlloc::size() { - return totalAllocated; - } - bool CodeAlloc::contains(const CodeList* blocks, NIns* p) { for (const CodeList *b = blocks; b != 0; b = b->next) { _nvprof("block contains",1); @@ -436,33 +394,6 @@ extern "C" void sync_instruction_memory(caddr_t v, u_int len); NanoAssert(b->higher->lower == b); } } - for (CodeList* avail = this->availblocks; avail; avail = avail->next) { - NanoAssert(avail->isFree && avail->size() >= minAllocSize); - } - - #if CROSS_CHECK_FREE_LIST - for(CodeList* term = heapblocks; term; term = term->next) { - for(CodeList* hb = term->lower; hb; hb = hb->lower) { - if (hb->isFree && hb->size() >= minAllocSize) { - bool found_on_avail = false; - for (CodeList* avail = this->availblocks; !found_on_avail && avail; avail = avail->next) { - found_on_avail = avail == hb; - } - - NanoAssert(found_on_avail); - } - } - } - for (CodeList* avail = this->availblocks; avail; avail = avail->next) { - bool found_in_heapblocks = false; - for(CodeList* term = heapblocks; !found_in_heapblocks && term; term = term->next) { - for(CodeList* hb = term->lower; hb; hb = hb->lower) { - found_in_heapblocks = hb == avail; - } - } - NanoAssert(found_in_heapblocks); - } - #endif /* CROSS_CHECK_FREE_LIST */ } #endif } diff --git a/js/src/nanojit/CodeAlloc.h b/js/src/nanojit/CodeAlloc.h index 67b26a92bf0..c6c3e4dcc64 100644 --- a/js/src/nanojit/CodeAlloc.h +++ b/js/src/nanojit/CodeAlloc.h @@ -42,9 +42,6 @@ namespace nanojit { - // Temporary tracemonkey hack until namespaces are sorted out. - using namespace MMgc; - /** return true if ptr is in the range [start, end) */ inline bool containsPtr(const NIns* start, const NIns* end, const NIns* ptr) { return ptr >= start && ptr < end; @@ -107,15 +104,9 @@ namespace nanojit static const size_t sizeofMinBlock = offsetof(CodeList, code); static const size_t minAllocSize = LARGEST_UNDERRUN_PROT; - /** Terminator blocks. All active and free allocations - are reachable by traversing this chain and each - element's lower chain. */ + GCHeap* heap; CodeList* heapblocks; - /** Reusable blocks. */ - CodeList* availblocks; - size_t totalAllocated; - /** remove one block from a list */ static CodeList* removeBlock(CodeList* &list); @@ -134,22 +125,8 @@ namespace nanojit /** find the beginning of the heapblock terminated by term */ static CodeList* firstBlock(CodeList* term); - // - // CodeAlloc's SPI. Implementations must be defined by nanojit embedder. - // allocation failures should cause an exception or longjmp; nanojit - // intentionally does not check for null. - // - - /** allocate nbytes of memory to hold code. Never return null! */ - void* allocCodeChunk(size_t nbytes); - - /** free a block previously allocated by allocCodeMem. nbytes will - * match the previous allocCodeMem, but is provided here as well - * to mirror the mmap()/munmap() api. */ - void freeCodeChunk(void* addr, size_t nbytes); - public: - CodeAlloc(); + CodeAlloc(GCHeap*); ~CodeAlloc(); /** allocate some memory for code, return pointers to the region. */ @@ -180,9 +157,6 @@ namespace nanojit /** return the number of bytes in all the code blocks in "code", including block overhead */ static size_t size(const CodeList* code); - /** return the total number of bytes held by this CodeAlloc. */ - size_t size(); - /** print out stats about heap usage */ void logStats(); diff --git a/js/src/nanojit/Fragmento.cpp b/js/src/nanojit/Fragmento.cpp index 3c3a4674bd2..41b2b85b699 100644 --- a/js/src/nanojit/Fragmento.cpp +++ b/js/src/nanojit/Fragmento.cpp @@ -58,16 +58,17 @@ namespace nanojit /** * This is the main control center for creating and managing fragments. */ - Fragmento::Fragmento(AvmCore* core, LogControl* logc, uint32_t cacheSizeLog2, CodeAlloc* codeAlloc) + Fragmento::Fragmento(AvmCore* core, LogControl* logc, uint32_t cacheSizeLog2) : #ifdef NJ_VERBOSE enterCounts(NULL), mergeCounts(NULL), labels(NULL), #endif - _core(core), - _codeAlloc(codeAlloc), _frags(core->GetGC()), + _freePages(core->GetGC(), 1024), + _allocList(core->GetGC()), + _gcHeap(NULL), _max_pages(1 << (calcSaneCacheSize(cacheSizeLog2) - NJ_LOG2_PAGE_SIZE)), _pagesGrowth(1) { @@ -98,21 +99,119 @@ namespace nanojit _allocList.set_meminfo_name("Fragmento._allocList"); #endif NanoAssert(_max_pages > _pagesGrowth); // shrink growth if needed - verbose_only( enterCounts = NJ_NEW(core->gc, BlockHist)(core->gc); ) - verbose_only( mergeCounts = NJ_NEW(core->gc, BlockHist)(core->gc); ) + _core = core; + GC *gc = core->GetGC(); + _assm = NJ_NEW(gc, nanojit::Assembler)(this, logc); + verbose_only( enterCounts = NJ_NEW(gc, BlockHist)(gc); ) + verbose_only( mergeCounts = NJ_NEW(gc, BlockHist)(gc); ) memset(&_stats, 0, sizeof(_stats)); } Fragmento::~Fragmento() { + AllocEntry *entry; + clearFrags(); + _frags.clear(); + _freePages.clear(); + while( _allocList.size() > 0 ) + { + //nj_dprintf("dealloc %x\n", (intptr_t)_allocList.get(_allocList.size()-1)); +#ifdef MEMORY_INFO + ChangeSizeExplicit("NanoJitMem", -1, _gcHeap->Size(_allocList.last())); +#endif + entry = _allocList.removeLast(); + _gcHeap->Free( entry->page, entry->allocSize ); + NJ_DELETE(entry); + } + NJ_DELETE(_assm); #if defined(NJ_VERBOSE) NJ_DELETE(enterCounts); NJ_DELETE(mergeCounts); #endif } + void Fragmento::trackPages() + { + const uint32_t pageUse = _stats.pages - _freePages.size(); + if (_stats.maxPageUse < pageUse) + _stats.maxPageUse = pageUse; + } + + Page* Fragmento::pageAlloc() + { + NanoAssert(sizeof(Page) == NJ_PAGE_SIZE); + if (!_freePages.size()) { + pagesGrow(_pagesGrowth); // try to get more mem + if ((_pagesGrowth << 1) < _max_pages) + _pagesGrowth <<= 1; + } + + trackPages(); + Page* page = 0; + if (_freePages.size()) + page = _freePages.removeLast(); + return page; + } + + void Fragmento::pagesRelease(PageList& l) + { + _freePages.add(l); + l.clear(); + NanoAssert(_freePages.size() <= _stats.pages); + } + + void Fragmento::pageFree(Page* page) + { + _freePages.add(page); + NanoAssert(_freePages.size() <= _stats.pages); + } + + void Fragmento::pagesGrow(int32_t count) + { + NanoAssert(!_freePages.size()); + MMGC_MEM_TYPE("NanojitFragmentoMem"); + Page* memory = 0; + GC *gc = _core->GetGC(); + if (_stats.pages < _max_pages) + { + AllocEntry *entry; + + // make sure we don't grow beyond _max_pages + if (_stats.pages + count > _max_pages) + count = _max_pages - _stats.pages; + if (count < 0) + count = 0; + // @todo nastiness that needs a fix'n + _gcHeap = gc->GetGCHeap(); + NanoAssert(int32_t(NJ_PAGE_SIZE)<=_gcHeap->kNativePageSize); + + // convert _max_pages to gc page count + int32_t gcpages = (count*NJ_PAGE_SIZE) / _gcHeap->kNativePageSize; + MMGC_MEM_TYPE("NanoJitMem"); + memory = (Page*)_gcHeap->Alloc(gcpages); +#ifdef MEMORY_INFO + ChangeSizeExplicit("NanoJitMem", 1, _gcHeap->Size(memory)); +#endif + NanoAssert((uintptr_t)memory == pageTop(memory)); + //nj_dprintf("head alloc of %d at %x of %d pages using nj page size of %d\n", gcpages, (intptr_t)memory, (intptr_t)_gcHeap->kNativePageSize, NJ_PAGE_SIZE); + + entry = NJ_NEW(gc, AllocEntry); + entry->page = memory; + entry->allocSize = gcpages; + _allocList.add(entry); + + _stats.pages += count; + Page* page = memory; + while(--count >= 0) + { + //nj_dprintf("Fragmento::pageGrow adding page %x ; %d\n", (unsigned)page, _freePages.size()+1); + _freePages.add(page++); + } + trackPages(); + } + } // Clear the fragment. This *does not* remove the fragment from the // map--the caller must take care of this. @@ -121,16 +220,19 @@ namespace nanojit Fragment *peer = f->peer; while (peer) { Fragment *next = peer->peer; - peer->releaseTreeMem(_codeAlloc); + peer->releaseTreeMem(this); NJ_DELETE(peer); peer = next; } - f->releaseTreeMem(_codeAlloc); + f->releaseTreeMem(this); NJ_DELETE(f); } void Fragmento::clearFrags() { + // reclaim any dangling native pages + _assm->pageReset(); + while (!_frags.isEmpty()) { clearFragment(_frags.removeLast()); } @@ -142,6 +244,11 @@ namespace nanojit //nj_dprintf("Fragmento.clearFrags %d free pages of %d\n", _stats.freePages, _stats.pages); } + Assembler* Fragmento::assm() + { + return _assm; + } + AvmCore* Fragmento::core() { return _core; @@ -192,6 +299,195 @@ namespace nanojit return f; } +#ifdef NJ_VERBOSE + struct fragstats { + int size; + uint64_t traceDur; + uint64_t interpDur; + int lir, lirbytes; + }; + + void Fragmento::dumpFragStats(Fragment *f, int level, fragstats &stat) + { + char buf[50]; + sprintf(buf, "%*c%s", 1+level, ' ', labels->format(f)); + + int called = f->hits(); + if (called >= 0) + called += f->_called; + else + called = -(1<blacklistLevel) - called - 1; + + uint32_t main = f->_native - f->_exitNative; + + char cause[200]; + if (f->_token && strcmp(f->_token,"loop")==0) + sprintf(cause,"%s %d", f->_token, f->xjumpCount); + else if (f->_token) { + if (f->eot_target) { + sprintf(cause,"%s %s", f->_token, labels->format(f->eot_target)); + } else { + strcpy(cause, f->_token); + } + } + else + cause[0] = 0; + + _assm->outputf("%-10s %7d %6d %6d %6d %4d %9llu %9llu %-12s %s", buf, + called, f->guardCount, main, f->_native, f->compileNbr, f->traceTicks/1000, f->interpTicks/1000, + cause, labels->format(f->ip)); + + stat.size += main; + stat.traceDur += f->traceTicks; + stat.interpDur += f->interpTicks; + stat.lir += f->_lir; + stat.lirbytes += f->_lirbytes; + + for (Fragment *x = f->branches; x != 0; x = x->nextbranch) + if (x->kind != MergeTrace) + dumpFragStats(x,level+1,stat); + for (Fragment *x = f->branches; x != 0; x = x->nextbranch) + if (x->kind == MergeTrace) + dumpFragStats(x,level+1,stat); + + if (f->isAnchor() && f->branches != 0) { + _assm->output(""); + } + } + + class DurData { public: + DurData(): frag(0), traceDur(0), interpDur(0), size(0) {} + DurData(int): frag(0), traceDur(0), interpDur(0), size(0) {} + DurData(Fragment* f, uint64_t td, uint64_t id, int32_t s) + : frag(f), traceDur(td), interpDur(id), size(s) {} + Fragment* frag; + uint64_t traceDur; + uint64_t interpDur; + int32_t size; + }; + + void Fragmento::dumpRatio(const char *label, BlockHist *hist) + { + int total=0, unique=0; + for (int i = 0, n=hist->size(); i < n; i++) { + const void * id = hist->keyAt(i); + int c = hist->get(id); + if (c > 1) { + //_assm->outputf("%d %X", c, id); + unique += 1; + } + else if (c == 1) { + unique += 1; + } + total += c; + } + _assm->outputf("%s total %d unique %d ratio %.1f%", label, total, unique, double(total)/unique); + } + + void Fragmento::dumpStats() + { + _assm->output(""); + dumpRatio("inline", enterCounts); + dumpRatio("merges", mergeCounts); + _assm->outputf("abc %d il %d (%.1fx) abc+il %d (%.1fx)", + _stats.abcsize, _stats.ilsize, (double)_stats.ilsize/_stats.abcsize, + _stats.abcsize + _stats.ilsize, + double(_stats.abcsize+_stats.ilsize)/_stats.abcsize); + + int32_t count = _frags.size(); + int32_t pages = _stats.pages; + int32_t maxPageUse = _stats.maxPageUse; + int32_t free = _freePages.size(); + int32_t flushes = _stats.flushes; + if (!count) + { + _assm->outputf("No fragments in cache, %d flushes", flushes); + return; + } + + _assm->outputf("\nFragment statistics"); + _assm->outputf(" loop trees: %d", count); + _assm->outputf(" flushes: %d", flushes); + _assm->outputf(" compiles: %d / %d", _stats.compiles, _stats.totalCompiles); + _assm->outputf(" used: %dk / %dk", (pages-free)<<(NJ_LOG2_PAGE_SIZE-10), pages<<(NJ_LOG2_PAGE_SIZE-10)); + _assm->outputf(" maxPageUse: %dk", (maxPageUse)<<(NJ_LOG2_PAGE_SIZE-10)); + _assm->output("\ntrace calls guards main native gen T-trace T-interp"); + + avmplus::SortedMap durs(_core->gc); + uint64_t totaldur=0; + fragstats totalstat = { 0,0,0,0,0 }; + for (int32_t i=0; ipeer) + break; + f = f->peer; + } + } + uint64_t totaltrace = totalstat.traceDur; + int totalsize = totalstat.size; + + _assm->outputf(""); + _assm->outputf("lirbytes %d / lir %d = %.1f bytes/lir", totalstat.lirbytes, + totalstat.lir, double(totalstat.lirbytes)/totalstat.lir); + _assm->outputf(" trace interp"); + _assm->outputf("%9lld (%2d%%) %9lld (%2d%%)", + totaltrace/1000, int(100.0*totaltrace/totaldur), + (totaldur-totaltrace)/1000, int(100.0*(totaldur-totaltrace)/totaldur)); + _assm->outputf(""); + _assm->outputf("trace ticks trace interp size"); + for (int32_t i=durs.size()-1; i >= 0; i--) { + uint64_t bothDur = durs.keyAt(i); + DurData d = durs.get(bothDur); + int size = d.size; + _assm->outputf("%-4s %9lld (%2d%%) %9lld (%2d%%) %9lld (%2d%%) %6d (%2d%%) %s", + labels->format(d.frag), + bothDur/1000, int(100.0*bothDur/totaldur), + d.traceDur/1000, int(100.0*d.traceDur/totaldur), + d.interpDur/1000, int(100.0*d.interpDur/totaldur), + size, int(100.0*size/totalsize), + labels->format(d.frag->ip)); + } + + } + + void Fragmento::countBlock(BlockHist *hist, const void* ip) + { + int c = hist->count(ip); + if (_assm->_logc->lcbits & LC_Assembly) + _assm->outputf("++ %s %d", labels->format(ip), c); + } + + void Fragmento::countIL(uint32_t il, uint32_t abc) + { + _stats.ilsize += il; + _stats.abcsize += abc; + } + +#ifdef AVMPLUS_VERBOSE + void Fragmento::drawTrees(char *fileName) { + drawTraceTrees(this, this->_frags, this->_core, fileName); + } +#endif +#endif // NJ_VERBOSE + // // Fragment // @@ -230,15 +526,16 @@ namespace nanojit fragEntry(NULL), loopEntry(NULL), vmprivate(NULL), - codeList(0), _code(NULL), - _hits(0) + _hits(0), + _pages(NULL) { } Fragment::~Fragment() { onDestroy(); + NanoAssert(_pages == 0); } void Fragment::blacklist() @@ -281,23 +578,28 @@ namespace nanojit lastIns = 0; } - void Fragment::releaseCode(CodeAlloc *codeAlloc) + void Fragment::releaseCode(Fragmento* frago) { _code = 0; - codeAlloc->freeAll(codeList); + while(_pages) + { + Page* next = _pages->next; + frago->pageFree(_pages); + _pages = next; + } } - void Fragment::releaseTreeMem(CodeAlloc *codeAlloc) + void Fragment::releaseTreeMem(Fragmento* frago) { releaseLirBuffer(); - releaseCode(codeAlloc); + releaseCode(frago); // now do it for all branches Fragment* branch = branches; while(branch) { Fragment* next = branch->nextbranch; - branch->releaseTreeMem(codeAlloc); // @todo safer here to recurse in case we support nested trees + branch->releaseTreeMem(frago); // @todo safer here to recurse in case we support nested trees NJ_DELETE(branch); branch = next; } diff --git a/js/src/nanojit/Fragmento.h b/js/src/nanojit/Fragmento.h index a5a2e0681e6..67e82e2e599 100644 --- a/js/src/nanojit/Fragmento.h +++ b/js/src/nanojit/Fragmento.h @@ -52,6 +52,27 @@ namespace nanojit struct GuardRecord; class Assembler; + struct PageHeader + { + struct Page *next; + }; + struct Page: public PageHeader + { + union { + // Conceptually, the lir array holds mostly LIns values (plus some + // skip payloads and call arguments). But we use int8_t as the + // element type here so the array size can be expressed in bytes. + int8_t lir[NJ_PAGE_SIZE-sizeof(PageHeader)]; + NIns code[(NJ_PAGE_SIZE-sizeof(PageHeader))/sizeof(NIns)]; + }; + }; + struct AllocEntry : public avmplus::GCObject + { + Page *page; + uint32_t allocSize; + }; + typedef avmplus::List AllocList; + typedef avmplus::GCSortedMap BlockSortedMap; class BlockHist: public BlockSortedMap { @@ -74,10 +95,15 @@ namespace nanojit class Fragmento : public avmplus::GCFinalizedObject { public: - Fragmento(AvmCore* core, LogControl* logc, uint32_t cacheSizeLog2, CodeAlloc *codeAlloc); + Fragmento(AvmCore* core, LogControl* logc, uint32_t cacheSizeLog2); ~Fragmento(); + void addMemory(void* firstPage, uint32_t pageCount); // gives memory to the Assembler + Assembler* assm(); AvmCore* core(); + Page* pageAlloc(); + void pageFree(Page* page); + void pagesRelease(PageList& list); Fragment* getLoop(const void* ip); Fragment* getAnchor(const void* ip); @@ -91,12 +117,18 @@ namespace nanojit Fragment* newBranch(Fragment *from, const void* ip); verbose_only ( uint32_t pageCount(); ) + verbose_only ( void dumpStats(); ) + verbose_only ( void dumpRatio(const char*, BlockHist*);) + verbose_only ( void dumpFragStats(Fragment*, int level, fragstats&); ) + verbose_only ( void countBlock(BlockHist*, const void* pc); ) + verbose_only ( void countIL(uint32_t il, uint32_t abc); ) verbose_only( void addLabel(Fragment* f, const char *prefix, int id); ) // stats struct { uint32_t pages; // pages consumed + uint32_t maxPageUse; // highwater mark of (pages-freePages) uint32_t flushes, ilsize, abcsize, compiles, totalCompiles; } _stats; @@ -109,11 +141,21 @@ namespace nanojit void drawTrees(char *fileName); #endif + uint32_t cacheUsed() const { return (_stats.pages-_freePages.size())< Fragment ptr */ + PageList _freePages; + + /* unmanaged mem */ + AllocList _allocList; + avmplus::GCHeap* _gcHeap; const uint32_t _max_pages; uint32_t _pagesGrowth; @@ -139,13 +181,14 @@ namespace nanojit ~Fragment(); NIns* code() { return _code; } - void setCode(NIns* codee) { _code = codee; } - int32_t& hits() { return _hits; } + Page* pages() { return _pages; } + void setCode(NIns* codee, Page* pages) { _code = codee; _pages = pages; } + int32_t& hits() { return _hits; } void blacklist(); bool isBlacklisted() { return _hits < 0; } void releaseLirBuffer(); - void releaseCode(CodeAlloc *alloc); - void releaseTreeMem(CodeAlloc *alloc); + void releaseCode(Fragmento* frago); + void releaseTreeMem(Fragmento* frago); bool isAnchor() { return anchor == this; } bool isRoot() { return root == this; } void onDestroy(); @@ -183,11 +226,12 @@ namespace nanojit NIns* fragEntry; NIns* loopEntry; void* vmprivate; - CodeList* codeList; private: NIns* _code; // ptr to start of code - int32_t _hits; + GuardRecord* _links; // code which is linked (or pending to be) to this fragment + int32_t _hits; + Page* _pages; // native code pages }; } #endif // __nanojit_Fragmento__ diff --git a/js/src/nanojit/LIR.cpp b/js/src/nanojit/LIR.cpp index 1e882c0f206..3b712638a01 100644 --- a/js/src/nanojit/LIR.cpp +++ b/js/src/nanojit/LIR.cpp @@ -109,41 +109,47 @@ namespace nanojit #endif /* NJ_PROFILE */ // LCompressedBuffer - LirBuffer::LirBuffer(Allocator& alloc) - : + LirBuffer::LirBuffer(Fragmento* frago) + : _frago(frago), #ifdef NJ_VERBOSE names(NULL), #endif - abi(ABI_FASTCALL), state(NULL), param1(NULL), sp(NULL), rp(NULL), - _allocator(alloc), _bytesAllocated(0) + abi(ABI_FASTCALL), + state(NULL), param1(NULL), sp(NULL), rp(NULL), + _pages(frago->core()->GetGC()) { - clear(); + rewind(); } LirBuffer::~LirBuffer() { clear(); verbose_only(if (names) NJ_DELETE(names);) + _frago = 0; } void LirBuffer::clear() { - // clear the stats, etc + // free all the memory and clear the stats + _frago->pagesRelease(_pages); + NanoAssert(!_pages.size()); _unused = 0; - _limit = 0; - _bytesAllocated = 0; _stats.lir = 0; + _noMem = 0; + _nextPage = 0; for (int i = 0; i < NumSavedRegs; ++i) savedRegs[i] = NULL; explicitSavedRegs = false; - chunkAlloc(); } - void LirBuffer::chunkAlloc() + void LirBuffer::rewind() { - _unused = (uintptr_t) _allocator.alloc(CHUNK_SZB); - NanoAssert(_unused != 0); // Allocator.alloc() never returns null. See Allocator.h - _limit = _unused + CHUNK_SZB; + clear(); + // pre-allocate the current and the next page we will be using + Page* start = pageAlloc(); + _unused = start ? uintptr_t(&start->lir[0]) : 0; + _nextPage = pageAlloc(); + NanoAssert((_unused && _nextPage) || _noMem); } int32_t LirBuffer::insCount() @@ -155,56 +161,75 @@ namespace nanojit size_t LirBuffer::byteCount() { - return _bytesAllocated - (_limit - _unused); + return ((_pages.size() ? _pages.size()-1 : 0) * sizeof(Page)) + + (_unused - pageTop(_unused)); + } + + Page* LirBuffer::pageAlloc() + { + Page* page = _frago->pageAlloc(); + if (page) + _pages.add(page); + else + _noMem = 1; + return page; } // Allocate a new page, and write the first instruction to it -- a skip // linking to last instruction of the previous page. - void LirBuffer::moveToNewChunk(uintptr_t addrOfLastLInsOnCurrentChunk) + void LirBuffer::moveToNewPage(uintptr_t addrOfLastLInsOnCurrentPage) { - chunkAlloc(); + // We don't want this to fail, so we always have a page in reserve. + NanoAssert(_nextPage); + _unused = uintptr_t(&_nextPage->lir[0]); + _nextPage = pageAlloc(); + NanoAssert(_nextPage || _noMem); + // Link LIR stream back to prior instruction. // Unlike all the ins*() functions, we don't call makeRoom() here // because we know we have enough space, having just started a new // page. LInsSk* insSk = (LInsSk*)_unused; LIns* ins = insSk->getLIns(); - ins->initLInsSk((LInsp)addrOfLastLInsOnCurrentChunk); + ins->initLInsSk((LInsp)addrOfLastLInsOnCurrentPage); _unused += sizeof(LInsSk); - verbose_only(_stats.lir++); + _stats.lir++; } // Make room for a single instruction. uintptr_t LirBuffer::makeRoom(size_t szB) { - // Make sure the size is ok + // Make sure the size is ok, and that we're not pointing to the + // PageHeader. NanoAssert(0 == szB % sizeof(void*)); - NanoAssert(sizeof(LIns) <= szB && szB <= MAX_LINS_SZB); - NanoAssert(_unused < _limit); + NanoAssert(sizeof(LIns) <= szB && szB <= NJ_MAX_LINS_SZB); + NanoAssert(_unused >= pageDataStart(_unused)); - // If the instruction won't fit on the current chunk, get a new chunk - if (_unused + szB > _limit) { - uintptr_t addrOfLastLInsOnChunk = _unused - sizeof(LIns); - moveToNewChunk(addrOfLastLInsOnChunk); + // If the instruction won't fit on the current page, move to the next + // page. + if (_unused + szB - 1 > pageBottom(_unused)) { + uintptr_t addrOfLastLInsOnPage = _unused - sizeof(LIns); + moveToNewPage(addrOfLastLInsOnPage); } - // We now know that we are on a chunk that has the requested amount of + // We now know that we are on a page that has the requested amount of // room: record the starting address of the requested space and bump // the pointer. uintptr_t startOfRoom = _unused; _unused += szB; - verbose_only(_stats.lir++); // count the instruction + _stats.lir++; // count the instruction - // If there's no more space on this chunk, move to a new one. + // If there's no more space on this page, move to the next page. // (This will only occur if the asked-for size filled up exactly to - // the end of the chunk.) This ensures that next time we enter this + // the end of the page.) This ensures that next time we enter this // function, _unused won't be pointing one byte past the end of - // the chunk, which would break everything. - if (_unused >= _limit) { - // Check we used exactly the remaining space - NanoAssert(_unused == _limit); - uintptr_t addrOfLastLInsOnChunk = _unused - sizeof(LIns); - moveToNewChunk(addrOfLastLInsOnChunk); + // the page, which would break everything. + if (_unused > pageBottom(startOfRoom)) { + // Check we only spilled over by one byte. + NanoAssert(_unused == pageTop(_unused)); + NanoAssert(_unused == pageBottom(startOfRoom) + 1); + uintptr_t addrOfLastLInsOnPage = _unused - sizeof(LIns); + moveToNewPage(addrOfLastLInsOnPage); } // Make sure it's word-aligned. @@ -317,16 +342,15 @@ namespace nanojit // NJ_MAX_SKIP_PAYLOAD_SZB, NJ_MAX_SKIP_PAYLOAD_SZB must also be a // multiple of the word size, which we check. payload_szB = alignUp(payload_szB, sizeof(void*)); - NanoAssert(0 == LirBuffer::MAX_SKIP_PAYLOAD_SZB % sizeof(void*)); - NanoAssert(sizeof(void*) <= payload_szB && payload_szB <= LirBuffer::MAX_SKIP_PAYLOAD_SZB); + NanoAssert(0 == NJ_MAX_SKIP_PAYLOAD_SZB % sizeof(void*)); + NanoAssert(sizeof(void*) <= payload_szB && payload_szB <= NJ_MAX_SKIP_PAYLOAD_SZB); uintptr_t payload = _buf->makeRoom(payload_szB + sizeof(LInsSk)); uintptr_t prevLInsAddr = payload - sizeof(LIns); LInsSk* insSk = (LInsSk*)(payload + payload_szB); LIns* ins = insSk->getLIns(); - // FIXME: restate these in a useful way. - // NanoAssert(prevLInsAddr >= pageDataStart(prevLInsAddr)); - // NanoAssert(samepage(prevLInsAddr, insSk)); + NanoAssert(prevLInsAddr >= pageDataStart(prevLInsAddr)); + NanoAssert(samepage(prevLInsAddr, insSk)); ins->initLInsSk((LInsp)prevLInsAddr); return ins; } @@ -371,6 +395,7 @@ namespace nanojit int argc = ((LInsp)i)->argc(); i -= sizeof(LInsC); // step over the instruction i -= argc*sizeof(LInsp); // step over the arguments + NanoAssert( samepage(i, _i) ); break; } @@ -2022,12 +2047,10 @@ namespace nanojit return i->arg(i->argc()-n-1); } - void compile(Fragmento* frago, Assembler* assm, Fragment* triggerFrag) + void compile(Assembler* assm, Fragment* triggerFrag) { + Fragmento *frago = triggerFrag->lirbuf->_frago; AvmCore *core = frago->core(); -#ifdef NJ_VERBOSE - LabelMap* labels = frago->labels; -#endif GC *gc = core->gc; verbose_only( @@ -2085,6 +2108,7 @@ namespace nanojit root = triggerFrag->root; root->fragEntry = 0; root->loopEntry = 0; + root->releaseCode(frago); // do the tree branches verbose_only( if (anyVerb) { @@ -2098,16 +2122,14 @@ namespace nanojit { verbose_only( if (anyVerb) { logc->printf("=== -- Compiling branch %s ip %s\n", - labels->format(frag), - labels->format(frag->ip)); + frago->labels->format(frag), + frago->labels->format(frag->ip)); }) - if (!assm->error()) { - assm->assemble(frag, loopJumps); - verbose_only(frago->_stats.compiles++); - verbose_only(frago->_stats.totalCompiles++); - } + assm->assemble(frag, loopJumps); verbose_only(if (asmVerb) - assm->outputf("## compiling branch %s ip %s", labels->format(frag), labels->format(frag->ip)); ) + assm->outputf("## compiling branch %s ip %s", + frago->labels->format(frag), + frago->labels->format(frag->ip)); ) NanoAssert(frag->kind == BranchTrace); RegAlloc* regs = NJ_NEW(gc, RegAlloc)(); @@ -2126,18 +2148,18 @@ namespace nanojit // now the the main trunk verbose_only( if (anyVerb) { logc->printf("=== -- Compile trunk %s: begin\n", - labels->format(root)); + frago->labels->format(root)); }) assm->assemble(root, loopJumps); verbose_only( if (anyVerb) { logc->printf("=== -- Compile trunk %s: end\n", - labels->format(root)); + frago->labels->format(root)); }) verbose_only( if (asmVerb) assm->outputf("## compiling trunk %s", - labels->format(root)); + frago->labels->format(root)); ) NanoAssert(!frago->core()->config.tree_opt || root == root->anchor || root->kind == MergeTrace); @@ -2167,10 +2189,6 @@ namespace nanojit root->fragEntry = 0; root->loopEntry = 0; } - else - { - CodeAlloc::moveAll(root->codeList, assm->codeList); - } /* BEGIN decorative postamble */ verbose_only( if (anyVerb) { @@ -2193,7 +2211,6 @@ namespace nanojit if (found) return found; return exprs.add(out->insLoad(v,base,disp), k); - } return out->insLoad(v, base, disp); } @@ -2227,8 +2244,8 @@ namespace nanojit #endif /* FEATURE_NANOJIT */ #if defined(NJ_VERBOSE) - LabelMap::LabelMap(AvmCore *core, nanojit::Allocator& a) - : allocator(a), names(core->gc), addrs(core->config.verbose_addrs), end(buf), core(core) + LabelMap::LabelMap(AvmCore *core) + : names(core->gc), addrs(core->config.verbose_addrs), end(buf), core(core) {} LabelMap::~LabelMap() diff --git a/js/src/nanojit/LIR.h b/js/src/nanojit/LIR.h index 4ad63d30a37..0ab5425524a 100644 --- a/js/src/nanojit/LIR.h +++ b/js/src/nanojit/LIR.h @@ -95,6 +95,7 @@ namespace nanojit struct GuardRecord; struct SideExit; + struct Page; enum AbiKind { ABI_FASTCALL, @@ -486,13 +487,7 @@ namespace nanojit private: // Last word: fields shared by all LIns kinds. The reservation fields // are read/written during assembly. - union { Reservation lastWord; - // force sizeof(LIns)==8 and 8-byte alignment on 64-bit machines. - // this is necessary because sizeof(Reservation)==4 and we want all - // instances of LIns to be pointer-aligned. - void* dummy; - }; // LIns-to-LInsXYZ converters. LInsOp0* toLInsOp0() const { return (LInsOp0*)( uintptr_t(this+1) - sizeof(LInsOp0) ); } @@ -658,6 +653,7 @@ namespace nanojit double imm64f() const; Reservation* resv() { return &lastWord; } void* payload() const; + inline Page* page() { return (Page*) alignTo(this,NJ_PAGE_SIZE); } inline int32_t size() const { NanoAssert(isop(LIR_ialloc)); return toLInsI()->imm32 << 2; @@ -844,6 +840,20 @@ namespace nanojit }; + // Each page has a header; the rest of it holds code. + #define NJ_PAGE_CODE_AREA_SZB (NJ_PAGE_SIZE - sizeof(PageHeader)) + + // The first instruction on a page is always a start instruction, or a + // payload-less skip instruction linking to the previous page. The + // biggest possible instruction would take up the entire rest of the page. + #define NJ_MAX_LINS_SZB (NJ_PAGE_CODE_AREA_SZB - sizeof(LInsSk)) + + // The maximum skip payload size is determined by the maximum instruction + // size. We require that a skip's payload be adjacent to the skip LIns + // itself. + #define NJ_MAX_SKIP_PAYLOAD_SZB (NJ_MAX_LINS_SZB - sizeof(LInsSk)) + + #ifdef NJ_VERBOSE extern const char* lirNames[]; @@ -852,7 +862,6 @@ namespace nanojit */ class LabelMap MMGC_SUBCLASS_DECL { - Allocator& allocator; class Entry MMGC_SUBCLASS_DECL { public: @@ -868,7 +877,7 @@ namespace nanojit void formatAddr(const void *p, char *buf); public: avmplus::AvmCore *core; - LabelMap(avmplus::AvmCore *, Allocator& allocator); + LabelMap(avmplus::AvmCore *); ~LabelMap(); void add(const void *p, size_t size, size_t align, const char *name); void add(const void *p, size_t size, size_t align, avmplus::String*); @@ -879,8 +888,6 @@ namespace nanojit class LirNameMap MMGC_SUBCLASS_DECL { - Allocator& allocator; - template class CountMap: public avmplus::SortedMap { public: @@ -910,9 +917,8 @@ namespace nanojit void formatImm(int32_t c, char *buf); public: - LirNameMap(GC *gc, Allocator& allocator, LabelMap *r) - : allocator(allocator), - lircounts(gc), + LirNameMap(GC *gc, LabelMap *r) + : lircounts(gc), funccounts(gc), names(gc), labels(r) @@ -1082,10 +1088,13 @@ namespace nanojit class LirBuffer : public GCFinalizedObject { public: - LirBuffer(Allocator&); - ~LirBuffer(); + DWB(Fragmento*) _frago; + LirBuffer(Fragmento* frago); + virtual ~LirBuffer(); void clear(); + void rewind(); uintptr_t makeRoom(size_t szB); // make room for an instruction + bool outOMem() { return _noMem != 0; } debug_only (void validate() const;) verbose_only(DWB(LirNameMap*) names;) @@ -1105,32 +1114,14 @@ namespace nanojit LInsp savedRegs[NumSavedRegs]; bool explicitSavedRegs; - /** each chunk is just a raw area of LIns instances, with no header - and no more than 8-byte alignment. The chunk size is somewhat arbitrary - as long as it's well larger than 2*sizeof(LInsSk) */ - static const size_t CHUNK_SZB = 8000; - - /** the first instruction on a chunk is always a start instruction, or a - * payload-less skip instruction linking to the previous chunk. The biggest - * possible instruction would take up the entire rest of the chunk. */ - static const size_t MAX_LINS_SZB = CHUNK_SZB - sizeof(LInsSk); - - /** the maximum skip payload size is determined by the maximum instruction - * size. We require that a skip's payload be adjacent to the skip LIns - * itself. */ - static const size_t MAX_SKIP_PAYLOAD_SZB = MAX_LINS_SZB - sizeof(LInsSk); - protected: - friend class LirBufWriter; + Page* pageAlloc(); + void moveToNewPage(uintptr_t addrOfLastLInsOnCurrentPage); - /** get CHUNK_SZB more memory for LIR instructions */ - void chunkAlloc(); - void moveToNewChunk(uintptr_t addrOfLastLInsOnCurrentChunk); - - Allocator& _allocator; - uintptr_t _unused; // next unused instruction slot in the current LIR chunk - uintptr_t _limit; // one past the last usable byte of the current LIR chunk - size_t _bytesAllocated; + PageList _pages; + Page* _nextPage; // allocated in preperation of a needing to growing the buffer + uintptr_t _unused; // next unused instruction slot + int _noMem; // set if ran out of memory when writing to buffer }; class LirBufWriter : public LirWriter @@ -1195,7 +1186,7 @@ namespace nanojit class Assembler; - void compile(Fragmento *frago, Assembler *assm, Fragment *frag); + void compile(Assembler *assm, Fragment *frag); verbose_only(void live(GC *gc, LirBuffer *lirbuf);) class StackFilter: public LirFilter diff --git a/js/src/nanojit/NativeARM.cpp b/js/src/nanojit/NativeARM.cpp index dfd42c69eaa..08c1ca29e44 100644 --- a/js/src/nanojit/NativeARM.cpp +++ b/js/src/nanojit/NativeARM.cpp @@ -536,7 +536,7 @@ Assembler::nFragExit(LInsp guard) } #ifdef NJ_VERBOSE - if (config.show_stats) { + if (_frago->core()->config.show_stats) { // load R1 with Fragment *fromFrag, target fragment // will make use of this when calling fragenter(). int fromfrag = int((Fragment*)_thisfrag); @@ -813,6 +813,32 @@ Assembler::asm_call(LInsp ins) } } +void +Assembler::nMarkExecute(Page* page, int flags) +{ + NanoAssert(sizeof(Page) == NJ_PAGE_SIZE); +#ifdef UNDER_CE + static const DWORD kProtFlags[4] = { + PAGE_READONLY, // 0 + PAGE_READWRITE, // PAGE_WRITE + PAGE_EXECUTE_READ, // PAGE_EXEC + PAGE_EXECUTE_READWRITE // PAGE_EXEC|PAGE_WRITE + }; + DWORD prot = kProtFlags[flags & (PAGE_WRITE|PAGE_EXEC)]; + DWORD dwOld; + BOOL res = VirtualProtect(page, NJ_PAGE_SIZE, prot, &dwOld); + if (!res) + { + // todo: we can't abort or assert here, we have to fail gracefully. + NanoAssertMsg(false, "FATAL ERROR: VirtualProtect() failed\n"); + } +#endif +#ifdef AVMPLUS_PORTING_API + NanoJIT_PortAPI_MarkExecutable(page, (void*)((char*)page+NJ_PAGE_SIZE), flags); + // todo, must add error-handling to the portapi +#endif +} + Register Assembler::nRegisterAllocFromSet(int set) { @@ -1308,17 +1334,21 @@ Assembler::nativePageReset() void Assembler::nativePageSetup() { - if (!_nIns) - codeAlloc(codeStart, codeEnd, _nIns); - if (!_nExitIns) - codeAlloc(exitStart, exitEnd, _nExitIns); + if (!_nIns) _nIns = pageAlloc(); + if (!_nExitIns) _nExitIns = pageAlloc(true); + //nj_dprintf("assemble onto %x exits into %x\n", (int)_nIns, (int)_nExitIns); - // constpool starts at top of page and goes down, - // code starts at bottom of page and moves up if (!_nSlot) - _nSlot = codeStart; - if (!_nExitSlot) - _nExitSlot = exitStart; + { + // This needs to be done or the samepage macro gets confused; pageAlloc + // gives us a pointer to just past the end of the page. + _nIns--; + _nExitIns--; + + // constpool starts at top of page and goes down, + // code starts at bottom of page and moves up + _nSlot = (int*)pageDataStart(_nIns); + } } // Record the starting value of _nIns. On ARM, it is also necessary to record @@ -1341,28 +1371,42 @@ Assembler::resetInstructionPointer() NanoAssert(samepage(_nIns,_nSlot)); } +// Note: underrunProtect should not touch any registers, even IP; it +// might need to allocate a new page in the middle of an IP-using +// sequence. void Assembler::underrunProtect(int bytes) { NanoAssertMsg(bytes<=LARGEST_UNDERRUN_PROT, "constant LARGEST_UNDERRUN_PROT is too small"); - NanoAssert(_nSlot != 0 && int(_nIns)-int(_nSlot) <= 4096); - uintptr_t top = uintptr_t(_nSlot); - uintptr_t pc = uintptr_t(_nIns); - if (pc - bytes < top) + intptr_t u = bytes + sizeof(PageHeader)/sizeof(NIns) + 8; + if ( (samepage(_nIns,_nSlot) && (((intptr_t)_nIns-u) <= intptr_t(_nSlot+1))) || + (!samepage((intptr_t)_nIns-u,_nIns)) ) { - verbose_only(verbose_outputf(" %p:", _nIns);) NIns* target = _nIns; - if (_inExit) - codeAlloc(exitStart, exitEnd, _nIns); - else - codeAlloc(codeStart, codeEnd, _nIns); - _nSlot = _inExit ? exitStart : codeStart; + _nIns = pageAlloc(_inExit); - // _nSlot points to the first empty position in the new code block - // _nIns points just past the last empty position. - // Assume B_nochk won't ever try to write to _nSlot. See B_cond_chk macro. - B_nochk(target); + // XXX _nIns at this point points to one past the end of + // the page, intended to be written into using *(--_nIns). + // However, (guess) something seems to be storing the value + // of _nIns as is, and then later generating a jump to a bogus + // address. So pre-decrement to ensure that it's always + // valid; we end up skipping using the last instruction this + // way. + _nIns--; + + // Update slot, either to _nIns (if decremented above), or + // _nIns-1 once the above bug is fixed/found. + _nSlot = (int*)pageDataStart(_nIns); + + // If samepage() is used on _nIns and _nSlot, it'll fail, since _nIns + // points to one past the end of the page right now. Assume that + // JMP_nochk won't ever try to write to _nSlot, and so won't ever + // check samepage(). See B_cond_chk macro. + JMP_nochk(target); + } else if (!_nSlot) { + // make sure that there's always a slot pointer + _nSlot = (int*)pageDataStart(_nIns); } } @@ -1569,22 +1613,19 @@ Assembler::asm_ld_imm(Register d, int32_t imm, bool chk /* = true */) underrunProtect(LD32_size); } - int offset = PC_OFFSET_FROM(_nSlot, _nIns-1); + int offset = PC_OFFSET_FROM(_nSlot+1, _nIns-1); // If the offset is out of range, waste literal space until it is in range. while (offset <= -4096) { ++_nSlot; offset += sizeof(_nSlot); } - NanoAssert(isS12(offset) && (offset < 0)); + NanoAssert(isS12(offset) && (offset < -8)); // Write the literal. - *(_nSlot++) = imm; - asm_output("## imm= 0x%x", imm); + *(++_nSlot) = imm; // Load the literal. LDR_nochk(d,PC,offset); - NanoAssert(uintptr_t(_nIns) + 8 + offset == uintptr_t(_nSlot-1)); - NanoAssert(*((int32_t*)_nSlot-1) == imm); } // Branch to target address _t with condition _c, doing underrun diff --git a/js/src/nanojit/NativeARM.h b/js/src/nanojit/NativeARM.h index eee8cd94b47..323c3b91e7e 100644 --- a/js/src/nanojit/NativeARM.h +++ b/js/src/nanojit/NativeARM.h @@ -676,9 +676,6 @@ enum { #define B_cond(_c,_t) \ B_cond_chk(_c,_t,1) -#define B_nochk(_t) \ - B_cond_chk(AL,_t,0) - // NB: don't use COND_AL here, we shift the condition into place! #define JMP(_t) \ B_cond_chk(AL,_t,1) diff --git a/js/src/nanojit/Nativei386.cpp b/js/src/nanojit/Nativei386.cpp index 8c69e07109d..b9ae9c271cc 100644 --- a/js/src/nanojit/Nativei386.cpp +++ b/js/src/nanojit/Nativei386.cpp @@ -123,7 +123,7 @@ namespace nanojit void Assembler::nFragExit(LInsp guard) { SideExit *exit = guard->record()->exit; - bool trees = config.tree_opt; + bool trees = _frago->core()->config.tree_opt; Fragment *frag = exit->target; GuardRecord *lr = 0; bool destKnown = (frag && frag->fragEntry); @@ -231,6 +231,52 @@ namespace nanojit SUBi(SP, extra); } + void Assembler::nMarkExecute(Page* page, int flags) + { + NanoAssert(sizeof(Page) == NJ_PAGE_SIZE); + #if defined WIN32 || defined WIN64 + DWORD dwIgnore; + static const DWORD kProtFlags[4] = + { + PAGE_READONLY, // 0 + PAGE_READWRITE, // PAGE_WRITE + PAGE_EXECUTE_READ, // PAGE_EXEC + PAGE_EXECUTE_READWRITE // PAGE_EXEC|PAGE_WRITE + }; + DWORD prot = kProtFlags[flags & (PAGE_WRITE|PAGE_EXEC)]; + BOOL res = VirtualProtect(page, NJ_PAGE_SIZE, prot, &dwIgnore); + if (!res) + { + // todo: we can't abort or assert here, we have to fail gracefully. + NanoAssertMsg(false, "FATAL ERROR: VirtualProtect() failed\n"); + } + #elif defined AVMPLUS_UNIX || defined AVMPLUS_MAC + static const int kProtFlags[4] = + { + PROT_READ, // 0 + PROT_READ|PROT_WRITE, // PAGE_WRITE + PROT_READ|PROT_EXEC, // PAGE_EXEC + PROT_READ|PROT_WRITE|PROT_EXEC // PAGE_EXEC|PAGE_WRITE + }; + int prot = kProtFlags[flags & (PAGE_WRITE|PAGE_EXEC)]; + intptr_t addr = (intptr_t)page; + addr &= ~((uintptr_t)NJ_PAGE_SIZE - 1); + NanoAssert(addr == (intptr_t)page); + #if defined SOLARIS + if (mprotect((char *)addr, NJ_PAGE_SIZE, prot) == -1) + #else + if (mprotect((void *)addr, NJ_PAGE_SIZE, prot) == -1) + #endif + { + // todo: we can't abort or assert here, we have to fail gracefully. + NanoAssertMsg(false, "FATAL ERROR: mprotect(PROT_EXEC) failed\n"); + abort(); + } + #else + (void)page; + #endif + } + Register Assembler::nRegisterAllocFromSet(int set) { Register r; @@ -1658,8 +1704,8 @@ namespace nanojit void Assembler::nativePageSetup() { - if (!_nIns) codeAlloc(codeStart, codeEnd, _nIns); - if (!_nExitIns) codeAlloc(exitStart, exitEnd, _nExitIns); + if (!_nIns) _nIns = pageAlloc(); + if (!_nExitIns) _nExitIns = pageAlloc(true); } // Reset the _nIns pointer to the starting value. This can be used to roll @@ -1679,15 +1725,15 @@ namespace nanojit // enough room for n bytes void Assembler::underrunProtect(int n) { - NIns *eip = _nIns; NanoAssertMsg(n<=LARGEST_UNDERRUN_PROT, "constant LARGEST_UNDERRUN_PROT is too small"); - if (eip - n < (_inExit ? exitStart : codeStart)) { + NIns *eip = this->_nIns; + Page *p = (Page*)pageTop(eip-1); + NIns *top = (NIns*) &p->code[0]; + if (eip - n < top) { // We are done with the current page. Tell Valgrind that new code // has been generated. - if (_inExit) - codeAlloc(exitStart, exitEnd, _nIns); - else - codeAlloc(codeStart, codeEnd, _nIns); + VALGRIND_DISCARD_TRANSLATIONS(pageTop(p), NJ_PAGE_SIZE); + _nIns = pageAlloc(_inExit); JMP(eip); } } diff --git a/js/src/nanojit/avmplus.cpp b/js/src/nanojit/avmplus.cpp index ec6a223f121..8a7d2439223 100644 --- a/js/src/nanojit/avmplus.cpp +++ b/js/src/nanojit/avmplus.cpp @@ -32,18 +32,7 @@ * ***** END LICENSE BLOCK ***** */ -#include "nanojit.h" - -#ifdef SOLARIS - #include - #include - #include - #include - extern "C" caddr_t _getfp(void); - typedef caddr_t maddr_ptr; -#else - typedef void *maddr_ptr; -#endif +#include "avmplus.h" using namespace avmplus; @@ -53,10 +42,6 @@ GC* AvmCore::gc = &_gc; GCHeap GC::heap; String* AvmCore::k_str[] = { (String*)"" }; -void -avmplus::AvmLog(char const *msg, ...) { -} - #ifdef _DEBUG // NanoAssertFail matches JS_Assert in jsutil.cpp. void NanoAssertFail() @@ -71,115 +56,3 @@ void NanoAssertFail() abort(); } #endif - -#ifdef WIN32 -void -VMPI_setPageProtection(void *address, - size_t size, - bool executableFlag, - bool writeableFlag) -{ - DWORD oldProtectFlags = 0; - DWORD newProtectFlags = 0; - if ( executableFlag && writeableFlag ) { - newProtectFlags = PAGE_EXECUTE_READWRITE; - } else if ( executableFlag ) { - newProtectFlags = PAGE_EXECUTE_READ; - } else if ( writeableFlag ) { - newProtectFlags = PAGE_READWRITE; - } else { - newProtectFlags = PAGE_READONLY; - } - - BOOL retval; - MEMORY_BASIC_INFORMATION mbi; - do { - VirtualQuery(address, &mbi, sizeof(MEMORY_BASIC_INFORMATION)); - size_t markSize = size > mbi.RegionSize ? mbi.RegionSize : size; - - retval = VirtualProtect(address, markSize, newProtectFlags, &oldProtectFlags); - NanoAssert(retval); - - address = (char*) address + markSize; - size -= markSize; - } while(size > 0 && retval); - - // We should not be clobbering PAGE_GUARD protections - NanoAssert((oldProtectFlags & PAGE_GUARD) == 0); -} - -#else // !WIN32 - -void VMPI_setPageProtection(void *address, - size_t size, - bool executableFlag, - bool writeableFlag) -{ - int bitmask = sysconf(_SC_PAGESIZE) - 1; - // mprotect requires that the addresses be aligned on page boundaries - void *endAddress = (void*) ((char*)address + size); - void *beginPage = (void*) ((size_t)address & ~bitmask); - void *endPage = (void*) (((size_t)endAddress + bitmask) & ~bitmask); - size_t sizePaged = (size_t)endPage - (size_t)beginPage; - - int flags = PROT_READ; - if (executableFlag) { - flags |= PROT_EXEC; - } - if (writeableFlag) { - flags |= PROT_WRITE; - } - int retval = mprotect((maddr_ptr)beginPage, (unsigned int)sizePaged, flags); - AvmAssert(retval == 0); - (void)retval; -} - -#endif // WIN32 - - - -#ifdef WIN32 -void* -nanojit::CodeAlloc::allocCodeChunk(size_t nbytes) { - return VirtualAlloc(NULL, - nbytes, - MEM_COMMIT | MEM_RESERVE, - PAGE_EXECUTE_READWRITE); -} - -void -nanojit::CodeAlloc::freeCodeChunk(void *p, size_t nbytes) { - VirtualFree(p, 0, MEM_RELEASE); -} - -#elif defined(AVMPLUS_UNIX) - -void* -nanojit::CodeAlloc::allocCodeChunk(size_t nbytes) { - return mmap(NULL, - nbytes, - PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_PRIVATE | MAP_ANON, - -1, - 0); -} - -void -nanojit::CodeAlloc::freeCodeChunk(void *p, size_t nbytes) { - munmap(p, nbytes); -} - -#else // !WIN32 && !AVMPLUS_UNIX - -void* -nanojit::CodeAlloc::allocCodeChunk(size_t nbytes) { - return valloc(nbytes); -} - -void -nanojit::CodeAlloc::freeCodeChunk(void *p, size_t nbytes) { - free(p); -} - -#endif // WIN32 - diff --git a/js/src/nanojit/avmplus.h b/js/src/nanojit/avmplus.h index f8a7f8e573a..60b16d41972 100644 --- a/js/src/nanojit/avmplus.h +++ b/js/src/nanojit/avmplus.h @@ -288,19 +288,12 @@ namespace MMgc { #define MMGC_MEM_TYPE(x) -extern void VMPI_setPageProtection(void *address, - size_t size, - bool executableFlag, - bool writeableFlag); - namespace avmplus { using namespace MMgc; typedef int FunctionID; - extern void AvmLog(char const *msg, ...); - class String { }; diff --git a/js/src/nanojit/nanojit.h b/js/src/nanojit/nanojit.h index 150ca6e11d3..6e27b20d899 100644 --- a/js/src/nanojit/nanojit.h +++ b/js/src/nanojit/nanojit.h @@ -120,12 +120,14 @@ namespace nanojit class LIns; struct SideExit; class RegAlloc; + struct Page; typedef avmplus::AvmCore AvmCore; typedef avmplus::OSDep OSDep; typedef avmplus::GCSortedMap FragmentMap; typedef avmplus::SortedMap RegAllocMap; typedef avmplus::List InsList; typedef avmplus::List StringList; + typedef avmplus::List PageList; const uint32_t MAXARGS = 8; @@ -293,9 +295,8 @@ namespace nanojit { // ------------------------------------------------------------------- -#include "Allocator.h" + #include "Native.h" -#include "CodeAlloc.h" #include "LIR.h" #include "RegAlloc.h" #include "Fragmento.h" diff --git a/js/src/vprof/vprof.h b/js/src/vprof/vprof.h index 4ae86a43a13..889361126ba 100644 --- a/js/src/vprof/vprof.h +++ b/js/src/vprof/vprof.h @@ -86,14 +86,16 @@ #define THREAD_SAFE 0 #ifdef _MSC_VER -typedef __int8 int8_t; -typedef __int16 int16_t; -typedef __int32 int32_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef signed char int8_t; +typedef short int16_t; +typedef unsigned int uint32_t; +typedef signed int int32_t; typedef __int64 int64_t; -typedef unsigned __int8 uint8_t; -typedef unsigned __int16 uint16_t; -typedef unsigned __int32 uint32_t; typedef unsigned __int64 uint64_t; +typedef long long int64_t; +typedef unsigned long long uint64_t; #else #include #endif @@ -118,11 +120,11 @@ int _histEntryValue (void* id, int64_t value); } #endif +#define DOPROF + #ifndef DOPROF #define _vprof(v) -#define _nvprof(n,v) #define _hprof(h) -#define _nhprof(n,h) #else #define _vprof(v,...) \