From 57a4d24b342b8a591f36958ef2cb2dfda29e3adf Mon Sep 17 00:00:00 2001 From: Jakob Olesen Date: Mon, 23 Nov 2015 15:28:46 -0800 Subject: [PATCH] Bug 1210554 - Add PatchShortRangeBranchToVeneer(). r=nbp This is the second part of the short branch handling in AssemblerBufferWithConstantPools. The PatchShortRangeBranchToVeneer() callback is called from finishPool() to patch short-range branches that are about to expire. Implement no-op versions of the callback for ARM and ARM64. These versions will never be called as long as no short-line branches are registered. They only exist to prevent linker errors in unoptimized builds. In an optimized build, the unused function calls will be optimized out because DeadlineSet<0>::empty() is hardwired to return true. --- js/src/jit/arm/Assembler-arm.h | 7 ++ js/src/jit/arm64/vixl/MozAssembler-vixl.cpp | 9 +- js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h | 2 + .../IonAssemblerBufferWithConstantPools.h | 72 +++++++++++++++- js/src/jsapi-tests/testAssemblerBuffer.cpp | 86 +++++++++++++++++++ 5 files changed, 171 insertions(+), 5 deletions(-) diff --git a/js/src/jit/arm/Assembler-arm.h b/js/src/jit/arm/Assembler-arm.h index 3e42a759122..54f55aa9362 100644 --- a/js/src/jit/arm/Assembler-arm.h +++ b/js/src/jit/arm/Assembler-arm.h @@ -1855,6 +1855,13 @@ class Assembler : public AssemblerShared // load using the index we'd computed previously as well as the address of // the pool start. static void PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr); + + // We're not tracking short-range branches for ARM for now. + static void PatchShortRangeBranchToVeneer(ARMBuffer*, unsigned rangeIdx, BufferOffset deadline, + BufferOffset veneer) + { + MOZ_CRASH(); + } // END API // Move our entire pool into the instruction stream. This is to force an diff --git a/js/src/jit/arm64/vixl/MozAssembler-vixl.cpp b/js/src/jit/arm64/vixl/MozAssembler-vixl.cpp index a2ab5198ae0..fc6875462ed 100644 --- a/js/src/jit/arm64/vixl/MozAssembler-vixl.cpp +++ b/js/src/jit/arm64/vixl/MozAssembler-vixl.cpp @@ -369,12 +369,13 @@ bool MozBaseAssembler::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr return false; // Nothing uses the return value. } - -uint32_t MozBaseAssembler::PlaceConstantPoolBarrier(int offset) { - MOZ_CRASH("PlaceConstantPoolBarrier"); +void +MozBaseAssembler::PatchShortRangeBranchToVeneer(ARMBuffer*, unsigned rangeIdx, + BufferOffset deadline, BufferOffset veneer) +{ + MOZ_CRASH(); } - struct PoolHeader { uint32_t data; diff --git a/js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h b/js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h index 6c24d979944..d1bdde4fe86 100644 --- a/js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h +++ b/js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h @@ -169,6 +169,8 @@ class MozBaseAssembler : public js::jit::AssemblerShared { // Static interface used by IonAssemblerBufferWithConstantPools. static void InsertIndexIntoTag(uint8_t* load, uint32_t index); static bool PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr); + static void PatchShortRangeBranchToVeneer(ARMBuffer*, unsigned rangeIdx, BufferOffset deadline, + BufferOffset veneer); static uint32_t PlaceConstantPoolBarrier(int offset); static void WritePoolHeader(uint8_t* start, js::jit::Pool* p, bool isNatural); diff --git a/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h b/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h index 91cf2398527..56aeb3d5d1f 100644 --- a/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h +++ b/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h @@ -114,6 +114,35 @@ // InsertIndexIntoTag(). The constPoolAddr is the final address of the // constant pool in the assembler buffer. // +// void Asm::PatchShortRangeBranchToVeneer(AssemblerBufferWithConstantPools*, +// unsigned rangeIdx, +// BufferOffset deadline, +// BufferOffset veneer) +// +// Patch a short-range branch to jump through a veneer before it goes out of +// range. +// +// rangeIdx, deadline +// These arguments were previously passed to registerBranchDeadline(). It is +// assumed that PatchShortRangeBranchToVeneer() knows how to compute the +// offset of the short-range branch from this information. +// +// veneer +// Space for a branch veneer, guaranteed to be <= deadline. At this +// position, guardSize_ * InstSize bytes are allocated. They should be +// initialized to the proper unconditional branch instruction. +// +// Unbound branches to the same unbound label are organized as a linked list: +// +// Label::offset -> Branch1 -> Branch2 -> Branch3 -> nil +// +// This callback should insert a new veneer branch into the list: +// +// Label::offset -> Branch1 -> Branch2 -> Veneer -> Branch3 -> nil +// +// When Assembler::bind() rewrites the branches with the real label offset, it +// probably has to bind Branch2 to target the veneer branch instead of jumping +// straight to the label. namespace js { namespace jit { @@ -351,6 +380,13 @@ class BranchDeadlineSet<0u> // The allocation unit size for pools. typedef int32_t PoolAllocUnit; +// Hysteresis given to short-range branches. +// +// If any short-range branches will go out of range in the next N bytes, +// generate a veneer for them in the current pool. The hysteresis prevents the +// creation of many tiny constant pools for branch veneers. +const size_t ShortRangeBranchHysteresis = 128; + struct Pool { private: @@ -885,11 +921,24 @@ struct AssemblerBufferWithConstantPools : public AssemblerBuffernextOffset().getOffset() + ShortRangeBranchHysteresis > + size_t(branchDeadlines_.earliestDeadline().getOffset()); + } + void finishPool() { JitSpew(JitSpew_Pools, "[%d] Attempting to finish pool %d with %d entries.", id, poolInfo_.length(), pool_.numEntries()); - if (pool_.numEntries() == 0) { + if (pool_.numEntries() == 0 && !hasExpirableShortRangeBranches()) { // If there is no data in the pool being dumped, don't dump anything. JitSpew(JitSpew_Pools, "[%d] Aborting because the pool is empty", id); return; @@ -906,6 +955,27 @@ struct AssemblerBufferWithConstantPools : public AssemblerBufferoom()) return; + // Now generate branch veneers for any short-range branches that are + // about to expire. + while (hasExpirableShortRangeBranches()) { + unsigned rangeIdx = branchDeadlines_.earliestDeadlineRange(); + BufferOffset deadline = branchDeadlines_.earliestDeadline(); + + // Stop tracking this branch. The Asm callback below may register + // new branches to track. + branchDeadlines_.removeDeadline(rangeIdx, deadline); + + // Make room for the veneer. Same as a pool guard branch. + BufferOffset veneer = this->putBytes(guardSize_ * InstSize, nullptr); + if (this->oom()) + return; + + // Fix the branch so it targets the veneer. + // The Asm class knows how to find the original branch given the + // (rangeIdx, deadline) pair. + Asm::PatchShortRangeBranchToVeneer(this, rangeIdx, deadline, veneer); + } + // We only reserved space for the guard branch and pool header. // Fill them in. BufferOffset afterPool = this->nextOffset(); diff --git a/js/src/jsapi-tests/testAssemblerBuffer.cpp b/js/src/jsapi-tests/testAssemblerBuffer.cpp index 36574f4a5a4..5887f7a2c2d 100644 --- a/js/src/jsapi-tests/testAssemblerBuffer.cpp +++ b/js/src/jsapi-tests/testAssemblerBuffer.cpp @@ -287,6 +287,23 @@ struct TestAssembler uint32_t* hdr = reinterpret_cast(start); *hdr = 0xffff0000 + p->getPoolSize(); } + + static void PatchShortRangeBranchToVeneer(AsmBufWithPool* buffer, unsigned rangeIdx, + js::jit::BufferOffset deadline, + js::jit::BufferOffset veneer) + { + size_t branchOff = deadline.getOffset() - BranchRange; + size_t veneerOff = veneer.getOffset(); + uint32_t *branch = buffer->getInst(js::jit::BufferOffset(branchOff)); + + MOZ_ASSERT((*branch & 0xffff0000) == 0xb1bb0000, + "Expected short-range branch instruction"); + // Copy branch offset to veneer. A real instruction set would require + // some adjustment of the label linked-list. + *buffer->getInst(veneer) = 0xb2bb0000 | (*branch & 0xffff); + MOZ_ASSERT(veneerOff > branchOff, "Veneer should follow branch"); + *branch = 0xb3bb0000 + (veneerOff - branchOff); + } }; } @@ -421,3 +438,72 @@ BEGIN_TEST(testAssemblerBuffer_AssemblerBufferWithConstantPools) return true; } END_TEST(testAssemblerBuffer_AssemblerBufferWithConstantPools) + +BEGIN_TEST(testAssemblerBuffer_AssemblerBufferWithConstantPools_ShortBranch) +{ + using js::jit::BufferOffset; + + AsmBufWithPool ab(/* guardSize= */ 1, + /* headerSize= */ 1, + /* instBufferAlign(unused)= */ 0, + /* poolMaxOffset= */ 17, + /* pcBias= */ 0, + /* alignFillInst= */ 0x11110000, + /* nopFillInst= */ 0xaaaa0000, + /* nopFill= */ 0); + + // Insert short-range branch. + BufferOffset br1 = ab.putInt(0xb1bb00cc); + ab.registerBranchDeadline(1, BufferOffset(br1.getOffset() + TestAssembler::BranchRange)); + ab.putInt(0x22220001); + BufferOffset off = ab.putInt(0x22220002); + ab.registerBranchDeadline(1, BufferOffset(off.getOffset() + TestAssembler::BranchRange)); + ab.putInt(0x22220003); + ab.putInt(0x22220004); + + // Second short-range branch that will be swiped up by hysteresis. + BufferOffset br2 = ab.putInt(0xb1bb0d2d); + ab.registerBranchDeadline(1, BufferOffset(br2.getOffset() + TestAssembler::BranchRange)); + + // Branch should not have been patched yet here. + CHECK_EQUAL(*ab.getInst(br1), 0xb1bb00cc); + CHECK_EQUAL(*ab.getInst(br2), 0xb1bb0d2d); + + // Cancel one of the pending branches. + // This is what will happen to most branches as they are bound before + // expiring by Assembler::bind(). + ab.unregisterBranchDeadline(1, BufferOffset(off.getOffset() + TestAssembler::BranchRange)); + + off = ab.putInt(0x22220006); + // Here we may or may not have patched the branch yet, but it is inevitable now: + // + // 0: br1 pc+36 + // 4: 0x22220001 + // 8: 0x22220002 (unpatched) + // 12: 0x22220003 + // 16: 0x22220004 + // 20: br2 pc+20 + // 24: 0x22220006 + CHECK_EQUAL(off.getOffset(), 24); + // 28: guard branch pc+16 + // 32: pool header + // 36: veneer1 + // 40: veneer2 + // 44: 0x22220007 + + off = ab.putInt(0x22220007); + CHECK_EQUAL(off.getOffset(), 44); + + // Now the branch must have been patched. + CHECK_EQUAL(*ab.getInst(br1), 0xb3bb0000 + 36); // br1 pc+36 (patched) + CHECK_EQUAL(*ab.getInst(BufferOffset(8)), 0x22220002u); // 0x22220002 (unpatched) + CHECK_EQUAL(*ab.getInst(br2), 0xb3bb0000 + 20); // br2 pc+20 (patched) + CHECK_EQUAL(*ab.getInst(BufferOffset(28)), 0xb0bb0010u); // br pc+16 (guard) + CHECK_EQUAL(*ab.getInst(BufferOffset(32)), 0xffff0000u); // pool header 0 bytes. + CHECK_EQUAL(*ab.getInst(BufferOffset(36)), 0xb2bb00ccu); // veneer1 w/ original 'cc' offset. + CHECK_EQUAL(*ab.getInst(BufferOffset(40)), 0xb2bb0d2du); // veneer2 w/ original 'cc' offset. + CHECK_EQUAL(*ab.getInst(BufferOffset(44)), 0x22220007u); + + return true; +} +END_TEST(testAssemblerBuffer_AssemblerBufferWithConstantPools_ShortBranch)