Bug 1210554 - Change representation of unbound Label linked lists. r=sstangl

Instead of storing byte offsets in the branch instructions using a
label, store instruction offsets, just like the finished branches do.
Use a 0 pc offset to terminate the linked list instead of -1.

This increases the maximum distance between linked branches to be the
same as the range of the branch instrructions. Previously, the
supported range was only 1/4 of what the branch instructions can
encode.

Provide protected functions for manipulating the linked list in
MozBaseAssembler, and rewrite Assembler::bind() and retarget() to use
them instead of decoding branches manually.

Move the LinkAndGet*OffsetTo functions into MozBaseAssembler. Our
version of these functions is completely different from the VIXL
versions.

js/src/jit/arm64/Assembler-arm64.cpp

@@ -225,27 +225,25 @@ Assembler::bind(Label* label, BufferOffset targetOffset)
 
     // Get the most recent instruction that used the label, as stored in the label.
     // This instruction is the head of an implicit linked list of label uses.
-    uint32_t branchOffset = label->offset();
-
-    while ((int32_t)branchOffset != LabelBase::INVALID_OFFSET) {
-        Instruction* link = getInstructionAt(BufferOffset(branchOffset));
+    BufferOffset branchOffset(label);
 
+    while (branchOffset.assigned()) {
         // Before overwriting the offset in this instruction, get the offset of
         // the next link in the implicit branch list.
-        uint32_t nextLinkOffset = uint32_t(link->ImmPCRawOffset());
-        if (nextLinkOffset != uint32_t(LabelBase::INVALID_OFFSET))
-            nextLinkOffset += branchOffset;
+        BufferOffset nextOffset = NextLink(branchOffset);
+
         // Linking against the actual (Instruction*) would be invalid,
         // since that Instruction could be anywhere in memory.
         // Instead, just link against the correct relative offset, assuming
         // no constant pools, which will be taken into consideration
         // during finalization.
-        ptrdiff_t relativeByteOffset = targetOffset.getOffset() - branchOffset;
-        Instruction* target = (Instruction*)(((uint8_t*)link) + relativeByteOffset);
+        ptrdiff_t relativeByteOffset = targetOffset.getOffset() - branchOffset.getOffset();
+        Instruction* link = getInstructionAt(branchOffset);
 
         // Write a new relative offset into the instruction.
-        link->SetImmPCOffsetTarget(target);
-        branchOffset = nextLinkOffset;
+        link->SetImmPCOffsetTarget(link + relativeByteOffset);
+
+        branchOffset = nextOffset;
     }
 
     // Bind the label, so that future uses may encode the offset immediately.
@@ -617,17 +615,18 @@ Assembler::retarget(Label* label, Label* target)
             // The target is not bound but used. Prepend label's branch list
             // onto target's.
             BufferOffset labelBranchOffset(label);
-            BufferOffset next;
 
             // Find the head of the use chain for label.
-            while (nextLink(labelBranchOffset, &next))
+            BufferOffset next = NextLink(labelBranchOffset);
+            while (next.assigned()) {
                 labelBranchOffset = next;
+                next = NextLink(next);
+            }
 
             // Then patch the head of label's use chain to the tail of target's
             // use chain, prepending the entire use chain of target.
-            Instruction* branch = getInstructionAt(labelBranchOffset);
+            SetNextLink(labelBranchOffset, BufferOffset(target));
             target->use(label->offset());
-            branch->SetImmPCOffsetTarget(branch - labelBranchOffset.getOffset());
         } else {
             // The target is unbound and unused. We can just take the head of
             // the list hanging off of label, and dump that into target.

js/src/jit/arm64/Assembler-arm64.h

@@ -243,14 +243,7 @@ class Assembler : public vixl::Assembler
     void Bind(uint8_t* rawCode, AbsoluteLabel* label, const void* address) {
         *reinterpret_cast<const void**>(rawCode + label->offset()) = address;
     }
-    bool nextLink(BufferOffset cur, BufferOffset* next) {
-        Instruction* link = getInstructionAt(cur);
-        uint32_t nextLinkOffset = uint32_t(link->ImmPCRawOffset());
-        if (nextLinkOffset == uint32_t(LabelBase::INVALID_OFFSET))
-            return false;
-        *next = BufferOffset(nextLinkOffset + cur.getOffset());
-        return true;
-    }
+
     void retarget(Label* cur, Label* next);
 
     // The buffer is about to be linked. Ensure any constant pools or

js/src/jit/arm64/vixl/Assembler-vixl.h

@@ -4225,17 +4225,6 @@ class Assembler : public MozBaseAssembler {
                             unsigned access_size,
                             LoadStoreScalingOption option);
 
-  // Link the current (not-yet-emitted) instruction to the specified label, then
-  // return an offset to be encoded in the instruction. If the label is not yet
-  // bound, an offset of 0 is returned.
-  ptrdiff_t LinkAndGetByteOffsetTo(BufferOffset branch, Label * label);
-  ptrdiff_t LinkAndGetInstructionOffsetTo(BufferOffset branch, Label * label);
-  ptrdiff_t LinkAndGetPageOffsetTo(BufferOffset branch, Label * label);
-
-  // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
-  template <int element_size>
-  ptrdiff_t LinkAndGetOffsetTo(BufferOffset branch, Label* label);
-
  protected:
   // Prevent generation of a literal pool for the next |maxInst| instructions.
   // Guarantees instruction linearity.

js/src/jit/arm64/vixl/Instructions-vixl.h

@@ -314,6 +314,7 @@ class Instruction {
   bool IsBranchLinkImm() const;
   bool IsTargetReachable(Instruction* target) const;
   ptrdiff_t ImmPCRawOffset() const;
+  void SetImmPCRawOffset(ptrdiff_t offset);
   void SetBits32(int msb, int lsb, unsigned value);
 
   // Is this a stack pointer synchronization instruction as inserted by

js/src/jit/arm64/vixl/MozAssembler-vixl.cpp

@@ -39,6 +39,51 @@ void Assembler::FinalizeCode() {
 #endif
 }
 
+// Unbound Label Representation.
+//
+// We can have multiple branches using the same label before it is bound.
+// Assembler::bind() must then be able to enumerate all the branches and patch
+// them to target the final label location.
+//
+// When a Label is unbound with uses, its offset is pointing to the tip of a
+// linked list of uses. The uses can be branches or adr/adrp instructions. In
+// the case of branches, the next member in the linked list is simply encoded
+// as the branch target. For adr/adrp, the relative pc offset is encoded in the
+// immediate field as a signed instruction offset.
+//
+// In both cases, the end of the list is encoded as a 0 pc offset, i.e. the
+// tail is pointing to itself.
+
+static const ptrdiff_t kEndOfLabelUseList = 0;
+
+BufferOffset
+MozBaseAssembler::NextLink(BufferOffset cur)
+{
+    Instruction* link = getInstructionAt(cur);
+    // Raw encoded offset.
+    ptrdiff_t offset = link->ImmPCRawOffset();
+    // End of the list is encoded as 0.
+    if (offset == kEndOfLabelUseList)
+        return BufferOffset();
+    // The encoded offset is the number of instructions to move.
+    return BufferOffset(cur.getOffset() + offset * kInstructionSize);
+}
+
+static ptrdiff_t
+EncodeOffset(BufferOffset cur, BufferOffset next)
+{
+    MOZ_ASSERT(next.assigned() && cur.assigned());
+    ptrdiff_t offset = next.getOffset() - cur.getOffset();
+    MOZ_ASSERT(offset % kInstructionSize == 0);
+    return offset / kInstructionSize;
+}
+
+void
+MozBaseAssembler::SetNextLink(BufferOffset cur, BufferOffset next)
+{
+    Instruction* link = getInstructionAt(cur);
+    link->SetImmPCRawOffset(EncodeOffset(cur, next));
+}
 
 // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
 //
@@ -50,9 +95,11 @@ void Assembler::FinalizeCode() {
 // multiple of element_size, then calculating the (scaled) offset between them.
 // This matches the semantics of adrp, for example.
 template <int element_size>
-ptrdiff_t Assembler::LinkAndGetOffsetTo(BufferOffset branch, Label* label) {
+ptrdiff_t
+MozBaseAssembler::LinkAndGetOffsetTo(BufferOffset branch, Label* label)
+{
   if (armbuffer_.oom())
-    return js::jit::LabelBase::INVALID_OFFSET;
+    return kEndOfLabelUseList;
 
   if (label->bound()) {
     // The label is bound: all uses are already linked.
@@ -65,33 +112,29 @@ ptrdiff_t Assembler::LinkAndGetOffsetTo(BufferOffset branch, Label* label) {
     // The label is unbound and unused: store the offset in the label itself
     // for patching by bind().
     label->use(branch.getOffset());
-    return js::jit::LabelBase::INVALID_OFFSET;
+    return kEndOfLabelUseList;
   }
 
   // The label is unbound but used. Create an implicit linked list between
   // the branches, and update the linked list head in the label struct.
-  ptrdiff_t prevHeadOffset = static_cast<ptrdiff_t>(label->offset());
+  ptrdiff_t offset = EncodeOffset(branch, BufferOffset(label));
   label->use(branch.getOffset());
-  VIXL_ASSERT(prevHeadOffset - branch.getOffset() != js::jit::LabelBase::INVALID_OFFSET);
-  return prevHeadOffset - branch.getOffset();
+  MOZ_ASSERT(offset != kEndOfLabelUseList);
+  return offset;
 }
 
-
-ptrdiff_t Assembler::LinkAndGetByteOffsetTo(BufferOffset branch, Label* label) {
+ptrdiff_t MozBaseAssembler::LinkAndGetByteOffsetTo(BufferOffset branch, Label* label) {
   return LinkAndGetOffsetTo<1>(branch, label);
 }
 
-
-ptrdiff_t Assembler::LinkAndGetInstructionOffsetTo(BufferOffset branch, Label* label) {
+ptrdiff_t MozBaseAssembler::LinkAndGetInstructionOffsetTo(BufferOffset branch, Label* label) {
   return LinkAndGetOffsetTo<kInstructionSize>(branch, label);
 }
 
-
-ptrdiff_t Assembler::LinkAndGetPageOffsetTo(BufferOffset branch, Label* label) {
+ptrdiff_t MozBaseAssembler::LinkAndGetPageOffsetTo(BufferOffset branch, Label* label) {
   return LinkAndGetOffsetTo<kPageSize>(branch, label);
 }
 
-
 BufferOffset Assembler::b(int imm26) {
   return EmitBranch(B | ImmUncondBranch(imm26));
 }

js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h

@@ -182,6 +182,27 @@ class MozBaseAssembler : public js::jit::AssemblerShared {
   static void RetargetNearBranch(Instruction* i, int offset, bool final = true);
   static void RetargetFarBranch(Instruction* i, uint8_t** slot, uint8_t* dest, Condition cond);
 
+ protected:
+  // Functions for managing Labels and linked lists of Label uses.
+
+  // Get the next Label user in the linked list of Label uses.
+  // Return an unassigned BufferOffset when the end of the list is reached.
+  BufferOffset NextLink(BufferOffset cur);
+
+  // Patch the instruction at cur to link to the instruction at next.
+  void SetNextLink(BufferOffset cur, BufferOffset next);
+
+  // Link the current (not-yet-emitted) instruction to the specified label,
+  // then return a raw offset to be encoded in the instruction.
+  ptrdiff_t LinkAndGetByteOffsetTo(BufferOffset branch, js::jit::Label* label);
+  ptrdiff_t LinkAndGetInstructionOffsetTo(BufferOffset branch, js::jit::Label* label);
+  ptrdiff_t LinkAndGetPageOffsetTo(BufferOffset branch, js::jit::Label* label);
+
+  // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
+  template <int element_size>
+  ptrdiff_t LinkAndGetOffsetTo(BufferOffset branch, js::jit::Label* label);
+
+
  protected:
   // The buffer into which code and relocation info are generated.
   ARMBuffer armbuffer_;

js/src/jit/arm64/vixl/MozInstructions-vixl.cpp

@@ -25,6 +25,7 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "jit/arm64/Architecture-arm64.h"
+#include "jit/arm64/vixl/Assembler-vixl.h"
 #include "jit/arm64/vixl/Instructions-vixl.h"
 
 namespace vixl {
@@ -130,6 +131,19 @@ ptrdiff_t Instruction::ImmPCRawOffset() const {
   return offset;
 }
 
+void
+Instruction::SetImmPCRawOffset(ptrdiff_t offset)
+{
+  if (IsPCRelAddressing()) {
+    // ADR and ADRP. We're encoding a raw offset here.
+    // See also SetPCRelImmTarget().
+    Instr imm = vixl::Assembler::ImmPCRelAddress(offset);
+    SetInstructionBits(Mask(~ImmPCRel_mask) | imm);
+  } else {
+    SetBranchImmTarget(this + (offset << kInstructionSizeLog2));
+  }
+}
+
 // Is this a stack pointer synchronization instruction as inserted by
 // MacroAssembler::syncStackPtr()?
 bool