tor-browser

MozAssembler-vixl.cpp (21418B)

---

// Copyright 2015, ARM Limited
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//   * Redistributions of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//   * Redistributions in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//   * Neither the name of ARM Limited nor the names of its contributors may be
//     used to endorse or promote products derived from this software without
//     specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "jit/arm64/vixl/Assembler-vixl.h"
#include "jit/Label.h"

namespace vixl {

using LabelDoc = js::jit::DisassemblerSpew::LabelDoc;

// Assembler
void Assembler::FinalizeCode() {
#ifdef DEBUG
 finalized_ = true;
#endif
}

// Unbound Label Representation.
//
// We can have multiple branches using the same label before it is bound.
// Assembler::bind() must then be able to enumerate all the branches and patch
// them to target the final label location.
//
// When a Label is unbound with uses, its offset is pointing to the tip of a
// linked list of uses. The uses can be branches or adr/adrp instructions. In
// the case of branches, the next member in the linked list is simply encoded
// as the branch target. For adr/adrp, the relative pc offset is encoded in the
// immediate field as a signed instruction offset.
//
// In both cases, the end of the list is encoded as a 0 pc offset, i.e. the
// tail is pointing to itself.

static const ptrdiff_t kEndOfLabelUseList = 0;

BufferOffset
MozBaseAssembler::NextLink(BufferOffset cur)
{
   Instruction* link = getInstructionAt(cur);
   // Raw encoded offset.
   ptrdiff_t offset = link->ImmPCRawOffset();
   // End of the list is encoded as 0.
   if (offset == kEndOfLabelUseList)
       return BufferOffset();
   // The encoded offset is the number of instructions to move.
   return BufferOffset(cur.getOffset() + offset * kInstructionSize);
}

static ptrdiff_t
EncodeOffset(BufferOffset cur, BufferOffset next)
{
   MOZ_ASSERT(next.assigned() && cur.assigned());
   ptrdiff_t offset = next.getOffset() - cur.getOffset();
   MOZ_ASSERT(offset % kInstructionSize == 0);
   return offset / kInstructionSize;
}

void
MozBaseAssembler::SetNextLink(BufferOffset cur, BufferOffset next)
{
   Instruction* link = getInstructionAt(cur);
   link->SetImmPCRawOffset(EncodeOffset(cur, next));
}

// A common implementation for the LinkAndGet<Type>OffsetTo helpers.
//
// If the label is bound, returns the offset as a multiple of 1 << elementShift.
// Otherwise, links the instruction to the label and returns the raw offset to
// encode. (This will be an instruction count.)
//
// The offset is calculated by aligning the PC and label addresses down to a
// multiple of 1 << elementShift, then calculating the (scaled) offset between
// them. This matches the semantics of adrp, for example. (Assuming that the
// assembler buffer is page-aligned, which it probably isn't.)
//
// For an unbound label, the returned offset will be encodable in the provided
// branch range. If the label is already bound, the caller is expected to make
// sure that it is in range, and emit the necessary branch instrutions if it
// isn't.
//
ptrdiff_t
MozBaseAssembler::LinkAndGetOffsetTo(BufferOffset branch, ImmBranchRangeType branchRange,
                                    unsigned elementShift, Label* label)
{
 if (armbuffer_.oom())
   return kEndOfLabelUseList;

 if (label->bound()) {
   // The label is bound: all uses are already linked.
   ptrdiff_t branch_offset = ptrdiff_t(branch.getOffset() >> elementShift);
   ptrdiff_t label_offset = ptrdiff_t(label->offset() >> elementShift);
   return label_offset - branch_offset;
 }

 // Keep track of short-range branches targeting unbound labels. We may need
 // to insert veneers in PatchShortRangeBranchToVeneer() below.
 if (branchRange < NumShortBranchRangeTypes) {
     // This is the last possible branch target.
     BufferOffset deadline(branch.getOffset() +
                           Instruction::ImmBranchMaxForwardOffset(branchRange));
     armbuffer_.registerBranchDeadline(branchRange, deadline);
 }

 // The label is unbound and previously unused: Store the offset in the label
 // itself for patching by bind().
 if (!label->used()) {
   label->use(branch.getOffset());
   return kEndOfLabelUseList;
 }

 // The label is unbound and has multiple users. Create a linked list between
 // the branches, and update the linked list head in the label struct. This is
 // not always trivial since the branches in the linked list have limited
 // ranges.

 // What is the earliest buffer offset that would be reachable by the branch
 // we're about to add?
 ptrdiff_t earliestReachable =
   branch.getOffset() + Instruction::ImmBranchMinBackwardOffset(branchRange);

 // If the existing instruction at the head of the list is within reach of the
 // new branch, we can simply insert the new branch at the front of the list.
 if (label->offset() >= earliestReachable) {
     ptrdiff_t offset = EncodeOffset(branch, BufferOffset(label));
     label->use(branch.getOffset());
     MOZ_ASSERT(offset != kEndOfLabelUseList);
     return offset;
 }

 // The label already has a linked list of uses, but we can't reach the head
 // of the list with the allowed branch range. Insert this branch at a
 // different position in the list.
 //
 // Find an existing branch, exbr, such that:
 //
 // 1.  The new branch can be reached by exbr, and either
 // 2a. The new branch can reach exbr's target, or
 // 2b. The exbr branch is at the end of the list.
 //
 // Then the new branch can be inserted after exbr in the linked list.
 //
 // We know that it is always possible to find an exbr branch satisfying these
 // conditions because of the PatchShortRangeBranchToVeneer() mechanism. All
 // branches are guaranteed to either be able to reach the end of the
 // assembler buffer, or they will be pointing to an unconditional branch that
 // can.
 //
 // In particular, the end of the list is always a viable candidate, so we'll
 // just get that.
 BufferOffset next(label);
 BufferOffset exbr;
 do {
     exbr = next;
     next = NextLink(next);
 } while (next.assigned());
 SetNextLink(exbr, branch);

 // This branch becomes the new end of the list.
 return kEndOfLabelUseList;
}

ptrdiff_t MozBaseAssembler::LinkAndGetByteOffsetTo(BufferOffset branch, Label* label) {
 return LinkAndGetOffsetTo(branch, UncondBranchRangeType, 0, label);
}

ptrdiff_t MozBaseAssembler::LinkAndGetInstructionOffsetTo(BufferOffset branch,
                                                         ImmBranchRangeType branchRange,
                                                         Label* label) {
 return LinkAndGetOffsetTo(branch, branchRange, kInstructionSizeLog2, label);
}

ptrdiff_t MozBaseAssembler::LinkAndGetPageOffsetTo(BufferOffset branch, Label* label) {
 return LinkAndGetOffsetTo(branch, UncondBranchRangeType, kPageSizeLog2, label);
}

BufferOffset Assembler::b(int imm26, const LabelDoc& doc) {
 return EmitBranch(B | ImmUncondBranch(imm26), doc);
}


void Assembler::b(Instruction* at, int imm26) {
 return EmitBranch(at, B | ImmUncondBranch(imm26));
}


BufferOffset Assembler::b(int imm19, Condition cond, const LabelDoc& doc) {
 return EmitBranch(B_cond | ImmCondBranch(imm19) | cond, doc);
}


void Assembler::b(Instruction* at, int imm19, Condition cond) {
 EmitBranch(at, B_cond | ImmCondBranch(imm19) | cond);
}


BufferOffset Assembler::b(Label* label) {
 // Encode the relative offset from the inserted branch to the label.
 LabelDoc doc = refLabel(label);
 return b(LinkAndGetInstructionOffsetTo(nextInstrOffset(), UncondBranchRangeType, label), doc);
}


BufferOffset Assembler::b(Label* label, Condition cond) {
 // Encode the relative offset from the inserted branch to the label.
 LabelDoc doc = refLabel(label);
 return b(LinkAndGetInstructionOffsetTo(nextInstrOffset(), CondBranchRangeType, label), cond, doc);
}

void Assembler::br(Instruction* at, const Register& xn) {
 VIXL_ASSERT(xn.Is64Bits());
 // No need for EmitBranch(): no immediate offset needs fixing.
 Emit(at, BR | Rn(xn));
}


void Assembler::blr(Instruction* at, const Register& xn) {
 VIXL_ASSERT(xn.Is64Bits());
 // No need for EmitBranch(): no immediate offset needs fixing.
 Emit(at, BLR | Rn(xn));
}


void Assembler::bl(int imm26, const LabelDoc& doc) {
 EmitBranch(BL | ImmUncondBranch(imm26), doc);
}


void Assembler::bl(Instruction* at, int imm26) {
 EmitBranch(at, BL | ImmUncondBranch(imm26));
}


void Assembler::bl(Label* label) {
 // Encode the relative offset from the inserted branch to the label.
 LabelDoc doc = refLabel(label);
 return bl(LinkAndGetInstructionOffsetTo(nextInstrOffset(), UncondBranchRangeType, label), doc);
}


void Assembler::cbz(const Register& rt, int imm19, const LabelDoc& doc) {
 EmitBranch(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt), doc);
}


void Assembler::cbz(Instruction* at, const Register& rt, int imm19) {
 EmitBranch(at, SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
}


void Assembler::cbz(const Register& rt, Label* label) {
 // Encode the relative offset from the inserted branch to the label.
 LabelDoc doc = refLabel(label);
 return cbz(rt, LinkAndGetInstructionOffsetTo(nextInstrOffset(), CondBranchRangeType, label), doc);
}


void Assembler::cbnz(const Register& rt, int imm19, const LabelDoc& doc) {
 EmitBranch(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt), doc);
}


void Assembler::cbnz(Instruction* at, const Register& rt, int imm19) {
 EmitBranch(at, SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
}


void Assembler::cbnz(const Register& rt, Label* label) {
 // Encode the relative offset from the inserted branch to the label.
 LabelDoc doc = refLabel(label);
 return cbnz(rt, LinkAndGetInstructionOffsetTo(nextInstrOffset(), CondBranchRangeType, label), doc);
}


void Assembler::tbz(const Register& rt, unsigned bit_pos, int imm14, const LabelDoc& doc) {
 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
 EmitBranch(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt), doc);
}


void Assembler::tbz(Instruction* at, const Register& rt, unsigned bit_pos, int imm14) {
 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
 EmitBranch(at, TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
}


void Assembler::tbz(const Register& rt, unsigned bit_pos, Label* label) {
 // Encode the relative offset from the inserted branch to the label.
 LabelDoc doc = refLabel(label);
 return tbz(rt, bit_pos, LinkAndGetInstructionOffsetTo(nextInstrOffset(), TestBranchRangeType, label), doc);
}


void Assembler::tbnz(const Register& rt, unsigned bit_pos, int imm14, const LabelDoc& doc) {
 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
 EmitBranch(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt), doc);
}


void Assembler::tbnz(Instruction* at, const Register& rt, unsigned bit_pos, int imm14) {
 VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
 EmitBranch(at, TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
}


void Assembler::tbnz(const Register& rt, unsigned bit_pos, Label* label) {
 // Encode the relative offset from the inserted branch to the label.
 LabelDoc doc = refLabel(label);
 return tbnz(rt, bit_pos, LinkAndGetInstructionOffsetTo(nextInstrOffset(), TestBranchRangeType, label), doc);
}


void Assembler::adr(const Register& rd, int imm21, const LabelDoc& doc) {
 VIXL_ASSERT(rd.Is64Bits());
 EmitBranch(ADR | ImmPCRelAddress(imm21) | Rd(rd), doc);
}


void Assembler::adr(Instruction* at, const Register& rd, int imm21) {
 VIXL_ASSERT(rd.Is64Bits());
 EmitBranch(at, ADR | ImmPCRelAddress(imm21) | Rd(rd));
}


void Assembler::adr(const Register& rd, Label* label) {
 // Encode the relative offset from the inserted adr to the label.
 LabelDoc doc = refLabel(label);
 return adr(rd, LinkAndGetByteOffsetTo(nextInstrOffset(), label), doc);
}


void Assembler::adrp(const Register& rd, int imm21, const LabelDoc& doc) {
 VIXL_ASSERT(rd.Is64Bits());
 EmitBranch(ADRP | ImmPCRelAddress(imm21) | Rd(rd), doc);
}


void Assembler::adrp(Instruction* at, const Register& rd, int imm21) {
 VIXL_ASSERT(rd.Is64Bits());
 EmitBranch(at, ADRP | ImmPCRelAddress(imm21) | Rd(rd));
}


void Assembler::adrp(const Register& rd, Label* label) {
 VIXL_ASSERT(AllowPageOffsetDependentCode());
 // Encode the relative offset from the inserted adr to the label.
 LabelDoc doc = refLabel(label);
 return adrp(rd, LinkAndGetPageOffsetTo(nextInstrOffset(), label), doc);
}


BufferOffset Assembler::ands(const Register& rd, const Register& rn, const Operand& operand) {
 return Logical(rd, rn, operand, ANDS);
}


BufferOffset Assembler::tst(const Register& rn, const Operand& operand) {
 return ands(AppropriateZeroRegFor(rn), rn, operand);
}


void Assembler::ldr(Instruction* at, const CPURegister& rt, int imm19) {
 LoadLiteralOp op = LoadLiteralOpFor(rt);
 Emit(at, op | ImmLLiteral(imm19) | Rt(rt));
}

void Assembler::ldrsw(Instruction* at, const CPURegister& rt, int imm19) {
 Emit(at, LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt));
}

BufferOffset Assembler::hint(SystemHint code) {
 return Emit(HINT | ImmHint(code));
}


void Assembler::hint(Instruction* at, SystemHint code) {
 Emit(at, HINT | ImmHint(code));
}


void Assembler::svc(Instruction* at, int code) {
 VIXL_ASSERT(IsUint16(code));
 Emit(at, SVC | ImmException(code));
}


void Assembler::nop(Instruction* at) {
 hint(at, NOP);
}


void Assembler::csdb(Instruction* at) {
 hint(at, CSDB);
}


BufferOffset Assembler::Logical(const Register& rd, const Register& rn,
                               const Operand& operand, LogicalOp op)
{
 VIXL_ASSERT(rd.size() == rn.size());
 if (operand.IsImmediate()) {
   int64_t immediate = operand.immediate();
   unsigned reg_size = rd.size();

   VIXL_ASSERT(immediate != 0);
   VIXL_ASSERT(immediate != -1);
   VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate));

   // If the operation is NOT, invert the operation and immediate.
   if ((op & NOT) == NOT) {
     op = static_cast<LogicalOp>(op & ~NOT);
     immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
   }

   unsigned n, imm_s, imm_r;
   if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
     // Immediate can be encoded in the instruction.
     return LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
   } else {
     // This case is handled in the macro assembler.
     VIXL_UNREACHABLE();
   }
 } else {
   VIXL_ASSERT(operand.IsShiftedRegister());
   VIXL_ASSERT(operand.reg().size() == rd.size());
   Instr dp_op = static_cast<Instr>(static_cast<Instr>(op) | LogicalShiftedFixed);
   return DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
 }
}


BufferOffset Assembler::LogicalImmediate(const Register& rd, const Register& rn,
                                        unsigned n, unsigned imm_s, unsigned imm_r, LogicalOp op)
{
   unsigned reg_size = rd.size();
   Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
   return Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
               ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg | Rn(rn));
}


BufferOffset Assembler::DataProcShiftedRegister(const Register& rd, const Register& rn,
                                               const Operand& operand, FlagsUpdate S, Instr op)
{
 VIXL_ASSERT(operand.IsShiftedRegister());
 VIXL_ASSERT(rn.Is64Bits() || (rn.Is32Bits() && IsUint5(operand.shift_amount())));
 return Emit(SF(rd) | op | Flags(S) |
             ShiftDP(operand.shift()) | ImmDPShift(operand.shift_amount()) |
             Rm(operand.reg()) | Rn(rn) | Rd(rd));
}


void MozBaseAssembler::InsertIndexIntoTag(uint8_t* load, uint32_t index) {
 // Store the js::jit::PoolEntry index into the instruction.
 // finishPool() will walk over all literal load instructions
 // and use PatchConstantPoolLoad() to patch to the final relative offset.
 *((uint32_t*)load) |= Assembler::ImmLLiteral(index);
}


bool MozBaseAssembler::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr) {
 Instruction* load = reinterpret_cast<Instruction*>(loadAddr);

 // The load currently contains the js::jit::PoolEntry's index,
 // as written by InsertIndexIntoTag().
 uint32_t index = load->ImmLLiteral();

 // Each entry in the literal pool is uint32_t-sized,
 // but literals may use multiple entries.
 uint32_t* constPool = reinterpret_cast<uint32_t*>(constPoolAddr);
 Instruction* source = reinterpret_cast<Instruction*>(&constPool[index]);

 load->SetImmLLiteral(source);
 return false; // Nothing uses the return value.
}

void
MozBaseAssembler::PatchShortRangeBranchToVeneer(ARMBuffer* buffer, unsigned rangeIdx,
                                               BufferOffset deadline, BufferOffset veneer)
{
 // Reconstruct the position of the branch from (rangeIdx, deadline).
 vixl::ImmBranchRangeType branchRange = static_cast<vixl::ImmBranchRangeType>(rangeIdx);
 BufferOffset branch(deadline.getOffset() - Instruction::ImmBranchMaxForwardOffset(branchRange));
 Instruction *branchInst = buffer->getInst(branch);
 Instruction *veneerInst = buffer->getInst(veneer);

 // Verify that the branch range matches what's encoded.
 MOZ_ASSERT(Instruction::ImmBranchTypeToRange(branchInst->BranchType()) == branchRange);

 // We want to insert veneer after branch in the linked list of instructions
 // that use the same unbound label.
 // The veneer should be an unconditional branch.
 ptrdiff_t nextElemOffset = branchInst->ImmPCRawOffset();

 // If offset is 0, this is the end of the linked list.
 if (nextElemOffset != kEndOfLabelUseList) {
     // Make the offset relative to veneer so it targets the same instruction
     // as branchInst.
     nextElemOffset *= kInstructionSize;
     nextElemOffset += branch.getOffset() - veneer.getOffset();
     nextElemOffset /= kInstructionSize;
 }
 Assembler::b(veneerInst, nextElemOffset);

 // Now point branchInst at veneer. See also SetNextLink() above.
 branchInst->SetImmPCRawOffset(EncodeOffset(branch, veneer));
}

struct PoolHeader {
 uint32_t data;

 struct Header {
   // The size should take into account the pool header.
   // The size is in units of Instruction (4bytes), not byte.
   union {
     struct {
       uint32_t size : 15;

// "Natural" guards are part of the normal instruction stream,
// while "non-natural" guards are inserted for the sole purpose
// of skipping around a pool.
       uint32_t isNatural : 1;
       uint32_t ONES : 16;
     };
     uint32_t data;
   };

   Header(int size_, bool isNatural_)
     : size(size_),
       isNatural(isNatural_),
       ONES(0xffff)
   { }

   Header(uint32_t data)
     : data(data)
   {
     VIXL_STATIC_ASSERT(sizeof(Header) == sizeof(uint32_t));
     VIXL_ASSERT(ONES == 0xffff);
   }

   uint32_t raw() const {
     VIXL_STATIC_ASSERT(sizeof(Header) == sizeof(uint32_t));
     return data;
   }
 };

 PoolHeader(int size_, bool isNatural_)
   : data(Header(size_, isNatural_).raw())
 { }

 uint32_t size() const {
   Header tmp(data);
   return tmp.size;
 }

 uint32_t isNatural() const {
   Header tmp(data);
   return tmp.isNatural;
 }
};


void MozBaseAssembler::WritePoolHeader(uint8_t* start, js::jit::Pool* p, bool isNatural) {
 static_assert(sizeof(PoolHeader) == 4);

 // Get the total size of the pool.
 const uintptr_t totalPoolSize = sizeof(PoolHeader) + p->getPoolSize();
 const uintptr_t totalPoolInstructions = totalPoolSize / kInstructionSize;

 VIXL_ASSERT((totalPoolSize & 0x3) == 0);
 VIXL_ASSERT(totalPoolInstructions < (1 << 15));

 PoolHeader header(totalPoolInstructions, isNatural);
 *(PoolHeader*)start = header;
}


void MozBaseAssembler::WritePoolFooter(uint8_t* start, js::jit::Pool* p, bool isNatural) {
 return;
}


void MozBaseAssembler::WritePoolGuard(BufferOffset branch, Instruction* inst, BufferOffset dest) {
 int byteOffset = dest.getOffset() - branch.getOffset();
 VIXL_ASSERT(byteOffset % kInstructionSize == 0);

 int instOffset = byteOffset >> kInstructionSizeLog2;
 Assembler::b(inst, instOffset);
}


}  // namespace vixl