tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

Assembler-x86-shared.cpp (11942B)

      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
      2 * vim: set ts=8 sts=2 et sw=2 tw=80:
      3 * This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "mozilla/Maybe.h"
      8 
      9 #include <algorithm>
     10 
     11 #include "jit/AutoWritableJitCode.h"
     12 #if defined(JS_CODEGEN_X86)
     13 #  include "jit/x86/MacroAssembler-x86.h"
     14 #elif defined(JS_CODEGEN_X64)
     15 #  include "jit/x64/MacroAssembler-x64.h"
     16 #else
     17 #  error "Wrong architecture. Only x86 and x64 should build this file!"
     18 #endif
     19 
     20 #ifdef _MSC_VER
     21 #  include <intrin.h>  // for __cpuid
     22 #  if defined(_M_X64) && (_MSC_FULL_VER >= 160040219)
     23 #    include <immintrin.h>  // for _xgetbv
     24 #  endif
     25 #endif
     26 
     27 using namespace js;
     28 using namespace js::jit;
     29 
     30 void AssemblerX86Shared::copyJumpRelocationTable(uint8_t* dest) {
     31  if (jumpRelocations_.length()) {
     32    memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
     33  }
     34 }
     35 
     36 void AssemblerX86Shared::copyDataRelocationTable(uint8_t* dest) {
     37  if (dataRelocations_.length()) {
     38    memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
     39  }
     40 }
     41 
     42 /* static */
     43 void AssemblerX86Shared::TraceDataRelocations(JSTracer* trc, JitCode* code,
     44                                              CompactBufferReader& reader) {
     45  mozilla::Maybe<AutoWritableJitCode> awjc;
     46 
     47  while (reader.more()) {
     48    size_t offset = reader.readUnsigned();
     49    MOZ_ASSERT(offset >= sizeof(void*) && offset <= code->instructionsSize());
     50 
     51    uint8_t* src = code->raw() + offset;
     52    void* data = X86Encoding::GetPointer(src);
     53 
     54 #ifdef JS_PUNBOX64
     55    // Data relocations can be for Values or for raw pointers. If a Value is
     56    // zero-tagged, we can trace it as if it were a raw pointer. If a Value
     57    // is not zero-tagged, we have to interpret it as a Value to ensure that the
     58    // tag bits are masked off to recover the actual pointer.
     59 
     60    uintptr_t word = reinterpret_cast<uintptr_t>(data);
     61    if (word >> JSVAL_TAG_SHIFT) {
     62      // This relocation is a Value with a non-zero tag.
     63      Value value = Value::fromRawBits(word);
     64      MOZ_ASSERT_IF(value.isGCThing(),
     65                    gc::IsCellPointerValid(value.toGCThing()));
     66      TraceManuallyBarrieredEdge(trc, &value, "jit-masm-value");
     67      if (word != value.asRawBits()) {
     68        if (awjc.isNothing()) {
     69          awjc.emplace(code);
     70        }
     71        X86Encoding::SetPointer(src, value.bitsAsPunboxPointer());
     72      }
     73      continue;
     74    }
     75 #endif
     76 
     77    // This relocation is a raw pointer or a Value with a zero tag.
     78    gc::Cell* cell = static_cast<gc::Cell*>(data);
     79    MOZ_ASSERT(gc::IsCellPointerValid(cell));
     80    TraceManuallyBarrieredGenericPointerEdge(trc, &cell, "jit-masm-ptr");
     81    if (cell != data) {
     82      if (awjc.isNothing()) {
     83        awjc.emplace(code);
     84      }
     85      X86Encoding::SetPointer(src, cell);
     86    }
     87  }
     88 }
     89 
     90 void AssemblerX86Shared::executableCopy(void* buffer) {
     91  masm.executableCopy(buffer);
     92 }
     93 
     94 void AssemblerX86Shared::processCodeLabels(uint8_t* rawCode) {
     95  for (const CodeLabel& label : codeLabels_) {
     96    Bind(rawCode, label);
     97  }
     98 }
     99 
    100 AssemblerX86Shared::Condition AssemblerX86Shared::InvertCondition(
    101    Condition cond) {
    102  switch (cond) {
    103    case Zero:
    104      return NonZero;
    105    case NonZero:
    106      return Zero;
    107    case LessThan:
    108      return GreaterThanOrEqual;
    109    case LessThanOrEqual:
    110      return GreaterThan;
    111    case GreaterThan:
    112      return LessThanOrEqual;
    113    case GreaterThanOrEqual:
    114      return LessThan;
    115    case Above:
    116      return BelowOrEqual;
    117    case AboveOrEqual:
    118      return Below;
    119    case Below:
    120      return AboveOrEqual;
    121    case BelowOrEqual:
    122      return Above;
    123    case Overflow:
    124      return NoOverflow;
    125    case NoOverflow:
    126      return Overflow;
    127    case Signed:
    128      return NotSigned;
    129    case NotSigned:
    130      return Signed;
    131    case Parity:
    132      return NoParity;
    133    case NoParity:
    134      return Parity;
    135  }
    136  MOZ_CRASH("unexpected condition");
    137 }
    138 
    139 AssemblerX86Shared::Condition AssemblerX86Shared::UnsignedCondition(
    140    Condition cond) {
    141  switch (cond) {
    142    case Zero:
    143    case NonZero:
    144      return cond;
    145    case LessThan:
    146    case Below:
    147      return Below;
    148    case LessThanOrEqual:
    149    case BelowOrEqual:
    150      return BelowOrEqual;
    151    case GreaterThan:
    152    case Above:
    153      return Above;
    154    case AboveOrEqual:
    155    case GreaterThanOrEqual:
    156      return AboveOrEqual;
    157    default:
    158      MOZ_CRASH("unexpected condition");
    159  }
    160 }
    161 
    162 AssemblerX86Shared::Condition AssemblerX86Shared::ConditionWithoutEqual(
    163    Condition cond) {
    164  switch (cond) {
    165    case LessThan:
    166    case LessThanOrEqual:
    167      return LessThan;
    168    case Below:
    169    case BelowOrEqual:
    170      return Below;
    171    case GreaterThan:
    172    case GreaterThanOrEqual:
    173      return GreaterThan;
    174    case Above:
    175    case AboveOrEqual:
    176      return Above;
    177    default:
    178      MOZ_CRASH("unexpected condition");
    179  }
    180 }
    181 
    182 AssemblerX86Shared::DoubleCondition AssemblerX86Shared::InvertCondition(
    183    DoubleCondition cond) {
    184  switch (cond) {
    185    case DoubleEqual:
    186      return DoubleNotEqualOrUnordered;
    187    case DoubleEqualOrUnordered:
    188      return DoubleNotEqual;
    189    case DoubleNotEqualOrUnordered:
    190      return DoubleEqual;
    191    case DoubleNotEqual:
    192      return DoubleEqualOrUnordered;
    193    case DoubleLessThan:
    194      return DoubleGreaterThanOrEqualOrUnordered;
    195    case DoubleLessThanOrUnordered:
    196      return DoubleGreaterThanOrEqual;
    197    case DoubleLessThanOrEqual:
    198      return DoubleGreaterThanOrUnordered;
    199    case DoubleLessThanOrEqualOrUnordered:
    200      return DoubleGreaterThan;
    201    case DoubleGreaterThan:
    202      return DoubleLessThanOrEqualOrUnordered;
    203    case DoubleGreaterThanOrUnordered:
    204      return DoubleLessThanOrEqual;
    205    case DoubleGreaterThanOrEqual:
    206      return DoubleLessThanOrUnordered;
    207    case DoubleGreaterThanOrEqualOrUnordered:
    208      return DoubleLessThan;
    209    default:
    210      MOZ_CRASH("unexpected condition");
    211  }
    212 }
    213 
    214 CPUInfo::SSEVersion CPUInfo::maxSSEVersion = UnknownSSE;
    215 CPUInfo::SSEVersion CPUInfo::maxEnabledSSEVersion = UnknownSSE;
    216 bool CPUInfo::avxPresent = false;
    217 #ifdef ENABLE_WASM_AVX
    218 bool CPUInfo::avxEnabled = true;
    219 #else
    220 bool CPUInfo::avxEnabled = false;
    221 #endif
    222 bool CPUInfo::popcntPresent = false;
    223 bool CPUInfo::bmi1Present = false;
    224 bool CPUInfo::bmi2Present = false;
    225 bool CPUInfo::lzcntPresent = false;
    226 bool CPUInfo::avx2Present = false;
    227 bool CPUInfo::fmaPresent = false;
    228 bool CPUInfo::f16cPresent = false;
    229 
    230 namespace js {
    231 namespace jit {
    232 bool CPUFlagsHaveBeenComputed() { return CPUInfo::FlagsHaveBeenComputed(); }
    233 }  // namespace jit
    234 }  // namespace js
    235 
    236 static uintptr_t ReadXGETBV() {
    237  // We use a variety of low-level mechanisms to get at the xgetbv
    238  // instruction, including spelling out the xgetbv instruction as bytes,
    239  // because older compilers and assemblers may not recognize the instruction
    240  // by name.
    241  size_t xcr0EAX = 0;
    242 #if defined(_XCR_XFEATURE_ENABLED_MASK)
    243  xcr0EAX = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
    244 #elif defined(__GNUC__)
    245  // xgetbv returns its results in %eax and %edx, and for our purposes here,
    246  // we're only interested in the %eax value.
    247  asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0EAX) : "c"(0) : "%edx");
    248 #elif defined(_MSC_VER) && defined(_M_IX86)
    249  __asm {
    250        xor ecx, ecx
    251        _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0
    252        mov xcr0EAX, eax
    253  }
    254 #endif
    255  return xcr0EAX;
    256 }
    257 
    258 static void ReadCPUInfo(int* flagsEax, int* flagsEbx, int* flagsEcx,
    259                        int* flagsEdx) {
    260 #ifdef _MSC_VER
    261  int cpuinfo[4];
    262  __cpuid(cpuinfo, *flagsEax);
    263  *flagsEax = cpuinfo[0];
    264  *flagsEbx = cpuinfo[1];
    265  *flagsEcx = cpuinfo[2];
    266  *flagsEdx = cpuinfo[3];
    267 #elif defined(__GNUC__)
    268  // Some older 32-bits processors don't fill the ecx register with cpuid, so
    269  // clobber it before calling cpuid, so that there's no risk of picking
    270  // random bits indicating SSE3/SSE4 are present. Also make sure that it's
    271  // set to 0 as an input for BMI detection on all platforms.
    272  *flagsEcx = 0;
    273 #  ifdef JS_CODEGEN_X64
    274  asm("cpuid;"
    275      : "+a"(*flagsEax), "=b"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
    276 #  else
    277  // On x86, preserve ebx. The compiler needs it for PIC mode.
    278  asm("mov %%ebx, %%edi;"
    279      "cpuid;"
    280      "xchg %%edi, %%ebx;"
    281      : "+a"(*flagsEax), "=D"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
    282 #  endif
    283 #else
    284 #  error "Unsupported compiler"
    285 #endif
    286 }
    287 
    288 void CPUInfo::ComputeFlags() {
    289  MOZ_ASSERT(!FlagsHaveBeenComputed());
    290 
    291  int flagsEax = 1;
    292  int flagsEbx = 0;
    293  int flagsEcx = 0;
    294  int flagsEdx = 0;
    295  ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
    296 
    297  static constexpr int SSEBit = 1 << 25;
    298  static constexpr int SSE2Bit = 1 << 26;
    299  static constexpr int SSE3Bit = 1 << 0;
    300  static constexpr int SSSE3Bit = 1 << 9;
    301  static constexpr int SSE41Bit = 1 << 19;
    302  static constexpr int SSE42Bit = 1 << 20;
    303 
    304  if (flagsEcx & SSE42Bit) {
    305    maxSSEVersion = SSE4_2;
    306  } else if (flagsEcx & SSE41Bit) {
    307    maxSSEVersion = SSE4_1;
    308  } else if (flagsEcx & SSSE3Bit) {
    309    maxSSEVersion = SSSE3;
    310  } else if (flagsEcx & SSE3Bit) {
    311    maxSSEVersion = SSE3;
    312  } else if (flagsEdx & SSE2Bit) {
    313    maxSSEVersion = SSE2;
    314  } else if (flagsEdx & SSEBit) {
    315    maxSSEVersion = SSE;
    316  } else {
    317    maxSSEVersion = NoSSE;
    318  }
    319 
    320  if (maxEnabledSSEVersion != UnknownSSE) {
    321    maxSSEVersion = std::min(maxSSEVersion, maxEnabledSSEVersion);
    322  }
    323 
    324  static constexpr int AVXBit = 1 << 28;
    325  static constexpr int XSAVEBit = 1 << 27;
    326  bool avxSupported = (flagsEcx & AVXBit);
    327  avxPresent = avxSupported && (flagsEcx & XSAVEBit) && avxEnabled;
    328 
    329  // If the hardware supports AVX, check whether the OS supports it too.
    330  if (avxPresent) {
    331    size_t xcr0EAX = ReadXGETBV();
    332    static constexpr int xcr0SSEBit = 1 << 1;
    333    static constexpr int xcr0AVXBit = 1 << 2;
    334    avxPresent = (xcr0EAX & xcr0SSEBit) && (xcr0EAX & xcr0AVXBit);
    335  }
    336 
    337  // CMOV instruction are supposed to be supported by all CPU which have SSE2
    338  // enabled. While this might be true, this is not guaranteed by any
    339  // documentation, nor AMD, nor Intel.
    340  static constexpr int CMOVBit = 1 << 15;
    341  MOZ_RELEASE_ASSERT(flagsEdx & CMOVBit,
    342                     "CMOVcc instruction is not recognized by this CPU.");
    343 
    344  static constexpr int POPCNTBit = 1 << 23;
    345  popcntPresent = (flagsEcx & POPCNTBit);
    346 
    347  // Use the avxEnabled flag to enable/disable FMA.
    348  static constexpr int FMABit = 1 << 12;
    349  fmaPresent = (flagsEcx & FMABit) && avxEnabled;
    350 
    351  // Support for the F16C instruction set. Requires AVX support.
    352  static constexpr int F16CBit = 1 << 29;
    353  f16cPresent = avxPresent && (flagsEcx & F16CBit);
    354 
    355  flagsEax = 0x80000001;
    356  ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
    357 
    358  static constexpr int LZCNTBit = 1 << 5;
    359  lzcntPresent = (flagsEcx & LZCNTBit);
    360 
    361  flagsEax = 0x7;
    362  ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
    363 
    364  // BMI1/2 instructions can have a VEX prefix. If the CPU doesn't support AVX,
    365  // it may not be able to decode the VEX prefix. So only enable BMI1 when AVX
    366  // is also supported.
    367  //
    368  // NOTE: This doesn't affect real hardware, because if BMI1 is supported by
    369  // the CPU, then AVX is also supported. Emulators on the other hand can
    370  // disable specific CPU features and emulate a CPU which supports BMI1, but
    371  // not AVX.
    372  //
    373  // Old QEMU versions (before release 7.2) don't support AVX, but appear to
    374  // report BMI1 as supported. When using BMI1 instructions with a VEX prefix,
    375  // like for example ANDN, QEMU will then abort because it can't decode ANDN.
    376  // Therefore we only enable BMI1 when AVX is also reported as supported.
    377  static constexpr int BMI1Bit = 1 << 3;
    378  static constexpr int BMI2Bit = 1 << 8;
    379  static constexpr int AVX2Bit = 1 << 5;
    380  bmi1Present = avxSupported && (flagsEbx & BMI1Bit);
    381  bmi2Present = bmi1Present && (flagsEbx & BMI2Bit);
    382  avx2Present = avxPresent && (flagsEbx & AVX2Bit);
    383 
    384  MOZ_ASSERT(FlagsHaveBeenComputed());
    385 }