tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

JitRuntime.h (18395B)

      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
      2 * vim: set ts=8 sts=2 et sw=2 tw=80:
      3 * This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #ifndef jit_JitRuntime_h
      8 #define jit_JitRuntime_h
      9 
     10 #include "mozilla/Assertions.h"
     11 #include "mozilla/Atomics.h"
     12 #include "mozilla/EnumeratedArray.h"
     13 #include "mozilla/LinkedList.h"
     14 
     15 #include <stddef.h>
     16 #include <stdint.h>
     17 
     18 #include "jstypes.h"
     19 
     20 #include "jit/ABIFunctions.h"
     21 #include "jit/BaselineICList.h"
     22 #include "jit/BaselineJIT.h"
     23 #include "jit/CalleeToken.h"
     24 #include "jit/InterpreterEntryTrampoline.h"
     25 #include "jit/IonCompileTask.h"
     26 #include "jit/IonTypes.h"
     27 #include "jit/JitCode.h"
     28 #include "jit/JitHints.h"
     29 #include "jit/shared/Assembler-shared.h"
     30 #include "jit/TrampolineNatives.h"
     31 #include "js/AllocPolicy.h"
     32 #include "js/ProfilingFrameIterator.h"
     33 #include "js/TypeDecls.h"
     34 #include "js/UniquePtr.h"
     35 #include "js/Vector.h"
     36 #include "threading/ProtectedData.h"
     37 #include "vm/GeckoProfiler.h"
     38 #include "vm/Runtime.h"
     39 
     40 class JS_PUBLIC_API JSTracer;
     41 
     42 namespace js {
     43 
     44 class AutoLockHelperThreadState;
     45 class GCMarker;
     46 enum class ArraySortKind;
     47 
     48 namespace jit {
     49 
     50 class FrameSizeClass;
     51 class Label;
     52 class MacroAssembler;
     53 struct VMFunctionData;
     54 
     55 enum class VMFunctionId;
     56 
     57 enum class BaselineICFallbackKind : uint8_t {
     58 #define DEF_ENUM_KIND(kind) kind,
     59  IC_BASELINE_FALLBACK_CODE_KIND_LIST(DEF_ENUM_KIND)
     60 #undef DEF_ENUM_KIND
     61      Count
     62 };
     63 
     64 enum class BailoutReturnKind {
     65  GetProp,
     66  GetPropSuper,
     67  SetProp,
     68  GetElem,
     69  GetElemSuper,
     70  Call,
     71  New,
     72  Count
     73 };
     74 
     75 // Class storing code and offsets for all Baseline IC fallback trampolines. This
     76 // is stored in JitRuntime and generated when creating the JitRuntime.
     77 class BaselineICFallbackCode {
     78  JitCode* code_ = nullptr;
     79  using OffsetArray =
     80      mozilla::EnumeratedArray<BaselineICFallbackKind, uint32_t,
     81                               size_t(BaselineICFallbackKind::Count)>;
     82  OffsetArray offsets_ = {};
     83 
     84  // Keep track of offset into various baseline stubs' code at return
     85  // point from called script.
     86  using BailoutReturnArray =
     87      mozilla::EnumeratedArray<BailoutReturnKind, uint32_t,
     88                               size_t(BailoutReturnKind::Count)>;
     89  BailoutReturnArray bailoutReturnOffsets_ = {};
     90 
     91 public:
     92  BaselineICFallbackCode() = default;
     93  BaselineICFallbackCode(const BaselineICFallbackCode&) = delete;
     94  void operator=(const BaselineICFallbackCode&) = delete;
     95 
     96  void initOffset(BaselineICFallbackKind kind, uint32_t offset) {
     97    offsets_[kind] = offset;
     98  }
     99  void initCode(JitCode* code) { code_ = code; }
    100  void initBailoutReturnOffset(BailoutReturnKind kind, uint32_t offset) {
    101    bailoutReturnOffsets_[kind] = offset;
    102  }
    103  TrampolinePtr addr(BaselineICFallbackKind kind) const {
    104    return TrampolinePtr(code_->raw() + offsets_[kind]);
    105  }
    106  uint8_t* bailoutReturnAddr(BailoutReturnKind kind) const {
    107    return code_->raw() + bailoutReturnOffsets_[kind];
    108  }
    109 };
    110 
    111 enum class DebugTrapHandlerKind { Interpreter, Compiler, Count };
    112 
    113 enum class IonGenericCallKind { Call, Construct, Count };
    114 
    115 using EnterJitCode = void (*)(void*, unsigned int, Value*, InterpreterFrame*,
    116                              CalleeToken, JSObject*, size_t, Value*);
    117 
    118 class JitcodeGlobalTable;
    119 class PerfSpewerRangeRecorder;
    120 
    121 class JitRuntime {
    122 private:
    123  MainThreadData<uint64_t> nextCompilationId_{0};
    124 
    125  // Buffer for OSR from baseline to Ion. To avoid holding on to this for too
    126  // long it's also freed in EnterBaseline and EnterJit (after returning from
    127  // JIT code).
    128  MainThreadData<js::UniquePtr<uint8_t>> ionOsrTempData_{nullptr};
    129  MainThreadData<uint32_t> ionOsrTempDataSize_{0};
    130 
    131  // List of Ion compile tasks that should be freed. Used to batch multiple
    132  // tasks into a single IonFreeTask.
    133  MainThreadData<IonFreeCompileTasks> ionFreeTaskBatch_;
    134 
    135  // Shared exception-handler tail.
    136  WriteOnceData<uint32_t> exceptionTailOffset_{0};
    137  WriteOnceData<uint32_t> exceptionTailReturnValueCheckOffset_{0};
    138 
    139  // Shared profiler exit frame tail.
    140  WriteOnceData<uint32_t> profilerExitFrameTailOffset_{0};
    141 
    142  // Trampoline for entering JIT code.
    143  WriteOnceData<uint32_t> enterJITOffset_{0};
    144 
    145  // Generic bailout table; used if the bailout table overflows.
    146  WriteOnceData<uint32_t> bailoutHandlerOffset_{0};
    147 
    148  // Thunk that invalides an (Ion compiled) caller on the Ion stack.
    149  WriteOnceData<uint32_t> invalidatorOffset_{0};
    150 
    151  // Thunk that calls the GC pre barrier.
    152  WriteOnceData<uint32_t> valuePreBarrierOffset_{0};
    153  WriteOnceData<uint32_t> stringPreBarrierOffset_{0};
    154  WriteOnceData<uint32_t> objectPreBarrierOffset_{0};
    155  WriteOnceData<uint32_t> shapePreBarrierOffset_{0};
    156  WriteOnceData<uint32_t> wasmAnyRefPreBarrierOffset_{0};
    157 
    158  // Thunk called to finish compilation of an IonScript.
    159  WriteOnceData<uint32_t> lazyLinkStubOffset_{0};
    160 
    161  // Thunk to enter the interpreter from JIT code.
    162  WriteOnceData<uint32_t> interpreterStubOffset_{0};
    163 
    164  // Thunk to convert the value in R0 to int32 if it's a double.
    165  // Note: this stub treats -0 as +0 and may clobber R1.scratchReg().
    166  WriteOnceData<uint32_t> doubleToInt32ValueStubOffset_{0};
    167 
    168  // Thunk to do a generic call from Ion.
    169  mozilla::EnumeratedArray<IonGenericCallKind, WriteOnceData<uint32_t>,
    170                           size_t(IonGenericCallKind::Count)>
    171      ionGenericCallStubOffset_;
    172 
    173  // Thunk used by the debugger for breakpoint and step mode.
    174  mozilla::EnumeratedArray<DebugTrapHandlerKind, WriteOnceData<JitCode*>,
    175                           size_t(DebugTrapHandlerKind::Count)>
    176      debugTrapHandlers_;
    177 
    178  // BaselineInterpreter state.
    179  BaselineInterpreter baselineInterpreter_;
    180 
    181  // Code for trampolines and VMFunction wrappers.
    182  WriteOnceData<JitCode*> trampolineCode_{nullptr};
    183 
    184  // Thunk that calls into the C++ interpreter from the interpreter
    185  // entry trampoline that is generated with --emit-interpreter-entry
    186  WriteOnceData<uint32_t> vmInterpreterEntryOffset_{0};
    187 
    188  // Maps VMFunctionId to the offset of the wrapper code in trampolineCode_.
    189  using VMWrapperOffsets = Vector<uint32_t, 0, SystemAllocPolicy>;
    190  VMWrapperOffsets functionWrapperOffsets_;
    191 
    192  MainThreadData<BaselineICFallbackCode> baselineICFallbackCode_;
    193 
    194  // Global table of jitcode native address => bytecode address mappings.
    195  UnprotectedData<JitcodeGlobalTable*> jitcodeGlobalTable_{nullptr};
    196 
    197  // Map that stores Jit Hints for each script.
    198  MainThreadData<JitHintsMap*> jitHintsMap_{nullptr};
    199 
    200  // Map used to collect entry trampolines for the Interpreters which is used
    201  // for external profiling to identify which functions are being interpreted.
    202  MainThreadData<EntryTrampolineMap*> interpreterEntryMap_{nullptr};
    203 
    204 #ifdef DEBUG
    205  // The number of possible bailing places encountered before forcefully bailing
    206  // in that place if the counter reaches zero. Note that zero also means
    207  // inactive.
    208  MainThreadData<uint32_t> ionBailAfterCounter_{0};
    209 
    210  // Whether the bailAfter mechanism is enabled. Used to avoid generating the
    211  // Ion code instrumentation for ionBailAfterCounter_ if the testing function
    212  // isn't used.
    213  MainThreadData<bool> ionBailAfterEnabled_{false};
    214 #endif
    215 
    216  // Number of Ion compilations which were finished off thread and are
    217  // waiting to be lazily linked. This is only set while holding the helper
    218  // thread state lock, but may be read from at other times.
    219  using NumFinishedOffThreadTasksType =
    220      mozilla::Atomic<size_t, mozilla::SequentiallyConsistent>;
    221  NumFinishedOffThreadTasksType numFinishedOffThreadTasks_{0};
    222 
    223  // List of Ion compilation waiting to get linked.
    224  using IonCompileTaskList = mozilla::LinkedList<js::jit::IonCompileTask>;
    225  MainThreadData<IonCompileTaskList> ionLazyLinkList_;
    226  MainThreadData<size_t> ionLazyLinkListSize_{0};
    227 
    228  // Pointer to trampoline code for each TrampolineNative. The JSFunction has
    229  // a JitEntry pointer that points to an item in this array.
    230  using TrampolineNativeJitEntryArray =
    231      mozilla::EnumeratedArray<TrampolineNative, void*,
    232                               size_t(TrampolineNative::Count)>;
    233  TrampolineNativeJitEntryArray trampolineNativeJitEntries_{};
    234 
    235 #ifdef DEBUG
    236  // Flag that can be set from JIT code to indicate it's invalid to call
    237  // arbitrary JS code in a particular region. This is checked in RunScript.
    238  MainThreadData<uint32_t> disallowArbitraryCode_{false};
    239 #endif
    240 
    241  bool generateTrampolines(JSContext* cx);
    242  bool generateBaselineICFallbackCode(JSContext* cx);
    243 
    244  void generateLazyLinkStub(MacroAssembler& masm);
    245  void generateInterpreterStub(MacroAssembler& masm);
    246  void generateDoubleToInt32ValueStub(MacroAssembler& masm);
    247  void generateProfilerExitFrameTailStub(MacroAssembler& masm,
    248                                         Label* profilerExitTail);
    249  void generateExceptionTailStub(MacroAssembler& masm, Label* profilerExitTail,
    250                                 Label* bailoutTail);
    251  void generateBailoutTailStub(MacroAssembler& masm, Label* bailoutTail);
    252  void generateEnterJIT(JSContext* cx, MacroAssembler& masm);
    253  void generateEnterJitShared(MacroAssembler& masm, Register argcReg,
    254                              Register argvReg, Register calleeTokenReg,
    255                              Register scratch, Register scratch2,
    256                              Register scratch3);
    257  void generateBailoutHandler(MacroAssembler& masm, Label* bailoutTail);
    258  void generateInvalidator(MacroAssembler& masm, Label* bailoutTail);
    259  uint32_t generatePreBarrier(JSContext* cx, MacroAssembler& masm,
    260                              MIRType type);
    261  void generateIonGenericCallStub(MacroAssembler& masm,
    262                                  IonGenericCallKind kind);
    263 
    264  // Helper functions for generateIonGenericCallStub
    265  void generateIonGenericCallBoundFunction(MacroAssembler& masm, Label* entry,
    266                                           Label* vmCall);
    267  void generateIonGenericCallNativeFunction(MacroAssembler& masm,
    268                                            bool isConstructing);
    269  void generateIonGenericCallFunCall(MacroAssembler& masm, Label* entry,
    270                                     Label* vmCall);
    271  void generateIonGenericCallArgumentsShift(MacroAssembler& masm, Register argc,
    272                                            Register curr, Register end,
    273                                            Register scratch, Label* done);
    274  void generateIonGenericHandleUnderflow(MacroAssembler& masm,
    275                                         bool isConstructing, Label* vmCall);
    276 
    277  JitCode* generateDebugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind);
    278 
    279  bool generateVMWrapper(JSContext* cx, MacroAssembler& masm, VMFunctionId id,
    280                         const VMFunctionData& f, DynFn nativeFun,
    281                         uint32_t* wrapperOffset);
    282 
    283  bool generateVMWrappers(JSContext* cx, MacroAssembler& masm,
    284                          PerfSpewerRangeRecorder& rangeRecorder);
    285 
    286  uint32_t startTrampolineCode(MacroAssembler& masm);
    287 
    288  TrampolinePtr trampolineCode(uint32_t offset) const {
    289    MOZ_ASSERT(offset > 0);
    290    MOZ_ASSERT(offset < trampolineCode_->instructionsSize());
    291    return TrampolinePtr(trampolineCode_->raw() + offset);
    292  }
    293 
    294  void generateBaselineInterpreterEntryTrampoline(MacroAssembler& masm);
    295  void generateInterpreterEntryTrampoline(MacroAssembler& masm);
    296 
    297  using TrampolineNativeJitEntryOffsets =
    298      mozilla::EnumeratedArray<TrampolineNative, uint32_t,
    299                               size_t(TrampolineNative::Count)>;
    300  void generateTrampolineNatives(MacroAssembler& masm,
    301                                 TrampolineNativeJitEntryOffsets& offsets,
    302                                 PerfSpewerRangeRecorder& rangeRecorder);
    303  uint32_t generateArraySortTrampoline(MacroAssembler& masm,
    304                                       ArraySortKind kind);
    305 
    306  void bindLabelToOffset(Label* label, uint32_t offset) {
    307    MOZ_ASSERT(!trampolineCode_);
    308    label->bind(offset);
    309  }
    310 
    311 public:
    312  JitCode* generateEntryTrampolineForScript(JSContext* cx, JSScript* script);
    313 
    314  JitRuntime() = default;
    315  ~JitRuntime();
    316  [[nodiscard]] bool initialize(JSContext* cx);
    317 
    318  static void TraceAtomZoneRoots(JSTracer* trc);
    319  [[nodiscard]] static bool MarkJitcodeGlobalTableIteratively(GCMarker* marker);
    320  static void TraceWeakJitcodeGlobalTable(JSRuntime* rt, JSTracer* trc);
    321 
    322  const BaselineICFallbackCode& baselineICFallbackCode() const {
    323    return baselineICFallbackCode_.ref();
    324  }
    325 
    326  IonCompilationId nextCompilationId() {
    327    return IonCompilationId(nextCompilationId_++);
    328  }
    329 
    330  [[nodiscard]] bool addIonCompileToFreeTaskBatch(IonCompileTask* task) {
    331    return ionFreeTaskBatch_.ref().append(task);
    332  }
    333  void maybeStartIonFreeTask(bool force);
    334 
    335  UniquePtr<LifoAlloc> tryReuseIonLifoAlloc();
    336 
    337 #ifdef DEBUG
    338  bool disallowArbitraryCode() const { return disallowArbitraryCode_; }
    339  void clearDisallowArbitraryCode() { disallowArbitraryCode_ = false; }
    340  const void* addressOfDisallowArbitraryCode() const {
    341    return &disallowArbitraryCode_.refNoCheck();
    342  }
    343  static size_t offsetOfDisallowArbitraryCode() {
    344    return offsetof(JitRuntime, disallowArbitraryCode_);
    345  }
    346 #endif
    347 
    348  uint8_t* allocateIonOsrTempData(size_t size);
    349  void freeIonOsrTempData();
    350 
    351  TrampolinePtr getVMWrapper(VMFunctionId funId) const {
    352    MOZ_ASSERT(trampolineCode_);
    353    return trampolineCode(functionWrapperOffsets_[size_t(funId)]);
    354  }
    355 
    356  bool ensureDebugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind);
    357  JitCode* debugTrapHandler(DebugTrapHandlerKind kind) const {
    358    MOZ_ASSERT(debugTrapHandlers_[kind]);
    359    return debugTrapHandlers_[kind];
    360  }
    361 
    362  BaselineInterpreter& baselineInterpreter() { return baselineInterpreter_; }
    363  const BaselineInterpreter& baselineInterpreter() const {
    364    return baselineInterpreter_;
    365  }
    366 
    367  TrampolinePtr getGenericBailoutHandler() const {
    368    return trampolineCode(bailoutHandlerOffset_);
    369  }
    370 
    371  TrampolinePtr getExceptionTail() const {
    372    return trampolineCode(exceptionTailOffset_);
    373  }
    374  TrampolinePtr getExceptionTailReturnValueCheck() const {
    375    return trampolineCode(exceptionTailReturnValueCheckOffset_);
    376  }
    377 
    378  TrampolinePtr getProfilerExitFrameTail() const {
    379    return trampolineCode(profilerExitFrameTailOffset_);
    380  }
    381 
    382  uint32_t vmInterpreterEntryOffset() { return vmInterpreterEntryOffset_; }
    383 
    384  TrampolinePtr getInvalidationThunk() const {
    385    return trampolineCode(invalidatorOffset_);
    386  }
    387 
    388  EnterJitCode enterJit() const {
    389    return JS_DATA_TO_FUNC_PTR(EnterJitCode,
    390                               trampolineCode(enterJITOffset_).value);
    391  }
    392 
    393  // Return the registers from the native caller frame of the given JIT frame.
    394  // Nothing{} if frameStackAddress is NOT pointing at a native-to-JIT entry
    395  // frame, or if the information is not accessible/implemented on this
    396  // platform.
    397  static mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
    398  getCppEntryRegisters(JitFrameLayout* frameStackAddress);
    399 
    400  TrampolinePtr preBarrier(MIRType type) const {
    401    switch (type) {
    402      case MIRType::Value:
    403        return trampolineCode(valuePreBarrierOffset_);
    404      case MIRType::String:
    405        return trampolineCode(stringPreBarrierOffset_);
    406      case MIRType::Object:
    407        return trampolineCode(objectPreBarrierOffset_);
    408      case MIRType::Shape:
    409        return trampolineCode(shapePreBarrierOffset_);
    410      case MIRType::WasmAnyRef:
    411        return trampolineCode(wasmAnyRefPreBarrierOffset_);
    412      default:
    413        MOZ_CRASH();
    414    }
    415  }
    416 
    417  TrampolinePtr lazyLinkStub() const {
    418    return trampolineCode(lazyLinkStubOffset_);
    419  }
    420  TrampolinePtr interpreterStub() const {
    421    return trampolineCode(interpreterStubOffset_);
    422  }
    423 
    424  TrampolinePtr getDoubleToInt32ValueStub() const {
    425    return trampolineCode(doubleToInt32ValueStubOffset_);
    426  }
    427 
    428  TrampolinePtr getIonGenericCallStub(IonGenericCallKind kind) const {
    429    return trampolineCode(ionGenericCallStubOffset_[kind]);
    430  }
    431 
    432  void** trampolineNativeJitEntry(TrampolineNative native) {
    433    void** jitEntry = &trampolineNativeJitEntries_[native];
    434    MOZ_ASSERT(*jitEntry >= trampolineCode_->raw());
    435    MOZ_ASSERT(*jitEntry <
    436               trampolineCode_->raw() + trampolineCode_->instructionsSize());
    437    return jitEntry;
    438  }
    439  TrampolineNative trampolineNativeForJitEntry(void** entry) {
    440    MOZ_RELEASE_ASSERT(entry >= trampolineNativeJitEntries_.begin());
    441    size_t index = entry - trampolineNativeJitEntries_.begin();
    442    MOZ_RELEASE_ASSERT(index < size_t(TrampolineNative::Count));
    443    return TrampolineNative(index);
    444  }
    445 
    446  bool hasJitcodeGlobalTable() const { return jitcodeGlobalTable_ != nullptr; }
    447 
    448  JitcodeGlobalTable* getJitcodeGlobalTable() {
    449    MOZ_ASSERT(hasJitcodeGlobalTable());
    450    return jitcodeGlobalTable_;
    451  }
    452 
    453  bool hasJitHintsMap() const { return jitHintsMap_ != nullptr; }
    454 
    455  JitHintsMap* getJitHintsMap() {
    456    MOZ_ASSERT(hasJitHintsMap());
    457    return jitHintsMap_;
    458  }
    459 
    460  bool hasInterpreterEntryMap() const {
    461    return interpreterEntryMap_ != nullptr;
    462  }
    463 
    464  EntryTrampolineMap* getInterpreterEntryMap() {
    465    MOZ_ASSERT(hasInterpreterEntryMap());
    466    return interpreterEntryMap_;
    467  }
    468 
    469  bool isProfilerInstrumentationEnabled(JSRuntime* rt) {
    470    return rt->geckoProfiler().enabled();
    471  }
    472 
    473  bool isOptimizationTrackingEnabled(JSRuntime* rt) {
    474    return isProfilerInstrumentationEnabled(rt);
    475  }
    476 
    477 #ifdef DEBUG
    478  void* addressOfIonBailAfterCounter() { return &ionBailAfterCounter_; }
    479 
    480  // Set after how many bailing places we should forcefully bail.
    481  // Zero disables this feature.
    482  void setIonBailAfterCounter(uint32_t after) { ionBailAfterCounter_ = after; }
    483  bool ionBailAfterEnabled() const { return ionBailAfterEnabled_; }
    484  void setIonBailAfterEnabled(bool enabled) { ionBailAfterEnabled_ = enabled; }
    485 #endif
    486 
    487  size_t numFinishedOffThreadTasks() const {
    488    return numFinishedOffThreadTasks_;
    489  }
    490  NumFinishedOffThreadTasksType& numFinishedOffThreadTasksRef(
    491      const AutoLockHelperThreadState& locked) {
    492    return numFinishedOffThreadTasks_;
    493  }
    494 
    495  IonCompileTaskList& ionLazyLinkList(JSRuntime* rt);
    496 
    497  size_t ionLazyLinkListSize() const { return ionLazyLinkListSize_; }
    498 
    499  void ionLazyLinkListRemove(JSRuntime* rt, js::jit::IonCompileTask* task);
    500  void ionLazyLinkListAdd(JSRuntime* rt, js::jit::IonCompileTask* task);
    501 };
    502 
    503 }  // namespace jit
    504 }  // namespace js
    505 
    506 #endif /* jit_JitRuntime_h */