tor-browser

JitScript.h (21957B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_JitScript_h
#define jit_JitScript_h

#include "mozilla/Assertions.h"
#include "mozilla/Atomics.h"
#include "mozilla/Attributes.h"
#include "mozilla/HashFunctions.h"
#include "mozilla/LinkedList.h"
#include "mozilla/Maybe.h"
#include "mozilla/MemoryReporting.h"

#include <stddef.h>
#include <stdint.h>

#include "jstypes.h"
#include "NamespaceImports.h"

#include "ds/LifoAlloc.h"
#include "gc/Barrier.h"
#include "jit/BaselineIC.h"
#include "js/TypeDecls.h"
#include "js/UniquePtr.h"
#include "js/Vector.h"
#include "util/TrailingArray.h"
#include "vm/EnvironmentObject.h"

class JS_PUBLIC_API JSScript;
class JS_PUBLIC_API JSTracer;
struct JS_PUBLIC_API JSContext;

namespace JS {
class Zone;
}

namespace js {

class SystemAllocPolicy;

namespace gc {
class AllocSite;
}

namespace jit {

class BaselineScript;
class ICStubSpace;
class InliningRoot;
class IonScript;
class JitScript;
class JitZone;

// Magic values indicating compilation has been disabled or the script
// is already scheduled for background compilation.
static constexpr uintptr_t DisabledScript = 0x1;
static constexpr uintptr_t QueuedScript = 0x3;
static constexpr uintptr_t CompilingScript = 0x5;

static constexpr uint32_t SpecialScriptBit = 0x1;
static_assert((DisabledScript & SpecialScriptBit) != 0);
static_assert((QueuedScript & SpecialScriptBit) != 0);
static_assert((CompilingScript & SpecialScriptBit) != 0);

static BaselineScript* const BaselineDisabledScriptPtr =
   reinterpret_cast<BaselineScript*>(DisabledScript);
static BaselineScript* const BaselineQueuedScriptPtr =
   reinterpret_cast<BaselineScript*>(QueuedScript);
static BaselineScript* const BaselineCompilingScriptPtr =
   reinterpret_cast<BaselineScript*>(CompilingScript);

static IonScript* const IonDisabledScriptPtr =
   reinterpret_cast<IonScript*>(DisabledScript);
static IonScript* const IonCompilingScriptPtr =
   reinterpret_cast<IonScript*>(CompilingScript);

/* [SMDOC] ICScript Lifetimes
*
* An ICScript owns an array of ICEntries, each of which owns a linked
* list of ICStubs.
*
* A JitScript contains an embedded ICScript. If it has done any trial
* inlining, it also owns an InliningRoot. The InliningRoot owns all
* of the ICScripts that have been created for inlining into the
* corresponding JitScript. This ties the lifetime of the inlined
* ICScripts to the lifetime of the JitScript itself.
*
* We store pointers to ICScripts in two other places: on the stack in
* BaselineFrame, and in IC stubs for CallInlinedFunction.
*
* The ICScript pointer in a BaselineFrame either points to the
* ICScript embedded in the JitScript for that frame, or to an inlined
* ICScript owned by a caller. In each case, there must be a frame on
* the stack corresponding to the JitScript that owns the current
* ICScript, which will keep the ICScript alive.
*
* Each ICStub is owned by an ICScript and, indirectly, a
* JitScript. An ICStub that uses CallInlinedFunction contains an
* ICScript for use by the callee. The ICStub and the callee ICScript
* are always owned by the same JitScript, so the callee ICScript will
* not be freed while the ICStub is alive.
*
* The lifetime of an ICScript is independent of the lifetimes of the
* BaselineScript and IonScript/WarpScript to which it
* corresponds. They can be destroyed and recreated, and the ICScript
* will remain valid.
*
* When we discard JIT code, we mark ICScripts that are active on the stack as
* active and then purge all of the inactive ICScripts. We also purge ICStubs,
* including the CallInlinedFunction stub at the trial inining call site, and
* reset the ICStates to allow trial inlining again later.
*
* If there's a BaselineFrame for an inlined ICScript, we'll preserve both this
* ICScript and the IC chain for the call site in the caller's ICScript.
* See ICScript::purgeStubs and ICScript::purgeInactiveICScripts.
*/

class alignas(uintptr_t) ICScript final : public TrailingArray<ICScript> {
public:
 ICScript(uint32_t warmUpCount, Offset fallbackStubsOffset, Offset endOffset,
          uint32_t depth, uint32_t bytecodeSize,
          InliningRoot* inliningRoot = nullptr)
     : inliningRoot_(inliningRoot),
       warmUpCount_(warmUpCount),
       fallbackStubsOffset_(fallbackStubsOffset),
       endOffset_(endOffset),
       depth_(depth),
       bytecodeSize_(bytecodeSize) {}

 ~ICScript();

 bool isInlined() const { return depth_ > 0; }

 void initICEntries(JSContext* cx, JSScript* script);

 ICEntry& icEntry(size_t index) {
   MOZ_ASSERT(index < numICEntries());
   return icEntries()[index];
 }

 ICFallbackStub* fallbackStub(size_t index) {
   MOZ_ASSERT(index < numICEntries());
   return fallbackStubs() + index;
 }

 ICEntry* icEntryForStub(const ICFallbackStub* stub) {
   size_t index = stub - fallbackStubs();
   MOZ_ASSERT(index < numICEntries());
   return &icEntry(index);
 }
 ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
   size_t index = entry - icEntries();
   MOZ_ASSERT(index < numICEntries());
   return fallbackStub(index);
 }

 InliningRoot* inliningRoot() const { return inliningRoot_; }
 uint32_t depth() const { return depth_; }

 uint32_t bytecodeSize() const { return bytecodeSize_; }

 void resetWarmUpCount(uint32_t count) { warmUpCount_ = count; }

 static constexpr size_t offsetOfFirstStub(uint32_t entryIndex) {
   return sizeof(ICScript) + entryIndex * sizeof(ICEntry) +
          ICEntry::offsetOfFirstStub();
 }

 static constexpr Offset offsetOfWarmUpCount() {
   return offsetof(ICScript, warmUpCount_);
 }
 static constexpr Offset offsetOfDepth() { return offsetof(ICScript, depth_); }

 static constexpr Offset offsetOfICEntries() { return sizeof(ICScript); }
 uint32_t numICEntries() const {
   return numElements<ICEntry>(icEntriesOffset(), fallbackStubsOffset());
 }

 static constexpr Offset offsetOfEnvAllocSite() {
   return offsetof(ICScript, envAllocSite_);
 }

 ICEntry* interpreterICEntryFromPCOffset(uint32_t pcOffset);

 ICEntry& icEntryFromPCOffset(uint32_t pcOffset);

 [[nodiscard]] bool addInlinedChild(JSContext* cx,
                                    js::UniquePtr<ICScript> child,
                                    uint32_t pcOffset);
 ICScript* findInlinedChild(uint32_t pcOffset);
 void removeInlinedChild(uint32_t pcOffset);
 bool hasInlinedChild(uint32_t pcOffset);

 void purgeStubs(Zone* zone, ICStubSpace& newStubSpace);

 void purgeInactiveICScripts();

 bool active() const { return active_; }
 void setActive() { active_ = true; }
 void resetActive() { active_ = false; }

 gc::AllocSite* getOrCreateAllocSite(JSScript* outerScript, uint32_t pcOffset);

 void ensureEnvAllocSite(JSScript* outerScript);
 gc::AllocSite* maybeEnvAllocSite() const { return envAllocSite_; }

 void prepareForDestruction(Zone* zone);

 void trace(JSTracer* trc);
 bool traceWeak(JSTracer* trc);

#ifdef DEBUG
 mozilla::HashNumber hash(JSContext* cx);
#endif

private:
 class CallSite {
  public:
   CallSite(ICScript* callee, uint32_t pcOffset)
       : callee_(callee), pcOffset_(pcOffset) {}
   ICScript* callee_;
   uint32_t pcOffset_;
 };

 // If this ICScript was created for trial inlining or has another
 // ICScript inlined into it, a pointer to the root of the inlining
 // tree. Otherwise, nullptr.
 InliningRoot* inliningRoot_ = nullptr;

 // ICScripts that have been inlined into this ICScript.
 js::UniquePtr<Vector<CallSite>> inlinedChildren_;

 // List of allocation sites referred to by ICs in this script.
 static constexpr size_t AllocSiteChunkSize = 256;
 LifoAlloc allocSitesSpace_{AllocSiteChunkSize, js::BackgroundMallocArena};
 Vector<gc::AllocSite*, 0, SystemAllocPolicy> allocSites_;

 // Optional alloc site to use when allocating environment chain objects.
 gc::AllocSite* envAllocSite_ = nullptr;

 // Number of times this copy of the script has been called or has had
 // backedges taken.  Reset if the script's JIT code is forcibly discarded.
 // See also the ScriptWarmUpData class.
 mozilla::Atomic<uint32_t, mozilla::Relaxed> warmUpCount_ = {};

 // The offset of the ICFallbackStub array.
 Offset fallbackStubsOffset_;

 // The size of this allocation.
 Offset endOffset_;

 // The inlining depth of this ICScript. 0 for the inlining root.
 uint32_t depth_;

 // Bytecode size of the JSScript corresponding to this ICScript.
 uint32_t bytecodeSize_;

 // Flag set when discarding JIT code to indicate this script is on the stack
 // and should not be discarded.
 bool active_ = false;

 Offset icEntriesOffset() const { return offsetOfICEntries(); }
 Offset fallbackStubsOffset() const { return fallbackStubsOffset_; }
 Offset endOffset() const { return endOffset_; }

public:
 ICEntry* icEntries() { return offsetToPointer<ICEntry>(icEntriesOffset()); }

private:
 ICFallbackStub* fallbackStubs() {
   return offsetToPointer<ICFallbackStub>(fallbackStubsOffset());
 }

 JitScript* outerJitScript();

 friend class JitScript;
};

// [SMDOC] JitScript
//
// JitScript stores type inference data, Baseline ICs and other JIT-related data
// for a script. Scripts with a JitScript can run in the Baseline Interpreter.
//
// IC Data
// =======
// All IC data for Baseline (Interpreter and JIT) is stored in an ICScript. Each
// JitScript contains an ICScript as the last field. Additional free-standing
// ICScripts may be created during trial inlining. Ion has its own IC chains
// stored in IonScript.
//
// For each IC we store an ICEntry, which points to the first ICStub in the
// chain, and an ICFallbackStub. Note that multiple stubs in the same zone can
// share Baseline IC code. This works because the stub data is stored in the
// ICStub instead of baked in in the stub code.
//
// Storing this separate from BaselineScript allows us to use the same ICs in
// the Baseline Interpreter and Baseline JIT. It also simplifies debug mode OSR
// because the JitScript can be reused when we have to recompile the
// BaselineScript.
//
// An ICScript contains a list of IC entries and a list of fallback stubs.
// There's one ICEntry and ICFallbackStub for each JOF_IC bytecode op.
//
// The ICScript also contains the warmUpCount for the script.
//
// Inlining Data
// =============
// JitScript also contains a list of Warp compilations inlining this script, for
// invalidation.
//
// Memory Layout
// =============
// JitScript contains an ICScript as the last field. ICScript has trailing
// (variable length) arrays for ICEntry and ICFallbackStub. The memory layout is
// as follows:
//
//  Item                    | Offset
//  ------------------------+------------------------
//  JitScript               | 0
//  -->ICScript  (field)    |
//     ICEntry[]            | icEntriesOffset()
//     ICFallbackStub[]     | fallbackStubsOffset()
//
// These offsets are also used to compute numICEntries.
class alignas(uintptr_t) JitScript final
   : public mozilla::LinkedListElement<JitScript>,
     public TrailingArray<JitScript> {
 friend class ::JSScript;

 // Profile string used by the profiler for Baseline Interpreter frames.
 const char* profileString_ = nullptr;

 HeapPtr<JSScript*> owningScript_;

 // Baseline code for the script. Either nullptr, BaselineDisabledScriptPtr,
 // BaselineQueuedScriptPtr, BaselineCompilingScriptPtr,
 // or a valid BaselineScript*.
 GCStructPtr<BaselineScript*> baselineScript_;

 // Ion code for this script. Either nullptr, IonDisabledScriptPtr,
 // IonCompilingScriptPtr or a valid IonScript*.
 GCStructPtr<IonScript*> ionScript_;

 // For functions that need a CallObject and/or NamedLambdaObject, the template
 // objects used by the Baseline JIT and Ion. If the function needs both a
 // named lambda object and a call object, the named lambda object template is
 // linked via the call object's enclosing environment. This field is set the
 // first time the Baseline JIT compiles this script.
 mozilla::Maybe<HeapPtr<EnvironmentObject*>> templateEnv_;

 // The size of this allocation.
 Offset endOffset_ = 0;

 // Analysis data computed lazily the first time this script is compiled or
 // inlined by WarpBuilder.
 mozilla::Maybe<bool> usesEnvironmentChain_;

 struct Flags {
   // True if this script entered Ion via OSR at a loop header.
   bool hadIonOSR : 1;
   bool ranBytecodeAnalysis : 1;
 };
 Flags flags_ = {};  // Zero-initialize flags.

 js::UniquePtr<InliningRoot> inliningRoot_;

#ifdef DEBUG
 // If the last warp compilation invalidated because of TranspiledCacheIR
 // bailouts, this is a hash of the ICScripts used in that compilation.
 // When recompiling, we assert that the hash has changed.
 mozilla::Maybe<mozilla::HashNumber> failedICHash_;

 // To avoid pathological cases, we skip the check if we have purged
 // stubs due to GC pressure.
 bool hasPurgedStubs_ = false;
#endif

 // Value of the warmup counter when the last IC stub was attached,
 // used for Ion hints.
 uint32_t warmUpCountAtLastICStub_ = 0;

 ICScript icScript_;
 // End of fields.

 Offset endOffset() const { return endOffset_; }

public:
 JitScript(JSScript* script, Offset fallbackStubsOffset, Offset endOffset,
           const char* profileString);

 ~JitScript();

 JSScript* owningScript() const { return owningScript_; }

 [[nodiscard]] bool ensureHasCachedBaselineJitData(JSContext* cx,
                                                   HandleScript script);
 [[nodiscard]] bool ensureHasCachedIonData(JSContext* cx, HandleScript script);

 void setHadIonOSR() { flags_.hadIonOSR = true; }
 bool hadIonOSR() const { return flags_.hadIonOSR; }

 void setRanBytecodeAnalysis() { flags_.ranBytecodeAnalysis = true; }
 bool ranBytecodeAnalysis() const { return flags_.ranBytecodeAnalysis; }

 uint32_t numICEntries() const { return icScript_.numICEntries(); }

#ifdef DEBUG
 bool hasActiveICScript() const;
#endif
 void resetAllActiveFlags();

 void ensureProfileString(JSContext* cx, JSScript* script);
 void ensureProfilerScriptSource(JSContext* cx, JSScript* script);

 const char* profileString() const {
   MOZ_ASSERT(profileString_);
   return profileString_;
 }

 static void Destroy(Zone* zone, JitScript* script);

 static constexpr Offset offsetOfICEntries() { return sizeof(JitScript); }

 static constexpr size_t offsetOfBaselineScript() {
   return offsetof(JitScript, baselineScript_);
 }
 static constexpr size_t offsetOfIonScript() {
   return offsetof(JitScript, ionScript_);
 }
 static constexpr size_t offsetOfICScript() {
   return offsetof(JitScript, icScript_);
 }
 static constexpr size_t offsetOfWarmUpCount() {
   return offsetOfICScript() + ICScript::offsetOfWarmUpCount();
 }

 uint32_t warmUpCount() const { return icScript_.warmUpCount_; }
 void incWarmUpCount() { icScript_.warmUpCount_++; }
 void resetWarmUpCount(uint32_t count);

 void prepareForDestruction(Zone* zone);

 void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, size_t* data,
                             size_t* allocSites) const;

 ICEntry& icEntry(size_t index) { return icScript_.icEntry(index); }

 ICFallbackStub* fallbackStub(size_t index) {
   return icScript_.fallbackStub(index);
 }

 ICEntry* icEntryForStub(const ICFallbackStub* stub) {
   return icScript_.icEntryForStub(stub);
 }
 ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
   return icScript_.fallbackStubForICEntry(entry);
 }

 void trace(JSTracer* trc);
 void traceWeak(JSTracer* trc);
 void purgeStubs(JSScript* script, ICStubSpace& newStubSpace);

 void purgeInactiveICScripts();

 ICEntry& icEntryFromPCOffset(uint32_t pcOffset) {
   return icScript_.icEntryFromPCOffset(pcOffset);
 };

 size_t allocBytes() const { return endOffset(); }

 EnvironmentObject* templateEnvironment() const { return templateEnv_.ref(); }

 std::pair<CallObject*, NamedLambdaObject*> functionEnvironmentTemplates(
     JSFunction* fun) const;

 bool usesEnvironmentChain() const { return *usesEnvironmentChain_; }

 bool resetAllocSites(bool resetNurserySites, bool resetPretenuredSites);
 bool hasPretenuredAllocSites();

 void updateLastICStubCounter() { warmUpCountAtLastICStub_ = warmUpCount(); }
 uint32_t warmUpCountAtLastICStub() const { return warmUpCountAtLastICStub_; }

 bool hasEnvAllocSite() const { return icScript_.envAllocSite_; }

private:
 // Methods to set baselineScript_ to a BaselineScript*, nullptr, or
 // BaselineDisabledScriptPtr.
 void setBaselineScriptImpl(JSScript* script, BaselineScript* baselineScript);
 void setBaselineScriptImpl(JS::GCContext* gcx, JSScript* script,
                            BaselineScript* baselineScript);

public:
 // Methods for getting/setting/clearing a BaselineScript*.
 bool hasBaselineScript() const {
   bool res = baselineScript_ &&
              baselineScript_ != BaselineDisabledScriptPtr &&
              baselineScript_ != BaselineQueuedScriptPtr &&
              baselineScript_ != BaselineCompilingScriptPtr;
   MOZ_ASSERT_IF(!res, !hasIonScript());
   return res;
 }
 BaselineScript* baselineScript() const {
   MOZ_ASSERT(hasBaselineScript());
   return baselineScript_;
 }
 void setBaselineScript(JSScript* script, BaselineScript* baselineScript) {
   MOZ_ASSERT(!hasBaselineScript());
   setBaselineScriptImpl(script, baselineScript);
   MOZ_ASSERT(hasBaselineScript());
 }
 [[nodiscard]] BaselineScript* clearBaselineScript(JS::GCContext* gcx,
                                                   JSScript* script) {
   BaselineScript* baseline = baselineScript();
   setBaselineScriptImpl(gcx, script, nullptr);
   return baseline;
 }
 bool isBaselineQueued() const {
   return baselineScript_ == BaselineQueuedScriptPtr;
 }
 void clearIsBaselineQueued(JSScript* script) {
   MOZ_ASSERT(isBaselineQueued());
   setBaselineScriptImpl(script, nullptr);
 }
 bool isBaselineCompiling() const {
   return baselineScript_ == BaselineCompilingScriptPtr;
 }
 void setIsBaselineCompiling(JSScript* script) {
   MOZ_ASSERT(baselineScript_ == nullptr);
   setBaselineScriptImpl(script, BaselineCompilingScriptPtr);
 }
 void clearIsBaselineCompiling(JSScript* script) {
   MOZ_ASSERT(isBaselineCompiling());
   setBaselineScriptImpl(script, nullptr);
 }

private:
 // Methods to set ionScript_ to an IonScript*, nullptr, or one of the special
 // Ion{Disabled,Compiling}ScriptPtr values.
 void setIonScriptImpl(JS::GCContext* gcx, JSScript* script,
                       IonScript* ionScript);
 void setIonScriptImpl(JSScript* script, IonScript* ionScript);

 // Helper that calls the passed function for the outer ICScript and for each
 // inlined ICScript.
 template <typename F>
 void forEachICScript(const F& f);
 template <typename F>
 void forEachICScript(const F& f) const;

public:
 // Methods for getting/setting/clearing an IonScript*.
 bool hasIonScript() const {
   bool res = ionScript_ && ionScript_ != IonDisabledScriptPtr &&
              ionScript_ != IonCompilingScriptPtr;
   MOZ_ASSERT_IF(res, baselineScript_);
   return res;
 }
 IonScript* ionScript() const {
   MOZ_ASSERT(hasIonScript());
   return ionScript_;
 }
 void setIonScript(JSScript* script, IonScript* ionScript) {
   MOZ_ASSERT(!hasIonScript());
   setIonScriptImpl(script, ionScript);
   MOZ_ASSERT(hasIonScript());
 }
 [[nodiscard]] IonScript* clearIonScript(JS::GCContext* gcx,
                                         JSScript* script) {
   IonScript* ion = ionScript();
   setIonScriptImpl(gcx, script, nullptr);
   return ion;
 }

 // Methods for off-thread compilation.
 bool isIonCompilingOffThread() const {
   return ionScript_ == IonCompilingScriptPtr;
 }
 void setIsIonCompilingOffThread(JSScript* script) {
   MOZ_ASSERT(ionScript_ == nullptr);
   setIonScriptImpl(script, IonCompilingScriptPtr);
 }
 void clearIsIonCompilingOffThread(JSScript* script) {
   MOZ_ASSERT(isIonCompilingOffThread());
   setIonScriptImpl(script, nullptr);
 }
 ICScript* icScript() { return &icScript_; }

 bool hasInliningRoot() const { return !!inliningRoot_; }
 InliningRoot* inliningRoot() const { return inliningRoot_.get(); }
 InliningRoot* getOrCreateInliningRoot(JSContext* cx, JSScript* script);

 inline void notePurgedStubs() {
#ifdef DEBUG
   failedICHash_.reset();
   hasPurgedStubs_ = true;
#endif
 }

#ifdef DEBUG
 bool hasPurgedStubs() const { return hasPurgedStubs_; }
 bool hasFailedICHash() const { return failedICHash_.isSome(); }
 mozilla::HashNumber getFailedICHash() { return failedICHash_.extract(); }
 void setFailedICHash(mozilla::HashNumber hash) {
   MOZ_ASSERT(failedICHash_.isNothing());
   if (!hasPurgedStubs_) {
     failedICHash_.emplace(hash);
   }
 }
#endif

 inline void clearFailedICHash() {
#ifdef DEBUG
   failedICHash_.reset();
#endif
 }
};

// Ensures no JitScripts are purged in the current zone.
class MOZ_RAII AutoKeepJitScripts {
 jit::JitZone* zone_;
 bool prev_;

 AutoKeepJitScripts(const AutoKeepJitScripts&) = delete;
 void operator=(const AutoKeepJitScripts&) = delete;

public:
 explicit inline AutoKeepJitScripts(JSContext* cx);
 inline ~AutoKeepJitScripts();
};

// Mark ICScripts on the stack as active, so that they are not discarded
// during GC, and copy active Baseline IC stubs to the new stub space.
void MarkActiveICScriptsAndCopyStubs(Zone* zone, ICStubSpace& newStubSpace);

#ifdef JS_STRUCTURED_SPEW
void JitSpewBaselineICStats(JSScript* script, const char* dumpReason);
#endif

}  // namespace jit
}  // namespace js

#endif /* jit_JitScript_h */