[ tor-browser ].git.dasho

TrialInlining.cpp (35676B)
      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
      2 * vim: set ts=8 sts=2 et sw=2 tw=80:
      3 * This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "jit/TrialInlining.h"
      8 
      9 #include "mozilla/DebugOnly.h"
     10 
     11 #include "jit/BaselineCacheIRCompiler.h"
     12 #include "jit/BaselineFrame.h"
     13 #include "jit/BaselineIC.h"
     14 #include "jit/BytecodeAnalysis.h"
     15 #include "jit/CacheIRCloner.h"
     16 #include "jit/CacheIRHealth.h"
     17 #include "jit/CacheIRWriter.h"
     18 #include "jit/InlineScriptTree.h"
     19 #include "jit/Ion.h"  // TooManyFormalArguments
     20 #include "jit/StubFolding.h"
     21 
     22 #include "vm/BytecodeLocation-inl.h"
     23 
     24 using mozilla::Maybe;
     25 
     26 namespace js {
     27 namespace jit {
     28 
     29 bool DoTrialInlining(JSContext* cx, BaselineFrame* frame) {
     30  RootedScript script(cx, frame->script());
     31  ICScript* icScript = frame->icScript();
     32  bool isRecursive = icScript->depth() > 0;
     33 
     34 #ifdef JS_CACHEIR_SPEW
     35  if (cx->spewer().enabled(cx, script, SpewChannel::CacheIRHealthReport)) {
     36    for (uint32_t i = 0; i < icScript->numICEntries(); i++) {
     37      ICEntry& entry = icScript->icEntry(i);
     38      ICFallbackStub* fallbackStub = icScript->fallbackStub(i);
     39 
     40      // If the IC is megamorphic or generic, then we have already
     41      // spewed the IC report on transition.
     42      if (!(uint8_t(fallbackStub->state().mode()) > 0)) {
     43        jit::ICStub* stub = entry.firstStub();
     44        bool sawNonZeroCount = false;
     45        while (!stub->isFallback()) {
     46          uint32_t count = stub->enteredCount();
     47          if (count > 0 && sawNonZeroCount) {
     48            CacheIRHealth cih;
     49            cih.healthReportForIC(cx, &entry, fallbackStub, script,
     50                                  SpewContext::TrialInlining);
     51            break;
     52          }
     53 
     54          if (count > 0 && !sawNonZeroCount) {
     55            sawNonZeroCount = true;
     56          }
     57 
     58          stub = stub->toCacheIRStub()->next();
     59        }
     60      }
     61    }
     62  }
     63 #endif
     64 
     65  if (!script->canIonCompile()) {
     66    return true;
     67  }
     68 
     69  // Don't do trial inlining in scripts that are too large.
     70  if (JitOptions.limitScriptSize &&
     71      script->length() > JitOptions.ionMaxScriptSize) {
     72    // Baseline should elide trial inlining calls if the script is big.
     73    MOZ_ASSERT(frame->runningInInterpreter());
     74    return true;
     75  }
     76 
     77  // Maximum trial inlining depth for ICScripts. This is smaller than
     78  // InlineScriptTree::MaxDepth because we can reach the bigger number if we are
     79  // doing monomorphic inlining using shared IC data.
     80  const uint32_t MaxICScriptDepth = 4;
     81  static_assert(MaxICScriptDepth <= InlineScriptTree::MaxDepth,
     82                "Trial inlining depth must not exceed max inlining depth");
     83  if (icScript->depth() > MaxICScriptDepth) {
     84    return true;
     85  }
     86 
     87  InliningRoot* root = isRecursive ? icScript->inliningRoot()
     88                                   : script->jitScript()->inliningRoot();
     89  if (JitSpewEnabled(JitSpew_WarpTrialInlining)) {
     90    // Eagerly create the inlining root when it's used in the spew output.
     91    if (!root) {
     92      MOZ_ASSERT(!isRecursive);
     93      root = script->jitScript()->getOrCreateInliningRoot(cx, script);
     94      if (!root) {
     95        return false;
     96      }
     97    }
     98    UniqueChars funName;
     99    if (script->function() && script->function()->fullDisplayAtom()) {
    100      funName =
    101          AtomToPrintableString(cx, script->function()->fullDisplayAtom());
    102    }
    103 
    104    JitSpew(
    105        JitSpew_WarpTrialInlining,
    106        "Trial inlining for %s script '%s' (%s:%u:%u (%p)) (inliningRoot=%p)",
    107        (isRecursive ? "inner" : "outer"),
    108        funName ? funName.get() : "<unnamed>", script->filename(),
    109        script->lineno(), script->column().oneOriginValue(), frame->script(),
    110        root);
    111    JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
    112  }
    113 
    114  TrialInliner inliner(cx, script, icScript);
    115  return inliner.tryInlining();
    116 }
    117 
    118 void TrialInliner::cloneSharedPrefix(ICCacheIRStub* stub,
    119                                     const uint8_t* endOfPrefix,
    120                                     CacheIRWriter& writer) {
    121  CacheIRReader reader(stub->stubInfo());
    122  CacheIRCloner cloner(stub);
    123  while (reader.currentPosition() < endOfPrefix) {
    124    CacheOp op = reader.readOp();
    125    cloner.cloneOp(op, reader, writer);
    126  }
    127 }
    128 
    129 bool TrialInliner::replaceICStub(ICEntry& entry, ICFallbackStub* fallback,
    130                                 CacheIRWriter& writer, CacheKind kind) {
    131  MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate);
    132 
    133  fallback->discardStubs(cx()->zone(), &entry);
    134 
    135  // Note: AttachBaselineCacheIRStub never throws an exception.
    136  ICAttachResult result = AttachBaselineCacheIRStub(
    137      cx(), writer, kind, script_, icScript_, fallback, "TrialInline");
    138  if (result == ICAttachResult::Attached) {
    139    MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Inlined);
    140    return true;
    141  }
    142 
    143  MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate);
    144  icScript_->removeInlinedChild(fallback->pcOffset());
    145 
    146  if (result == ICAttachResult::OOM) {
    147    ReportOutOfMemory(cx());
    148    return false;
    149  }
    150 
    151  // We failed to attach a new IC stub due to CacheIR size limits. Disable trial
    152  // inlining for this location and return true.
    153  MOZ_ASSERT(result == ICAttachResult::TooLarge);
    154  fallback->setTrialInliningState(TrialInliningState::Failure);
    155  return true;
    156 }
    157 
    158 ICCacheIRStub* TrialInliner::maybeSingleStub(const ICEntry& entry) {
    159  // Look for a single non-fallback stub followed by stubs with entered-count 0.
    160  // Allow one optimized stub before the fallback stub to support the
    161  // CallIRGenerator::emitCalleeGuard optimization where we first try a
    162  // GuardSpecificFunction guard before falling back to GuardFunctionHasScript.
    163  ICStub* stub = entry.firstStub();
    164  if (stub->isFallback()) {
    165    return nullptr;
    166  }
    167  ICStub* next = stub->toCacheIRStub()->next();
    168  if (next->enteredCount() != 0) {
    169    return nullptr;
    170  }
    171 
    172  ICFallbackStub* fallback = nullptr;
    173  if (next->isFallback()) {
    174    fallback = next->toFallbackStub();
    175  } else {
    176    ICStub* nextNext = next->toCacheIRStub()->next();
    177    if (!nextNext->isFallback() || nextNext->enteredCount() != 0) {
    178      return nullptr;
    179    }
    180    fallback = nextNext->toFallbackStub();
    181  }
    182 
    183  if (fallback->trialInliningState() != TrialInliningState::Candidate) {
    184    return nullptr;
    185  }
    186 
    187  return stub->toCacheIRStub();
    188 }
    189 
    190 Maybe<InlinableOpData> FindInlinableOpData(ICCacheIRStub* stub,
    191                                           BytecodeLocation loc) {
    192  if (loc.isInvokeOp()) {
    193    Maybe<InlinableCallData> call = FindInlinableCallData(stub);
    194    if (call.isSome()) {
    195      return call;
    196    }
    197  }
    198  if (loc.isGetPropOp() || loc.isGetElemOp()) {
    199    Maybe<InlinableGetterData> getter = FindInlinableGetterData(stub);
    200    if (getter.isSome()) {
    201      return getter;
    202    }
    203  }
    204  if (loc.isSetPropOp()) {
    205    Maybe<InlinableSetterData> setter = FindInlinableSetterData(stub);
    206    if (setter.isSome()) {
    207      return setter;
    208    }
    209  }
    210  return mozilla::Nothing();
    211 }
    212 
    213 Maybe<InlinableCallData> FindInlinableCallData(ICCacheIRStub* stub) {
    214  Maybe<InlinableCallData> data;
    215 
    216  const CacheIRStubInfo* stubInfo = stub->stubInfo();
    217  const uint8_t* stubData = stub->stubDataStart();
    218 
    219  ObjOperandId calleeGuardOperand;
    220  CallFlags flags;
    221  JSScript* targetScript = nullptr;
    222 
    223  CacheIRReader reader(stubInfo);
    224  while (reader.more()) {
    225    const uint8_t* opStart = reader.currentPosition();
    226 
    227    CacheOp op = reader.readOp();
    228    CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
    229    uint32_t argLength = opInfo.argLength;
    230    mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
    231 
    232    switch (op) {
    233      case CacheOp::GuardSpecificFunction: {
    234        // If we see a guard for a scripted function, remember which
    235        // operand we are guarding.
    236        MOZ_ASSERT(data.isNothing());
    237        auto args = reader.argsForGuardSpecificFunction();
    238        uint32_t targetOffset = args.expectedOffset;
    239        uintptr_t rawFunction =
    240            stubInfo->getStubRawWord(stubData, targetOffset);
    241        JSFunction* function = reinterpret_cast<JSFunction*>(rawFunction);
    242        if (function->hasBytecode()) {
    243          calleeGuardOperand = args.funId;
    244          targetScript = function->nonLazyScript();
    245        }
    246        break;
    247      }
    248      case CacheOp::GuardFunctionScript: {
    249        MOZ_ASSERT(data.isNothing());
    250        auto args = reader.argsForGuardFunctionScript();
    251        calleeGuardOperand = args.objId;
    252        uint32_t targetOffset = args.expectedOffset;
    253        uintptr_t rawScript = stubInfo->getStubRawWord(stubData, targetOffset);
    254        targetScript = reinterpret_cast<JSScript*>(rawScript);
    255        break;
    256      }
    257      case CacheOp::CallScriptedFunction: {
    258        // If we see a call, check if `callee` is the previously guarded
    259        // operand. If it is, we know the target and can inline.
    260        auto args = reader.argsForCallScriptedFunction();
    261        flags = args.flags;
    262        MOZ_ASSERT(args.argcFixed <= MaxUnrolledArgCopy);
    263 
    264        if (args.calleeId == calleeGuardOperand) {
    265          MOZ_ASSERT(args.argcId.id() == 0);
    266          MOZ_ASSERT(data.isNothing());
    267          data.emplace();
    268          data->endOfSharedPrefix = opStart;
    269        }
    270        break;
    271      }
    272      case CacheOp::CallInlinedFunction: {
    273        auto args = reader.argsForCallInlinedFunction();
    274        flags = args.flags;
    275        MOZ_ASSERT(args.argcFixed <= MaxUnrolledArgCopy);
    276 
    277        if (args.calleeId == calleeGuardOperand) {
    278          MOZ_ASSERT(args.argcId.id() == 0);
    279          MOZ_ASSERT(data.isNothing());
    280          data.emplace();
    281          data->endOfSharedPrefix = opStart;
    282          uintptr_t rawICScript =
    283              stubInfo->getStubRawWord(stubData, args.icScriptOffset);
    284          data->icScript = reinterpret_cast<ICScript*>(rawICScript);
    285        }
    286        break;
    287      }
    288      default:
    289        if (!opInfo.transpile) {
    290          return mozilla::Nothing();
    291        }
    292        if (data.isSome()) {
    293          MOZ_ASSERT(op == CacheOp::ReturnFromIC);
    294        }
    295        reader.skip(argLength);
    296        break;
    297    }
    298    MOZ_ASSERT(argStart + argLength == reader.currentPosition());
    299  }
    300 
    301  if (data.isSome()) {
    302    // Warp only supports inlining Standard and FunCall calls.
    303    if (flags.getArgFormat() != CallFlags::Standard &&
    304        flags.getArgFormat() != CallFlags::FunCall) {
    305      return mozilla::Nothing();
    306    }
    307    data->calleeOperand = calleeGuardOperand;
    308    data->callFlags = flags;
    309    data->target = targetScript;
    310  }
    311  return data;
    312 }
    313 
    314 Maybe<InlinableGetterData> FindInlinableGetterData(ICCacheIRStub* stub) {
    315  Maybe<InlinableGetterData> data;
    316 
    317  const CacheIRStubInfo* stubInfo = stub->stubInfo();
    318  const uint8_t* stubData = stub->stubDataStart();
    319 
    320  ObjOperandId maybeCalleeOperand;
    321  JSScript* targetScript = nullptr;
    322 
    323  CacheIRReader reader(stubInfo);
    324  while (reader.more()) {
    325    const uint8_t* opStart = reader.currentPosition();
    326 
    327    CacheOp op = reader.readOp();
    328    CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
    329    uint32_t argLength = opInfo.argLength;
    330    mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
    331 
    332    switch (op) {
    333      case CacheOp::LoadObject: {
    334        // If we load a constant object, remember it in case it's the callee.
    335        auto [resultOperand, objOffset] = reader.argsForLoadObject();
    336        uintptr_t rawObject = stubInfo->getStubRawWord(stubData, objOffset);
    337        JSObject* object = reinterpret_cast<JSObject*>(rawObject);
    338        if (object->is<JSFunction>() &&
    339            object->as<JSFunction>().hasBytecode()) {
    340          maybeCalleeOperand = resultOperand;
    341          targetScript = object->as<JSFunction>().nonLazyScript();
    342        }
    343        break;
    344      }
    345      case CacheOp::GuardFunctionScript: {
    346        MOZ_ASSERT(data.isNothing());
    347        auto args = reader.argsForGuardFunctionScript();
    348        maybeCalleeOperand = args.objId;
    349        uint32_t targetOffset = args.expectedOffset;
    350        uintptr_t rawScript = stubInfo->getStubRawWord(stubData, targetOffset);
    351        targetScript = reinterpret_cast<JSScript*>(rawScript);
    352        break;
    353      }
    354      case CacheOp::CallScriptedGetterResult: {
    355        auto args = reader.argsForCallScriptedGetterResult();
    356        ObjOperandId calleeOperand = args.calleeId;
    357 
    358        if (maybeCalleeOperand == calleeOperand) {
    359          data.emplace();
    360          data->target = targetScript;
    361          data->receiverOperand = args.receiverId;
    362          data->calleeOperand = calleeOperand;
    363          data->sameRealm = args.sameRealm;
    364          data->endOfSharedPrefix = opStart;
    365        }
    366        break;
    367      }
    368      case CacheOp::CallInlinedGetterResult: {
    369        auto args = reader.argsForCallInlinedGetterResult();
    370        ObjOperandId calleeOperand = args.calleeId;
    371        uintptr_t rawICScript =
    372            stubInfo->getStubRawWord(stubData, args.icScriptOffset);
    373 
    374        if (maybeCalleeOperand == calleeOperand) {
    375          data.emplace();
    376          data->target = targetScript;
    377          data->receiverOperand = args.receiverId;
    378          data->calleeOperand = calleeOperand;
    379          data->icScript = reinterpret_cast<ICScript*>(rawICScript);
    380          data->sameRealm = args.sameRealm;
    381          data->endOfSharedPrefix = opStart;
    382        }
    383        break;
    384      }
    385      default:
    386        if (!opInfo.transpile) {
    387          return mozilla::Nothing();
    388        }
    389        if (data.isSome()) {
    390          MOZ_ASSERT(op == CacheOp::ReturnFromIC);
    391        }
    392        reader.skip(argLength);
    393        break;
    394    }
    395    MOZ_ASSERT(argStart + argLength == reader.currentPosition());
    396  }
    397 
    398  return data;
    399 }
    400 
    401 Maybe<InlinableSetterData> FindInlinableSetterData(ICCacheIRStub* stub) {
    402  Maybe<InlinableSetterData> data;
    403 
    404  const CacheIRStubInfo* stubInfo = stub->stubInfo();
    405  const uint8_t* stubData = stub->stubDataStart();
    406 
    407  ObjOperandId maybeCalleeOperand;
    408  JSScript* targetScript = nullptr;
    409 
    410  CacheIRReader reader(stubInfo);
    411  while (reader.more()) {
    412    const uint8_t* opStart = reader.currentPosition();
    413 
    414    CacheOp op = reader.readOp();
    415    CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
    416    uint32_t argLength = opInfo.argLength;
    417    mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
    418 
    419    switch (op) {
    420      case CacheOp::LoadObject: {
    421        // If we load a constant object, remember it in case it's the callee.
    422        auto [resultOperand, objOffset] = reader.argsForLoadObject();
    423        uintptr_t rawObject = stubInfo->getStubRawWord(stubData, objOffset);
    424        JSObject* object = reinterpret_cast<JSObject*>(rawObject);
    425        if (object->is<JSFunction>() &&
    426            object->as<JSFunction>().hasBytecode()) {
    427          maybeCalleeOperand = resultOperand;
    428          targetScript = object->as<JSFunction>().nonLazyScript();
    429        }
    430        break;
    431      }
    432      case CacheOp::GuardFunctionScript: {
    433        MOZ_ASSERT(data.isNothing());
    434        auto args = reader.argsForGuardFunctionScript();
    435        maybeCalleeOperand = args.objId;
    436        uint32_t targetOffset = args.expectedOffset;
    437        uintptr_t rawScript = stubInfo->getStubRawWord(stubData, targetOffset);
    438        targetScript = reinterpret_cast<JSScript*>(rawScript);
    439        break;
    440      }
    441      case CacheOp::CallScriptedSetter: {
    442        auto args = reader.argsForCallScriptedSetter();
    443        ObjOperandId calleeOperand = args.calleeId;
    444 
    445        if (maybeCalleeOperand == calleeOperand) {
    446          data.emplace();
    447          data->target = targetScript;
    448          data->receiverOperand = args.receiverId;
    449          data->calleeOperand = calleeOperand;
    450          data->rhsOperand = args.rhsId;
    451          data->sameRealm = args.sameRealm;
    452          data->endOfSharedPrefix = opStart;
    453        }
    454        break;
    455      }
    456      case CacheOp::CallInlinedSetter: {
    457        auto args = reader.argsForCallInlinedSetter();
    458        ObjOperandId calleeOperand = args.calleeId;
    459        uintptr_t rawICScript =
    460            stubInfo->getStubRawWord(stubData, args.icScriptOffset);
    461 
    462        if (maybeCalleeOperand == calleeOperand) {
    463          data.emplace();
    464          data->target = targetScript;
    465          data->receiverOperand = args.receiverId;
    466          data->calleeOperand = calleeOperand;
    467          data->rhsOperand = args.rhsId;
    468          data->icScript = reinterpret_cast<ICScript*>(rawICScript);
    469          data->sameRealm = args.sameRealm;
    470          data->endOfSharedPrefix = opStart;
    471        }
    472        break;
    473      }
    474      default:
    475        if (!opInfo.transpile) {
    476          return mozilla::Nothing();
    477        }
    478        if (data.isSome()) {
    479          MOZ_ASSERT(op == CacheOp::ReturnFromIC);
    480        }
    481        reader.skip(argLength);
    482        break;
    483    }
    484    MOZ_ASSERT(argStart + argLength == reader.currentPosition());
    485  }
    486 
    487  return data;
    488 }
    489 
    490 // Return the maximum number of actual arguments that will be passed to the
    491 // target function. This may be an overapproximation, for example when inlining
    492 // js::fun_call we may omit an argument.
    493 static uint32_t GetMaxCalleeNumActuals(BytecodeLocation loc) {
    494  switch (loc.getOp()) {
    495    case JSOp::GetProp:
    496    case JSOp::GetElem:
    497      // Getters do not pass arguments.
    498      return 0;
    499 
    500    case JSOp::SetProp:
    501    case JSOp::StrictSetProp:
    502      // Setters pass 1 argument.
    503      return 1;
    504 
    505    case JSOp::Call:
    506    case JSOp::CallContent:
    507    case JSOp::CallIgnoresRv:
    508    case JSOp::CallIter:
    509    case JSOp::CallContentIter:
    510    case JSOp::New:
    511    case JSOp::NewContent:
    512    case JSOp::SuperCall:
    513      return loc.getCallArgc();
    514 
    515    default:
    516      MOZ_CRASH("Unsupported op");
    517  }
    518 }
    519 
    520 /*static*/
    521 bool TrialInliner::IsValidInliningOp(JSOp op) {
    522  switch (op) {
    523    case JSOp::GetProp:
    524    case JSOp::GetElem:
    525    case JSOp::SetProp:
    526    case JSOp::StrictSetProp:
    527    case JSOp::Call:
    528    case JSOp::CallContent:
    529    case JSOp::CallIgnoresRv:
    530    case JSOp::CallIter:
    531    case JSOp::CallContentIter:
    532    case JSOp::New:
    533    case JSOp::NewContent:
    534    case JSOp::SuperCall:
    535      return true;
    536    default:
    537      break;
    538  }
    539  return false;
    540 }
    541 
    542 /*static*/
    543 bool TrialInliner::canInline(JSContext* cx, JSScript* script,
    544                             HandleScript caller, BytecodeLocation loc) {
    545  if (!script->hasJitScript()) {
    546    JitSpew(JitSpew_WarpTrialInlining, "SKIP: no JIT script");
    547    return false;
    548  }
    549  if (script->uninlineable()) {
    550    JitSpew(JitSpew_WarpTrialInlining, "SKIP: uninlineable flag");
    551    return false;
    552  }
    553  if (!script->canIonCompile()) {
    554    JitSpew(JitSpew_WarpTrialInlining, "SKIP: can't ion-compile");
    555    return false;
    556  }
    557  if (script->isDebuggee()) {
    558    JitSpew(JitSpew_WarpTrialInlining, "SKIP: is debuggee");
    559    return false;
    560  }
    561  // Don't inline cross-realm calls.
    562  if (script->realm() != caller->realm()) {
    563    JitSpew(JitSpew_WarpTrialInlining, "SKIP: cross-realm call");
    564    return false;
    565  }
    566  if (JitOptions.onlyInlineSelfHosted && !script->selfHosted()) {
    567    JitSpew(JitSpew_WarpTrialInlining, "SKIP: only inlining self hosted");
    568    return false;
    569  }
    570  if (!IsValidInliningOp(loc.getOp())) {
    571    JitSpew(JitSpew_WarpTrialInlining, "SKIP: non inlineable op");
    572    return false;
    573  }
    574 
    575  uint32_t maxCalleeNumActuals = GetMaxCalleeNumActuals(loc);
    576  if (maxCalleeNumActuals > ArgumentsObject::MaxInlinedArgs) {
    577    if (script->needsArgsObj()) {
    578      JitSpew(JitSpew_WarpTrialInlining,
    579              "SKIP: needs arguments object with %u actual args (maximum %u)",
    580              maxCalleeNumActuals, ArgumentsObject::MaxInlinedArgs);
    581      return false;
    582    }
    583    // The GetArgument(n) intrinsic in self-hosted code uses MGetInlinedArgument
    584    // too, so the same limit applies.
    585    if (script->usesArgumentsIntrinsics()) {
    586      JitSpew(JitSpew_WarpTrialInlining,
    587              "SKIP: uses GetArgument(i) with %u actual args (maximum %u)",
    588              maxCalleeNumActuals, ArgumentsObject::MaxInlinedArgs);
    589      return false;
    590    }
    591  }
    592 
    593  if (script->function() &&
    594      TooManyFormalArguments(script->function()->nargs())) {
    595    JitSpew(JitSpew_WarpTrialInlining, "SKIP: Too many formal arguments: %u",
    596            unsigned(script->function()->nargs()));
    597    return false;
    598  }
    599 
    600  if (TooManyFormalArguments(maxCalleeNumActuals)) {
    601    JitSpew(JitSpew_WarpTrialInlining, "SKIP: argc too large: %u",
    602            unsigned(loc.getCallArgc()));
    603    return false;
    604  }
    605 
    606  if (!script->hasBaselineScript() &&
    607      !script->jitScript()->ranBytecodeAnalysis()) {
    608    // If we don't have a baseline script, then we maybe haven't done
    609    // bytecode analysis yet. It's possible that the script is
    610    // uninlineable or can't be Ion compiled. Do bytecode analysis now.
    611    TempAllocator temp(&cx->tempLifoAlloc());
    612    BytecodeAnalysis analysis(temp, script);
    613    if (!analysis.init(temp)) {
    614      JitSpew(JitSpew_WarpTrialInlining, "SKIP: OOM in bytecode analysis");
    615      cx->recoverFromOutOfMemory();
    616      return false;
    617    }
    618    bool result = true;
    619    if (analysis.isInliningDisabled()) {
    620      JitSpew(JitSpew_WarpTrialInlining, "SKIP: uninlineable flag");
    621      script->disableIon();
    622      result = false;
    623    }
    624    if (analysis.isIonDisabled()) {
    625      JitSpew(JitSpew_WarpTrialInlining, "SKIP: can't ion-compile");
    626      script->setUninlineable();
    627      result = false;
    628    }
    629    script->jitScript()->setRanBytecodeAnalysis();
    630    return result;
    631  }
    632 
    633  return true;
    634 }
    635 
    636 static bool ShouldUseMonomorphicInlining(JSScript* targetScript) {
    637  switch (JitOptions.monomorphicInlining) {
    638    case UseMonomorphicInlining::Default:
    639      // Use heuristics below.
    640      break;
    641    case UseMonomorphicInlining::Always:
    642      return true;
    643    case UseMonomorphicInlining::Never:
    644      return false;
    645  }
    646 
    647  JitScript* jitScript = targetScript->jitScript();
    648  ICScript* icScript = jitScript->icScript();
    649 
    650  // Check for any ICs which are not monomorphic. The observation here is that
    651  // trial inlining can help us a lot in cases where it lets us further
    652  // specialize a script. But if it's already monomorphic, it's unlikely that
    653  // we will see significant specialization wins from trial inlining, so we
    654  // can use a cheaper and simpler inlining strategy.
    655  for (size_t i = 0; i < icScript->numICEntries(); i++) {
    656    ICEntry& entry = icScript->icEntry(i);
    657    ICFallbackStub* fallback = icScript->fallbackStub(i);
    658    if (fallback->enteredCount() > 0 ||
    659        fallback->state().mode() != ICState::Mode::Specialized) {
    660      return false;
    661    }
    662 
    663    ICStub* firstStub = entry.firstStub();
    664    if (firstStub != fallback) {
    665      for (ICStub* next = firstStub->toCacheIRStub()->next(); next;
    666           next = next->maybeNext()) {
    667        if (next->enteredCount() != 0) {
    668          return false;
    669        }
    670      }
    671    }
    672  }
    673 
    674  return true;
    675 }
    676 
    677 TrialInliningDecision TrialInliner::getInliningDecision(JSScript* targetScript,
    678                                                        ICCacheIRStub* stub,
    679                                                        BytecodeLocation loc) {
    680 #ifdef JS_JITSPEW
    681  if (JitSpewEnabled(JitSpew_WarpTrialInlining)) {
    682    UniqueChars funName;
    683    if (targetScript->function()) {
    684      if (JSAtom* atom = targetScript->function()->maybePartialDisplayAtom()) {
    685        funName = AtomToPrintableString(cx(), atom);
    686      }
    687    }
    688 
    689    JitSpew(JitSpew_WarpTrialInlining,
    690            "Inlining candidate JSOp::%s (offset=%u): callee script '%s' "
    691            "(%s:%u:%u)",
    692            CodeName(loc.getOp()), loc.bytecodeToOffset(script_),
    693            funName ? funName.get() : "<unnamed>", targetScript->filename(),
    694            targetScript->lineno(), targetScript->column().oneOriginValue());
    695    JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
    696  }
    697 #endif
    698 
    699  if (!canInline(cx(), targetScript, script_, loc)) {
    700    return TrialInliningDecision::NoInline;
    701  }
    702 
    703  // Don't inline (direct) recursive calls. This still allows recursion if
    704  // called through another function (f => g => f).
    705  if (script_ == targetScript) {
    706    JitSpew(JitSpew_WarpTrialInlining, "SKIP: recursion");
    707    return TrialInliningDecision::NoInline;
    708  }
    709 
    710  // Don't inline if the callee has a loop that was hot enough to enter Warp
    711  // via OSR. This helps prevent getting stuck in Baseline code for a long time.
    712  if (targetScript->jitScript()->hadIonOSR()) {
    713    JitSpew(JitSpew_WarpTrialInlining, "SKIP: had OSR");
    714    return TrialInliningDecision::NoInline;
    715  }
    716 
    717  // Ensure the total bytecode size does not exceed ionMaxScriptSize.
    718  size_t newTotalSize =
    719      inliningRootTotalBytecodeSize() + targetScript->length();
    720  if (newTotalSize > JitOptions.ionMaxScriptSize) {
    721    JitSpew(JitSpew_WarpTrialInlining, "SKIP: total size too big");
    722    return TrialInliningDecision::NoInline;
    723  }
    724 
    725  uint32_t entryCount = stub->enteredCount();
    726  if (entryCount < JitOptions.inliningEntryThreshold) {
    727    JitSpew(JitSpew_WarpTrialInlining, "SKIP: Entry count is %u (minimum %u)",
    728            unsigned(entryCount), unsigned(JitOptions.inliningEntryThreshold));
    729    return TrialInliningDecision::NoInline;
    730  }
    731 
    732  if (!JitOptions.isSmallFunction(targetScript)) {
    733    if (!targetScript->isInlinableLargeFunction()) {
    734      JitSpew(JitSpew_WarpTrialInlining, "SKIP: Length is %u (maximum %u)",
    735              unsigned(targetScript->length()),
    736              unsigned(JitOptions.smallFunctionMaxBytecodeLength));
    737      return TrialInliningDecision::NoInline;
    738    }
    739 
    740    JitSpew(JitSpew_WarpTrialInlining,
    741            "INFO: Ignored length (%u) of InlinableLargeFunction",
    742            unsigned(targetScript->length()));
    743  }
    744 
    745  // Decide between trial inlining or monomorphic inlining.
    746  if (!ShouldUseMonomorphicInlining(targetScript)) {
    747    return TrialInliningDecision::Inline;
    748  }
    749 
    750  JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
    751  JitSpew(JitSpew_WarpTrialInlining, "SUCCESS: Inlined monomorphically");
    752  return TrialInliningDecision::MonomorphicInline;
    753 }
    754 
    755 ICScript* TrialInliner::createInlinedICScript(JSScript* targetScript,
    756                                              BytecodeLocation loc) {
    757  MOZ_ASSERT(targetScript->hasJitScript());
    758 
    759  InliningRoot* root = getOrCreateInliningRoot();
    760  if (!root) {
    761    return nullptr;
    762  }
    763 
    764  // We don't have to check for overflow here because we have already
    765  // successfully allocated an ICScript with this number of entries
    766  // when creating the JitScript for the target function, and we
    767  // checked for overflow then.
    768  uint32_t fallbackStubsOffset =
    769      sizeof(ICScript) + targetScript->numICEntries() * sizeof(ICEntry);
    770  uint32_t allocSize = fallbackStubsOffset +
    771                       targetScript->numICEntries() * sizeof(ICFallbackStub);
    772 
    773  void* raw = cx()->pod_malloc<uint8_t>(allocSize);
    774  MOZ_ASSERT(uintptr_t(raw) % alignof(ICScript) == 0);
    775  if (!raw) {
    776    return nullptr;
    777  }
    778 
    779  uint32_t initialWarmUpCount = JitOptions.trialInliningInitialWarmUpCount;
    780 
    781  uint32_t depth = icScript_->depth() + 1;
    782  UniquePtr<ICScript> inlinedICScript(
    783      new (raw) ICScript(initialWarmUpCount, fallbackStubsOffset, allocSize,
    784                         depth, targetScript->length(), root));
    785 
    786  inlinedICScript->initICEntries(cx(), targetScript);
    787 
    788  uint32_t pcOffset = loc.bytecodeToOffset(script_);
    789  ICScript* result = inlinedICScript.get();
    790  if (!icScript_->addInlinedChild(cx(), std::move(inlinedICScript), pcOffset)) {
    791    return nullptr;
    792  }
    793  MOZ_ASSERT(result->numICEntries() == targetScript->numICEntries());
    794 
    795  if (targetScript->needsFunctionEnvironmentObjects()) {
    796    result->ensureEnvAllocSite(root->owningScript());
    797  }
    798 
    799  root->addToTotalBytecodeSize(targetScript->length());
    800 
    801  JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
    802  JitSpew(JitSpew_WarpTrialInlining,
    803          "SUCCESS: Outer ICScript: %p Inner ICScript: %p", icScript_, result);
    804 
    805  return result;
    806 }
    807 
    808 bool TrialInliner::maybeInlineCall(ICEntry& entry, ICFallbackStub* fallback,
    809                                   BytecodeLocation loc) {
    810  ICCacheIRStub* stub = maybeSingleStub(entry);
    811  if (!stub) {
    812 #ifdef JS_JITSPEW
    813    if (fallback->numOptimizedStubs() > 1) {
    814      JitSpew(JitSpew_WarpTrialInlining,
    815              "Inlining candidate JSOp::%s (offset=%u):", CodeName(loc.getOp()),
    816              fallback->pcOffset());
    817      JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
    818      JitSpew(JitSpew_WarpTrialInlining, "SKIP: Polymorphic (%u stubs)",
    819              (unsigned)fallback->numOptimizedStubs());
    820    }
    821 #endif
    822    return true;
    823  }
    824 
    825  MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
    826 
    827  // Look for a CallScriptedFunction with a known target.
    828  Maybe<InlinableCallData> data = FindInlinableCallData(stub);
    829  if (data.isNothing()) {
    830    return true;
    831  }
    832 
    833  MOZ_ASSERT(!data->icScript);
    834 
    835  TrialInliningDecision inlining = getInliningDecision(data->target, stub, loc);
    836  // Decide whether to inline the target.
    837  if (inlining == TrialInliningDecision::NoInline) {
    838    return true;
    839  }
    840 
    841  if (inlining == TrialInliningDecision::MonomorphicInline) {
    842    fallback->setTrialInliningState(TrialInliningState::MonomorphicInlined);
    843    return true;
    844  }
    845 
    846  ICScript* newICScript = createInlinedICScript(data->target, loc);
    847  if (!newICScript) {
    848    return false;
    849  }
    850 
    851  CacheIRWriter writer(cx());
    852  Int32OperandId argcId(writer.setInputOperandId(0));
    853  cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
    854 
    855  writer.callInlinedFunction(data->calleeOperand, argcId, newICScript,
    856                             data->callFlags,
    857                             ClampFixedArgc(loc.getCallArgc()));
    858  writer.returnFromIC();
    859 
    860  return replaceICStub(entry, fallback, writer, CacheKind::Call);
    861 }
    862 
    863 bool TrialInliner::maybeInlineGetter(ICEntry& entry, ICFallbackStub* fallback,
    864                                     BytecodeLocation loc, CacheKind kind) {
    865  ICCacheIRStub* stub = maybeSingleStub(entry);
    866  if (!stub) {
    867    return true;
    868  }
    869 
    870  MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
    871 
    872  Maybe<InlinableGetterData> data = FindInlinableGetterData(stub);
    873  if (data.isNothing()) {
    874    return true;
    875  }
    876 
    877  MOZ_ASSERT(!data->icScript);
    878 
    879  TrialInliningDecision inlining = getInliningDecision(data->target, stub, loc);
    880  // Decide whether to inline the target.
    881  if (inlining == TrialInliningDecision::NoInline) {
    882    return true;
    883  }
    884 
    885  if (inlining == TrialInliningDecision::MonomorphicInline) {
    886    fallback->setTrialInliningState(TrialInliningState::MonomorphicInlined);
    887    return true;
    888  }
    889 
    890  ICScript* newICScript = createInlinedICScript(data->target, loc);
    891  if (!newICScript) {
    892    return false;
    893  }
    894 
    895  CacheIRWriter writer(cx());
    896  ValOperandId valId(writer.setInputOperandId(0));
    897  if (kind == CacheKind::GetElem) {
    898    // Register the key operand.
    899    writer.setInputOperandId(1);
    900  }
    901  cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
    902 
    903  writer.callInlinedGetterResult(data->receiverOperand, data->calleeOperand,
    904                                 data->target->function(), newICScript,
    905                                 data->sameRealm);
    906  writer.returnFromIC();
    907 
    908  return replaceICStub(entry, fallback, writer, kind);
    909 }
    910 
    911 bool TrialInliner::maybeInlineSetter(ICEntry& entry, ICFallbackStub* fallback,
    912                                     BytecodeLocation loc, CacheKind kind) {
    913  ICCacheIRStub* stub = maybeSingleStub(entry);
    914  if (!stub) {
    915    return true;
    916  }
    917 
    918  MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
    919 
    920  Maybe<InlinableSetterData> data = FindInlinableSetterData(stub);
    921  if (data.isNothing()) {
    922    return true;
    923  }
    924 
    925  MOZ_ASSERT(!data->icScript);
    926 
    927  TrialInliningDecision inlining = getInliningDecision(data->target, stub, loc);
    928  // Decide whether to inline the target.
    929  if (inlining == TrialInliningDecision::NoInline) {
    930    return true;
    931  }
    932 
    933  if (inlining == TrialInliningDecision::MonomorphicInline) {
    934    fallback->setTrialInliningState(TrialInliningState::MonomorphicInlined);
    935    return true;
    936  }
    937 
    938  ICScript* newICScript = createInlinedICScript(data->target, loc);
    939  if (!newICScript) {
    940    return false;
    941  }
    942 
    943  CacheIRWriter writer(cx());
    944  ValOperandId objValId(writer.setInputOperandId(0));
    945  ValOperandId rhsValId(writer.setInputOperandId(1));
    946  cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
    947 
    948  writer.callInlinedSetter(data->receiverOperand, data->calleeOperand,
    949                           data->target->function(), data->rhsOperand,
    950                           newICScript, data->sameRealm);
    951  writer.returnFromIC();
    952 
    953  return replaceICStub(entry, fallback, writer, kind);
    954 }
    955 
    956 bool TrialInliner::tryInlining() {
    957  uint32_t numICEntries = icScript_->numICEntries();
    958  BytecodeLocation startLoc = script_->location();
    959 
    960  for (uint32_t icIndex = 0; icIndex < numICEntries; icIndex++) {
    961    ICEntry& entry = icScript_->icEntry(icIndex);
    962    ICFallbackStub* fallback = icScript_->fallbackStub(icIndex);
    963 
    964    if (!TryFoldingStubs(cx(), fallback, script_, icScript_)) {
    965      return false;
    966    }
    967 
    968    BytecodeLocation loc =
    969        startLoc + BytecodeLocationOffset(fallback->pcOffset());
    970    JSOp op = loc.getOp();
    971    switch (op) {
    972      case JSOp::Call:
    973      case JSOp::CallContent:
    974      case JSOp::CallIgnoresRv:
    975      case JSOp::CallIter:
    976      case JSOp::CallContentIter:
    977      case JSOp::New:
    978      case JSOp::NewContent:
    979      case JSOp::SuperCall:
    980        if (!maybeInlineCall(entry, fallback, loc)) {
    981          return false;
    982        }
    983        break;
    984      case JSOp::GetProp:
    985        if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetProp)) {
    986          return false;
    987        }
    988        break;
    989      case JSOp::GetElem:
    990        if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetElem)) {
    991          return false;
    992        }
    993        break;
    994      case JSOp::SetProp:
    995      case JSOp::StrictSetProp:
    996        if (!maybeInlineSetter(entry, fallback, loc, CacheKind::SetProp)) {
    997          return false;
    998        }
    999        break;
   1000      default:
   1001        break;
   1002    }
   1003  }
   1004 
   1005  return true;
   1006 }
   1007 
   1008 InliningRoot* TrialInliner::maybeGetInliningRoot() const {
   1009  if (auto* root = icScript_->inliningRoot()) {
   1010    return root;
   1011  }
   1012 
   1013  MOZ_ASSERT(!icScript_->isInlined());
   1014  return script_->jitScript()->inliningRoot();
   1015 }
   1016 
   1017 InliningRoot* TrialInliner::getOrCreateInliningRoot() {
   1018  if (auto* root = maybeGetInliningRoot()) {
   1019    return root;
   1020  }
   1021  return script_->jitScript()->getOrCreateInliningRoot(cx(), script_);
   1022 }
   1023 
   1024 size_t TrialInliner::inliningRootTotalBytecodeSize() const {
   1025  if (auto* root = maybeGetInliningRoot()) {
   1026    return root->totalBytecodeSize();
   1027  }
   1028  return script_->length();
   1029 }
   1030 
   1031 bool InliningRoot::addInlinedScript(UniquePtr<ICScript> icScript) {
   1032  return inlinedScripts_.append(std::move(icScript));
   1033 }
   1034 
   1035 void InliningRoot::trace(JSTracer* trc) {
   1036  TraceEdge(trc, &owningScript_, "inlining-root-owning-script");
   1037  for (auto& inlinedScript : inlinedScripts_) {
   1038    inlinedScript->trace(trc);
   1039  }
   1040 }
   1041 
   1042 bool InliningRoot::traceWeak(JSTracer* trc) {
   1043  bool allSurvived = true;
   1044  for (auto& inlinedScript : inlinedScripts_) {
   1045    if (!inlinedScript->traceWeak(trc)) {
   1046      allSurvived = false;
   1047    }
   1048  }
   1049  return allSurvived;
   1050 }
   1051 
   1052 void InliningRoot::purgeInactiveICScripts() {
   1053  mozilla::DebugOnly<uint32_t> totalSize = owningScript_->length();
   1054 
   1055  for (auto& inlinedScript : inlinedScripts_) {
   1056    if (inlinedScript->active()) {
   1057      totalSize += inlinedScript->bytecodeSize();
   1058    } else {
   1059      MOZ_ASSERT(inlinedScript->bytecodeSize() < totalBytecodeSize_);
   1060      totalBytecodeSize_ -= inlinedScript->bytecodeSize();
   1061    }
   1062  }
   1063 
   1064  MOZ_ASSERT(totalBytecodeSize_ == totalSize);
   1065 
   1066  Zone* zone = owningScript_->zone();
   1067 
   1068  inlinedScripts_.eraseIf([zone](auto& inlinedScript) {
   1069    if (inlinedScript->active()) {
   1070      return false;
   1071    }
   1072    inlinedScript->prepareForDestruction(zone);
   1073    return true;
   1074  });
   1075 }
   1076 
   1077 }  // namespace jit
   1078 }  // namespace js
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE