/*
* Copyright 2015 Mozilla Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "wasm/WasmIonCompile.h"
#include "mozilla/DebugOnly.h"
#include <algorithm>
#include <bit>
#include "jit/ABIArgGenerator.h"
#include "jit/CodeGenerator.h"
#include "jit/CompileInfo.h"
#include "jit/Ion.h"
#include "jit/IonOptimizationLevels.h"
#include "jit/MIR-wasm.h"
#include "jit/MIR.h"
#include "jit/ShuffleAnalysis.h"
#include "js/GCAPI.h" // JS::AutoSuppressGCAnalysis
#include "js/ScalarType.h" // js::Scalar::Type
#include "util/DifferentialTesting.h"
#include "wasm/WasmBaselineCompile.h"
#include "wasm/WasmBuiltinModule.h"
#include "wasm/WasmBuiltins.h"
#include "wasm/WasmCodegenTypes.h"
#include "wasm/WasmGC.h"
#include "wasm/WasmGcObject.h"
#include "wasm/WasmGenerator.h"
#include "wasm/WasmOpIter.h"
#include "wasm/WasmSignalHandlers.h"
#include "wasm/WasmStacks.h"
#include "wasm/WasmStubs.h"
#include "wasm/WasmValidate.h"
using namespace js;
using namespace js::jit;
using namespace js::wasm;
using mozilla::Nothing;
namespace {
using UniqueCompileInfo = UniquePtr<CompileInfo>;
using UniqueCompileInfoVector = Vector<UniqueCompileInfo,
1, SystemAllocPolicy>;
using BlockVector = Vector<MBasicBlock*,
8, SystemAllocPolicy>;
using DefVector = Vector<MDefinition*,
8, SystemAllocPolicy>;
using ControlInstructionVector =
Vector<MControlInstruction*,
8, SystemAllocPolicy>;
// [SMDOC] WebAssembly Exception Handling in Ion
// =======================================================
//
// ## Throwing instructions
//
// Wasm exceptions can be thrown by either a throw instruction (local throw),
// or by a wasm call.
//
// ## The "catching try control"
//
// We know we are in try-code if there is a surrounding ControlItem with
// LabelKind::Try. The innermost such control is called the
// "catching try control".
//
// ## Throws without a catching try control
//
// Such throws are implemented with an instance call that triggers the exception
// unwinding runtime. The exception unwinding runtime will not return to the
// function.
//
// ## "landing pad" and "pre-pad" blocks
//
// When an exception is thrown, the unwinder will search for the nearest
// enclosing try block and redirect control flow to it. The code that executes
// before any catch blocks is called the 'landing pad'. The 'landing pad' is
// responsible to:
// 1. Consume the pending exception state from
// Instance::pendingException(Tag)
// 2. Branch to the correct catch block, or else rethrow
//
// There is one landing pad for each try block. The immediate predecessors of
// the landing pad are called 'pre-pad' blocks. There is one pre-pad block per
// throwing instruction.
//
// ## Creating pre-pad blocks
//
// There are two possible sorts of pre-pad blocks, depending on whether we
// are branching after a local throw instruction, or after a wasm call:
//
// - If we encounter a local throw, we create the exception and tag objects,
// store them to Instance::pendingException(Tag), and then jump to the
// landing pad.
//
// - If we encounter a wasm call, we construct a MWasmCallCatchable which is a
// control instruction with either a branch to a fallthrough block or
// to a pre-pad block.
//
// The pre-pad block for a wasm call is empty except for a jump to the
// landing pad. It only exists to avoid critical edges which when split would
// violate the invariants of MWasmCallCatchable. The pending exception state
// is taken care of by the unwinder.
//
// Each pre-pad ends with a pending jump to the landing pad. The pending jumps
// to the landing pad are tracked in `tryPadPatches`. These are called
// "pad patches".
//
// ## Creating the landing pad
//
// When we exit try-code, we check if tryPadPatches has captured any control
// instructions (pad patches). If not, we don't compile any catches and we mark
// the rest as dead code.
//
// If there are pre-pad blocks, we join them to create a landing pad (or just
// "pad"). The pad's last two slots are the caught exception, and the
// exception's tag object.
//
// There are three different forms of try-catch/catch_all Wasm instructions,
// which result in different form of landing pad.
//
// 1. A catchless try, so a Wasm instruction of the form "try ... end".
// - In this case, we end the pad by rethrowing the caught exception.
//
// 2. A single catch_all after a try.
// - If the first catch after a try is a catch_all, then there won't be
// any more catches, but we need the exception and its tag object, in
// case the code in a catch_all contains "rethrow" instructions.
// - The Wasm instruction "rethrow", gets the exception and tag object to
// rethrow from the last two slots of the landing pad which, due to
// validation, is the l'th surrounding ControlItem.
// - We immediately GoTo to a new block after the pad and pop both the
// exception and tag object, as we don't need them anymore in this case.
//
// 3. Otherwise, there is one or more catch code blocks following.
// - In this case, we construct the landing pad by creating a sequence
// of compare and branch blocks that compare the pending exception tag
// object to the tag object of the current tagged catch block. This is
// done incrementally as we visit each tagged catch block in the bytecode
// stream. At every step, we update the ControlItem's block to point to
// the next block to be created in the landing pad sequence. The final
// block will either be a rethrow, if there is no catch_all, or else a
// jump to a catch_all block.
struct TryControl {
// Branches to bind to the try's landing pad.
ControlInstructionVector landingPadPatches;
// For `try_table`, the list of tagged catches and labels to branch to.
TryTableCatchVector catches;
// The pending exception for the try's landing pad.
MDefinition* pendingException;
// The pending exception's tag for the try's landing pad.
MDefinition* pendingExceptionTag;
// Whether this try is in the body and should catch any thrown exception.
bool inBody;
TryControl()
: pendingException(nullptr),
pendingExceptionTag(nullptr),
inBody(
false) {}
// Reset the try control for when it is cached in FunctionCompiler.
void reset() {
landingPadPatches.clearAndFree();
catches.clearAndFree();
inBody =
false;
}
};
using UniqueTryControl = UniquePtr<TryControl>;
using VectorUniqueTryControl = Vector<UniqueTryControl,
2, SystemAllocPolicy>;
struct ControlFlowPatch {
MControlInstruction* ins;
uint32_t index;
ControlFlowPatch(MControlInstruction* ins, uint32_t index)
: ins(ins), index(index) {}
};
using ControlFlowPatchVector = Vector<ControlFlowPatch,
0, SystemAllocPolicy>;
struct PendingBlockTarget {
ControlFlowPatchVector patches;
BranchHint hint = BranchHint::Invalid;
};
using PendingBlockTargetVector =
Vector<PendingBlockTarget,
0, SystemAllocPolicy>;
// Inlined functions accumulate all returns to be bound to a caller function
// after compilation is finished.
struct PendingInlineReturn {
PendingInlineReturn(MGoto* jump, DefVector&& results)
: jump(jump), results(std::move(results)) {}
MGoto* jump;
DefVector results;
};
using PendingInlineReturnVector =
Vector<PendingInlineReturn,
1, SystemAllocPolicy>;
// CallCompileState describes a call that is being compiled.
struct CallCompileState {
// A generator object that is passed each argument as it is compiled.
ABIArgGenerator abi;
// Whether we pass FP values through GPRs or FPRs.
bool hardFP =
true;
// The ABI we are using for this call.
ABIKind abiKind;
// Accumulates the register arguments while compiling arguments.
MWasmCallBase::Args regArgs;
// Reserved argument for passing Instance* to builtin instance method calls.
ABIArg instanceArg;
// The stack area in which the callee will write stack return values, or
// nullptr if no stack results.
MWasmStackResultArea* stackResultArea = nullptr;
// Indicates that the call is a return/tail call.
bool returnCall =
false;
// The landing pad patches for the nearest enclosing try-catch. This is
// non-null iff the call is catchable.
ControlInstructionVector* tryLandingPadPatches = nullptr;
// The index of the try note for a catchable call.
uint32_t tryNoteIndex = UINT32_MAX;
// The block to take for fallthrough execution for a catchable call.
MBasicBlock* fallthroughBlock = nullptr;
// The block to take for exceptional execution for a catchable call.
MBasicBlock* prePadBlock = nullptr;
explicit CallCompileState(ABIKind abiKind) : abi(abiKind), abiKind(abiKind) {
if (abiKind == ABIKind::System) {
// The system ABI follows the hardFP setting on ARM32.
#if defined(JS_CODEGEN_ARM)
hardFP = ARMFlags::UseHardFpABI();
abi.setUseHardFp(hardFP);
#endif
}
else {
#if defined(JS_CODEGEN_ARM)
MOZ_ASSERT(hardFP,
"The WASM ABI passes FP arguments in registers");
#endif
}
}
bool isCatchable()
const {
return tryLandingPadPatches != nullptr; }
};
struct Control {
MBasicBlock* block;
UniqueTryControl tryControl;
Control() : block(nullptr), tryControl(nullptr) {}
Control(Control&&) =
default;
Control(
const Control&) =
delete;
};
struct IonCompilePolicy {
// We store SSA definitions in the value stack.
using Value = MDefinition*;
using ValueVector = DefVector;
// We store loop headers and then/else blocks in the control flow stack.
// In the case of try-catch control blocks, we collect additional information
// regarding the possible paths from throws and calls to a landing pad, as
// well as information on the landing pad's handlers (its catches).
using ControlItem = Control;
};
using IonOpIter = OpIter<IonCompilePolicy>;
// Statistics for inlining (at all depths) into the root function.
struct InliningStats {
size_t inlinedDirectBytecodeSize =
0;
// sum of sizes of inlinees
size_t inlinedDirectFunctions =
0;
// number of inlinees
size_t inlinedCallRefBytecodeSize =
0;
// sum of sizes of inlinees
size_t inlinedCallRefFunctions =
0;
// number of inlinees
bool largeFunctionBackoff =
false;
// did large function backoff happen?
};
// Encapsulates the generation of MIR for a wasm function and any functions
// that become inlined into it.
class RootCompiler {
const CompilerEnvironment& compilerEnv_;
const CodeMetadata& codeMeta_;
const CodeTailMetadata* codeTailMeta_;
const ValTypeVector& locals_;
const FuncCompileInput& func_;
Decoder& decoder_;
FeatureUsage observedFeatures_;
CompileInfo compileInfo_;
const JitCompileOptions options_;
TempAllocator& alloc_;
MIRGraph mirGraph_;
MIRGenerator mirGen_;
// The current loop depth we're generating inside of. This includes all
// callee functions when we're generating an inlined function, and so it
// lives here on the root compiler.
uint32_t loopDepth_;
// The current stack of bytecode offsets of the caller functions of the
// function currently being inlined.
BytecodeOffsetVector inlinedCallerOffsets_;
InlinedCallerOffsetIndex inlinedCallerOffsetsIndex_;
// Compilation statistics for this function.
CompileStats funcStats_;
// Accumulated inlining statistics for this function.
InliningStats inliningStats_;
// The remaining inlining budget, in terms of bytecode bytes. This may go
// negative and so is signed.
int64_t localInliningBudget_;
// All jit::CompileInfo objects created during this compilation. This must
// be kept alive for as long as the MIR graph is alive.
UniqueCompileInfoVector compileInfos_;
// Cache of TryControl to minimize heap allocations.
VectorUniqueTryControl tryControlCache_;
// Reference to masm.tryNotes()
wasm::TryNoteVector& tryNotes_;
// Reference to masm.inliningContext()
wasm::InliningContext& inliningContext_;
public:
RootCompiler(
const CompilerEnvironment& compilerEnv,
const CodeMetadata& codeMeta,
const CodeTailMetadata* codeTailMeta, TempAllocator& alloc,
const ValTypeVector& locals,
const FuncCompileInput& func,
Decoder& decoder, wasm::TryNoteVector& tryNotes,
wasm::InliningContext& inliningContext)
: compilerEnv_(compilerEnv),
codeMeta_(codeMeta),
codeTailMeta_(codeTailMeta),
locals_(locals),
func_(func),
decoder_(decoder),
observedFeatures_(FeatureUsage::None),
compileInfo_(locals.length()),
alloc_(alloc),
mirGraph_(&alloc),
mirGen_(nullptr, options_, &alloc_, &mirGraph_, &compileInfo_,
IonOptimizations.get(OptimizationLevel::Wasm), &codeMeta),
loopDepth_(
0),
localInliningBudget_(
0),
tryNotes_(tryNotes),
inliningContext_(inliningContext) {}
const CompilerEnvironment& compilerEnv()
const {
return compilerEnv_; }
const CodeMetadata& codeMeta()
const {
return codeMeta_; }
const CodeTailMetadata* codeTailMeta()
const {
return codeTailMeta_; }
const FuncCompileInput& func()
const {
return func_; }
TempAllocator& alloc() {
return alloc_; }
MIRGraph& mirGraph() {
return mirGraph_; }
MIRGenerator& mirGen() {
return mirGen_; }
int64_t inliningBudget()
const {
return localInliningBudget_; }
FeatureUsage observedFeatures()
const {
return observedFeatures_; }
const CompileStats& funcStats()
const {
return funcStats_; }
void noteLargeFunctionBackoffWasApplied() {
inliningStats_.largeFunctionBackoff =
true;
}
uint32_t loopDepth()
const {
return loopDepth_; }
void startLoop() { loopDepth_++; }
void closeLoop() { loopDepth_--; }
[[nodiscard]]
bool generate();
InlinedCallerOffsetIndex inlinedCallerOffsetsIndex()
const {
return inlinedCallerOffsetsIndex_;
}
// Add a compile info for an inlined function. This keeps the inlined
// function's compile info alive for the outermost function's
// compilation.
[[nodiscard]] CompileInfo* startInlineCall(
uint32_t callerFuncIndex, BytecodeOffset callerOffset,
uint32_t calleeFuncIndex, uint32_t numLocals, size_t inlineeBytecodeSize,
InliningHeuristics::CallKind callKind);
void finishInlineCall();
// Add a try note and return the index.
[[nodiscard]]
bool addTryNote(uint32_t* tryNoteIndex) {
if (!tryNotes_.append(wasm::TryNote())) {
return false;
}
*tryNoteIndex = tryNotes_.length() -
1;
return true;
}
// Try to get a free TryControl from the cache, or allocate a new one.
[[nodiscard]] UniqueTryControl newTryControl() {
if (tryControlCache_.empty()) {
return UniqueTryControl(js_new<TryControl>());
}
UniqueTryControl tryControl = std::move(tryControlCache_.back());
tryControlCache_.popBack();
return tryControl;
}
// Release the TryControl to the cache.
void freeTryControl(UniqueTryControl&& tryControl) {
// Ensure that it's in a consistent state
tryControl->reset();
// Ignore any OOM, as we'll fail later
(
void)tryControlCache_.append(std::move(tryControl));
}
};
// Encapsulates the generation of MIR for a single function in a wasm module.
class FunctionCompiler {
// The root function compiler we are being compiled within.
RootCompiler& rootCompiler_;
// The caller function compiler, if any, that we are being inlined into.
// Note that `inliningDepth_` is zero for the first inlinee, one for the
// second inlinee, etc.
const FunctionCompiler* callerCompiler_;
const uint32_t inliningDepth_;
// Information about this function's bytecode and parsing state
IonOpIter iter_;
uint32_t functionBodyOffset_;
const FuncCompileInput& func_;
const ValTypeVector& locals_;
size_t lastReadCallSite_;
size_t numCallRefs_;
size_t numAllocSites_;
// CompileInfo for compiling the MIR for this function. Allocated inside of
// RootCompiler::compileInfos, and kept alive for the duration of the
// total compilation.
const jit::CompileInfo& info_;
MBasicBlock* curBlock_;
uint32_t maxStackArgBytes_;
// When generating a forward branch we haven't created the basic block that
// the branch needs to target. We handle this by accumulating all the branch
// instructions that want to target a block we have not yet created into
// `pendingBlocks_` and then patching them in `bindBranches`.
//
// For performance reasons we only grow `pendingBlocks_` as needed, never
// shrink it. So the length of the vector has no relation to the current
// nesting depth of wasm blocks. We use `pendingBlockDepth_` to track the
// current wasm block depth. We assert that all entries beyond the current
// block depth are empty.
uint32_t pendingBlockDepth_;
PendingBlockTargetVector pendingBlocks_;
// Control flow patches for exceptions that are caught without a landing
// pad they can directly jump to. This happens when either:
// (1) `delegate` targets the function body label.
// (2) A `try` ends without any cases, and there is no enclosing `try`.
// (3) There is no `try` in this function, but a caller function (when
// inlining) has a `try`.
//
// These exceptions will be rethrown using `emitBodyRethrowPad`.
ControlInstructionVector bodyRethrowPadPatches_;
// A vector of the returns in this function for use when we're being inlined
// into another function.
PendingInlineReturnVector pendingInlineReturns_;
// A block that all uncaught exceptions in this function will jump to. The
// inline caller will link this to the nearest enclosing catch handler.
MBasicBlock* pendingInlineCatchBlock_;
// Instance pointer argument to the current function.
MWasmParameter* instancePointer_;
MWasmParameter* stackResultPointer_;
public:
// Construct a FunctionCompiler for the root function of a compilation
FunctionCompiler(RootCompiler& rootCompiler, Decoder& decoder,
const FuncCompileInput& func,
const ValTypeVector& locals,
const CompileInfo& compileInfo)
: rootCompiler_(rootCompiler),
callerCompiler_(nullptr),
inliningDepth_(
0),
iter_(rootCompiler.codeMeta(), decoder, locals),
functionBodyOffset_(decoder.beginOffset()),
func_(func),
locals_(locals),
lastReadCallSite_(
0),
numCallRefs_(
0),
numAllocSites_(
0),
info_(compileInfo),
curBlock_(nullptr),
maxStackArgBytes_(
0),
pendingBlockDepth_(
0),
pendingInlineCatchBlock_(nullptr),
instancePointer_(nullptr),
stackResultPointer_(nullptr) {}
// Construct a FunctionCompiler for an inlined callee of a compilation
FunctionCompiler(
const FunctionCompiler* callerCompiler, Decoder& decoder,
const FuncCompileInput& func,
const ValTypeVector& locals,
const CompileInfo& compileInfo)
: rootCompiler_(callerCompiler->rootCompiler_),
callerCompiler_(callerCompiler),
inliningDepth_(callerCompiler_->inliningDepth() +
1),
iter_(rootCompiler_.codeMeta(), decoder, locals),
functionBodyOffset_(decoder.beginOffset()),
func_(func),
locals_(locals),
lastReadCallSite_(
0),
numCallRefs_(
0),
numAllocSites_(
0),
info_(compileInfo),
curBlock_(nullptr),
maxStackArgBytes_(
0),
pendingBlockDepth_(
0),
pendingInlineCatchBlock_(nullptr),
instancePointer_(callerCompiler_->instancePointer_),
stackResultPointer_(nullptr) {}
RootCompiler& rootCompiler() {
return rootCompiler_; }
const CodeMetadata& codeMeta()
const {
return rootCompiler_.codeMeta(); }
const CodeTailMetadata* codeTailMeta()
const {
return rootCompiler_.codeTailMeta();
}
IonOpIter& iter() {
return iter_; }
uint32_t relativeBytecodeOffset() {
return readBytecodeOffset() - functionBodyOffset_;
}
TempAllocator& alloc()
const {
return rootCompiler_.alloc(); }
// FIXME(1401675): Replace with BlockType.
uint32_t funcIndex()
const {
return func_.index; }
const FuncType& funcType()
const {
return codeMeta().getFuncType(func_.index);
}
bool isInlined()
const {
return callerCompiler_ != nullptr; }
uint32_t inliningDepth()
const {
return inliningDepth_; }
MBasicBlock* getCurBlock()
const {
return curBlock_; }
BytecodeOffset bytecodeOffset()
const {
return iter_.bytecodeOffset(); }
CallSiteDesc callSiteDesc(CallSiteKind kind) {
return CallSiteDesc(bytecodeOffset().offset(),
rootCompiler_.inlinedCallerOffsetsIndex(), kind);
}
TrapSiteDesc trapSiteDesc() {
return TrapSiteDesc(wasm::BytecodeOffset(bytecodeOffset()),
rootCompiler_.inlinedCallerOffsetsIndex());
}
TrapSiteDesc trapSiteDescWithCallSiteLineNumber() {
return TrapSiteDesc(wasm::BytecodeOffset(readCallSiteLineOrBytecode()),
rootCompiler_.inlinedCallerOffsetsIndex());
}
FeatureUsage featureUsage()
const {
return iter_.featureUsage(); }
[[nodiscard]]
bool initRoot() {
// We are not being inlined into something
MOZ_ASSERT(!callerCompiler_);
// Prepare the entry block for MIR generation:
const FuncType& ft = funcType();
const ArgTypeVector args(ft);
if (!mirGen().ensureBallast()) {
return false;
}
if (!newBlock(
/* prev */ nullptr, &curBlock_)) {
return false;
}
for (ABIArgIter i(args, ABIKind::Wasm); !i.done(); i++) {
MaybeRefType argRefType;
if (!args.isSyntheticStackResultPointerArg(i.index())) {
ValType argType = ft.arg(i.index());
argRefType = argType.isRefType() ? MaybeRefType(argType.refType())
: MaybeRefType();
}
MWasmParameter* ins =
MWasmParameter::
New(alloc(), *i, i.mirType(), argRefType);
curBlock_->add(ins);
if (args.isSyntheticStackResultPointerArg(i.index())) {
MOZ_ASSERT(stackResultPointer_ == nullptr);
stackResultPointer_ = ins;
}
else {
curBlock_->initSlot(info().localSlot(args.naturalIndex(i.index())),
ins);
}
if (!mirGen().ensureBallast()) {
return false;
}
}
// Set up a parameter that receives the hidden instance pointer argument.
instancePointer_ =
MWasmParameter::
New(alloc(), ABIArg(InstanceReg), MIRType::Pointer);
curBlock_->add(instancePointer_);
if (!mirGen().ensureBallast()) {
return false;
}
for (size_t i = args.lengthWithoutStackResults(); i < locals_.length();
i++) {
ValType slotValType = locals_[i];
#ifndef ENABLE_WASM_SIMD
if (slotValType == ValType::V128) {
return iter().fail(
"Ion has no SIMD support yet");
}
#endif
MDefinition* zero = constantZeroOfValType(slotValType);
curBlock_->initSlot(info().localSlot(i), zero);
if (!mirGen().ensureBallast()) {
return false;
}
}
return true;
}
[[nodiscard]]
bool initInline(
const DefVector& argValues) {
// "This is an inlined-callee FunctionCompiler"
MOZ_ASSERT(callerCompiler_);
// Prepare the entry block for MIR generation:
if (!mirGen().ensureBallast()) {
return false;
}
if (!newBlock(nullptr, &curBlock_)) {
return false;
}
MBasicBlock* pred = callerCompiler_->curBlock_;
pred->end(MGoto::
New(alloc(), curBlock_));
if (!curBlock_->addPredecessorWithoutPhis(pred)) {
return false;
}
// Set up args slots to point to passed argument values
const FuncType& type = funcType();
for (uint32_t argIndex =
0; argIndex < type.args().length(); argIndex++) {
curBlock_->initSlot(info().localSlot(argIndex), argValues[argIndex]);
}
// Set up a parameter that receives the hidden instance pointer argument.
instancePointer_ = callerCompiler_->instancePointer_;
// Initialize all local slots to zero value
for (size_t i = type.args().length(); i < locals_.length(); i++) {
ValType slotValType = locals_[i];
#ifndef ENABLE_WASM_SIMD
if (slotValType == ValType::V128) {
return iter().fail(
"Ion has no SIMD support yet");
}
#endif
MDefinition* zero = constantZeroOfValType(slotValType);
curBlock_->initSlot(info().localSlot(i), zero);
if (!mirGen().ensureBallast()) {
return false;
}
}
return true;
}
void finish() {
mirGen().accumulateWasmMaxStackArgBytes(maxStackArgBytes_);
MOZ_ASSERT(pendingBlockDepth_ ==
0);
#ifdef DEBUG
for (PendingBlockTarget& targets : pendingBlocks_) {
MOZ_ASSERT(targets.patches.empty());
}
#endif
MOZ_ASSERT(inDeadCode());
MOZ_ASSERT(done());
MOZ_ASSERT(func_.callSiteLineNums.length() == lastReadCallSite_);
MOZ_ASSERT_IF(
compilerEnv().mode() == CompileMode::LazyTiering,
codeTailMeta()->getFuncDefCallRefs(funcIndex()).length == numCallRefs_);
MOZ_ASSERT_IF(codeTailMeta(),
codeTailMeta()->getFuncDefAllocSites(funcIndex()).length ==
numAllocSites_);
MOZ_ASSERT_IF(!isInlined(),
pendingInlineReturns_.empty() && !pendingInlineCatchBlock_);
MOZ_ASSERT(bodyRethrowPadPatches_.empty());
}
/************************* Read-only interface (after local scope setup) */
MIRGenerator& mirGen()
const {
return rootCompiler_.mirGen(); }
MIRGraph& mirGraph()
const {
return rootCompiler_.mirGraph(); }
const CompileInfo& info()
const {
return info_; }
const CompilerEnvironment& compilerEnv()
const {
return rootCompiler_.compilerEnv();
}
MDefinition* getLocalDef(
unsigned slot) {
if (inDeadCode()) {
return nullptr;
}
return curBlock_->getSlot(info().localSlot(slot));
}
const ValTypeVector& locals()
const {
return locals_; }
/*********************************************************** Constants ***/
MDefinition* constantF32(
float f) {
if (inDeadCode()) {
return nullptr;
}
auto* cst = MWasmFloatConstant::NewFloat32(alloc(), f);
curBlock_->add(cst);
return cst;
}
// Hide all other overloads, to guarantee no implicit argument conversion.
template <
typename T>
MDefinition* constantF32(T) =
delete;
MDefinition* constantF64(
double d) {
if (inDeadCode()) {
return nullptr;
}
auto* cst = MWasmFloatConstant::NewDouble(alloc(), d);
curBlock_->add(cst);
return cst;
}
template <
typename T>
MDefinition* constantF64(T) =
delete;
MDefinition* constantI32(int32_t i) {
if (inDeadCode()) {
return nullptr;
}
MConstant* constant = MConstant::NewInt32(alloc(), i);
curBlock_->add(constant);
return constant;
}
template <
typename T>
MDefinition* constantI32(T) =
delete;
MDefinition* constantI64(int64_t i) {
if (inDeadCode()) {
return nullptr;
}
MConstant* constant = MConstant::NewInt64(alloc(), i);
curBlock_->add(constant);
return constant;
}
template <
typename T>
MDefinition* constantI64(T) =
delete;
// Produce an MConstant of the machine's target int type (Int32 or Int64).
MDefinition* constantTargetWord(intptr_t n) {
return targetIs64Bit() ? constantI64(int64_t(n)) : constantI32(int32_t(n));
}
template <
typename T>
MDefinition* constantTargetWord(T) =
delete;
#ifdef ENABLE_WASM_SIMD
MDefinition* constantV128(V128 v) {
if (inDeadCode()) {
return nullptr;
}
MWasmFloatConstant* constant = MWasmFloatConstant::NewSimd128(
alloc(), SimdConstant::CreateSimd128((int8_t*)v.bytes));
curBlock_->add(constant);
return constant;
}
template <
typename T>
MDefinition* constantV128(T) =
delete;
#endif
MDefinition* constantNullRef(MaybeRefType type) {
if (inDeadCode()) {
return nullptr;
}
// MConstant has a lot of baggage so we don't use that here.
MWasmNullConstant* constant = MWasmNullConstant::
New(alloc(), type);
curBlock_->add(constant);
return constant;
}
// Produce a zero constant for the specified ValType.
MDefinition* constantZeroOfValType(ValType valType) {
switch (valType.kind()) {
case ValType::I32:
return constantI32(
0);
case ValType::I64:
return constantI64(int64_t(
0));
#ifdef ENABLE_WASM_SIMD
case ValType::V128:
return constantV128(V128(
0));
#endif
case ValType::F32:
return constantF32(
0.
0f);
case ValType::F64:
return constantF64(
0.
0);
case ValType::Ref:
return constantNullRef(MaybeRefType(valType.refType()));
default:
MOZ_CRASH();
}
}
/***************************** Code generation (after local scope setup) */
void fence() {
if (inDeadCode()) {
return;
}
MWasmFence* ins = MWasmFence::
New(alloc());
curBlock_->add(ins);
}
template <
class T>
MDefinition* unary(MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::
New(alloc(), op);
curBlock_->add(ins);
return ins;
}
template <
class T>
MDefinition* unary(MDefinition* op, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::
New(alloc(), op, type);
curBlock_->add(ins);
return ins;
}
template <
class T>
MDefinition* binary(MDefinition* lhs, MDefinition* rhs) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::
New(alloc(), lhs, rhs);
curBlock_->add(ins);
return ins;
}
template <
class T>
MDefinition* binary(MDefinition* lhs, MDefinition* rhs, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::
New(alloc(), lhs, rhs, type);
curBlock_->add(ins);
return ins;
}
template <
class T>
MDefinition* binary(MDefinition* lhs, MDefinition* rhs, MIRType type,
MWasmBinaryBitwise::SubOpcode subOpc) {
if (inDeadCode()) {
return nullptr;
}
T* ins = T::
New(alloc(), lhs, rhs, type, subOpc);
curBlock_->add(ins);
return ins;
}
MDefinition* ursh(MDefinition* lhs, MDefinition* rhs, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MUrsh::NewWasm(alloc(), lhs, rhs, type);
curBlock_->add(ins);
return ins;
}
MDefinition* add(MDefinition* lhs, MDefinition* rhs, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MAdd::NewWasm(alloc(), lhs, rhs, type);
curBlock_->add(ins);
return ins;
}
bool mustPreserveNaN(MIRType type) {
return IsFloatingPointType(type) && !codeMeta().isAsmJS();
}
MDefinition* sub(MDefinition* lhs, MDefinition* rhs, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
// wasm can't fold x - 0.0 because of NaN with custom payloads.
MSub* ins = MSub::NewWasm(alloc(), lhs, rhs, type, mustPreserveNaN(type));
curBlock_->add(ins);
return ins;
}
MDefinition* nearbyInt(MDefinition* input, RoundingMode roundingMode) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MNearbyInt::
New(alloc(), input, input->type(), roundingMode);
curBlock_->add(ins);
return ins;
}
MDefinition* minMax(MDefinition* lhs, MDefinition* rhs, MIRType type,
bool isMax) {
if (inDeadCode()) {
return nullptr;
}
if (mustPreserveNaN(type)) {
// Convert signaling NaN to quiet NaNs.
MDefinition* zero = constantZeroOfValType(ValType::fromMIRType(type));
lhs = sub(lhs, zero, type);
rhs = sub(rhs, zero, type);
}
MMinMax* ins = MMinMax::NewWasm(alloc(), lhs, rhs, type, isMax);
curBlock_->add(ins);
return ins;
}
MDefinition* mul(MDefinition* lhs, MDefinition* rhs, MIRType type,
MMul::Mode mode) {
if (inDeadCode()) {
return nullptr;
}
// wasm can't fold x * 1.0 because of NaN with custom payloads.
auto* ins =
MMul::NewWasm(alloc(), lhs, rhs, type, mode, mustPreserveNaN(type));
curBlock_->add(ins);
return ins;
}
MDefinition* div(MDefinition* lhs, MDefinition* rhs, MIRType type,
bool unsignd) {
if (inDeadCode()) {
return nullptr;
}
bool trapOnError = !codeMeta().isAsmJS();
if (!unsignd && type == MIRType::Int32) {
// Enforce the signedness of the operation by coercing the operands
// to signed. Otherwise, operands that "look" unsigned to Ion but
// are not unsigned to Baldr (eg, unsigned right shifts) may lead to
// the operation being executed unsigned. Applies to mod() as well.
//
// Do this for Int32 only since Int64 is not subject to the same
// issues.
//
// Note the offsets passed to MWasmBuiltinTruncateToInt32 are wrong here,
// but it doesn't matter: they're not codegen'd to calls since inputs
// already are int32.
auto* lhs2 = createTruncateToInt32(lhs);
curBlock_->add(lhs2);
lhs = lhs2;
auto* rhs2 = createTruncateToInt32(rhs);
curBlock_->add(rhs2);
rhs = rhs2;
}
// For x86 and arm we implement i64 div via c++ builtin.
// A call to c++ builtin requires instance pointer.
#if defined(JS_CODEGEN_X86) ||
defined(JS_CODEGEN_ARM)
if (type == MIRType::Int64) {
auto* ins = MWasmBuiltinDivI64::
New(alloc(), lhs, rhs, instancePointer_,
unsignd, trapOnError, trapSiteDesc());
curBlock_->add(ins);
return ins;
}
#endif
auto* ins = MDiv::
New(alloc(), lhs, rhs, type, unsignd, trapOnError,
trapSiteDesc(), mustPreserveNaN(type));
curBlock_->add(ins);
return ins;
}
MInstruction* createTruncateToInt32(MDefinition* op) {
if (op->type() == MIRType::
Double || op->type() == MIRType::Float32) {
return MWasmBuiltinTruncateToInt32::
New(alloc(), op, instancePointer_);
}
return MTruncateToInt32::
New(alloc(), op);
}
MDefinition* mod(MDefinition* lhs, MDefinition* rhs, MIRType type,
bool unsignd) {
if (inDeadCode()) {
return nullptr;
}
bool trapOnError = !codeMeta().isAsmJS();
if (!unsignd && type == MIRType::Int32) {
// See block comment in div().
auto* lhs2 = createTruncateToInt32(lhs);
curBlock_->add(lhs2);
lhs = lhs2;
auto* rhs2 = createTruncateToInt32(rhs);
curBlock_->add(rhs2);
rhs = rhs2;
}
// For x86 and arm we implement i64 mod via c++ builtin.
// A call to c++ builtin requires instance pointer.
#if defined(JS_CODEGEN_X86) ||
defined(JS_CODEGEN_ARM)
if (type == MIRType::Int64) {
auto* ins = MWasmBuiltinModI64::
New(alloc(), lhs, rhs, instancePointer_,
unsignd, trapOnError, trapSiteDesc());
curBlock_->add(ins);
return ins;
}
#endif
// Should be handled separately because we call BuiltinThunk for this case
// and so, need to add the dependency from instancePointer.
if (type == MIRType::
Double) {
auto* ins = MWasmBuiltinModD::
New(alloc(), lhs, rhs, instancePointer_,
type, bytecodeOffset());
curBlock_->add(ins);
return ins;
}
auto* ins = MMod::
New(alloc(), lhs, rhs, type, unsignd, trapOnError,
trapSiteDesc());
curBlock_->add(ins);
return ins;
}
MDefinition* bitnot(MDefinition* op, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MBitNot::
New(alloc(), op, type);
curBlock_->add(ins);
return ins;
}
MDefinition* select(MDefinition* trueExpr, MDefinition* falseExpr,
MDefinition* condExpr) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MWasmSelect::
New(alloc(), trueExpr, falseExpr, condExpr);
curBlock_->add(ins);
return ins;
}
MDefinition* extendI32(MDefinition* op,
bool isUnsigned) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MExtendInt32ToInt64::
New(alloc(), op, isUnsigned);
curBlock_->add(ins);
return ins;
}
MDefinition* signExtend(MDefinition* op, uint32_t srcSize,
uint32_t targetSize) {
if (inDeadCode()) {
return nullptr;
}
MInstruction* ins;
switch (targetSize) {
case 4: {
MSignExtendInt32::Mode mode;
switch (srcSize) {
case 1:
mode = MSignExtendInt32::Byte;
break;
case 2:
mode = MSignExtendInt32::Half;
break;
default:
MOZ_CRASH(
"Bad sign extension");
}
ins = MSignExtendInt32::
New(alloc(), op, mode);
break;
}
case 8: {
MSignExtendInt64::Mode mode;
switch (srcSize) {
case 1:
mode = MSignExtendInt64::Byte;
break;
case 2:
mode = MSignExtendInt64::Half;
break;
case 4:
mode = MSignExtendInt64::Word;
break;
default:
MOZ_CRASH(
"Bad sign extension");
}
ins = MSignExtendInt64::
New(alloc(), op, mode);
break;
}
default: {
MOZ_CRASH(
"Bad sign extension");
}
}
curBlock_->add(ins);
return ins;
}
MDefinition* wrapI32(MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MWrapInt64ToInt32::
New(alloc(), op,
/*bottomHalf=*/true);
curBlock_->add(ins);
return ins;
}
MDefinition* convertI64ToFloatingPoint(MDefinition* op, MIRType type,
bool isUnsigned) {
if (inDeadCode()) {
return nullptr;
}
#if defined(JS_CODEGEN_ARM)
auto* ins = MBuiltinInt64ToFloatingPoint::
New(
alloc(), op, instancePointer_, type, bytecodeOffset(), isUnsigned);
#else
auto* ins = MInt64ToFloatingPoint::
New(alloc(), op, type, bytecodeOffset(),
isUnsigned);
#endif
curBlock_->add(ins);
return ins;
}
MDefinition* rotate(MDefinition* input, MDefinition* count, MIRType type,
bool left) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MRotate::
New(alloc(), input, count, type, left);
curBlock_->add(ins);
return ins;
}
template <
class T>
MDefinition* truncate(MDefinition* op, TruncFlags flags) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = T::
New(alloc(), op, flags, trapSiteDesc());
curBlock_->add(ins);
return ins;
}
#if defined(JS_CODEGEN_ARM)
MDefinition* truncateWithInstance(MDefinition* op, TruncFlags flags) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MWasmBuiltinTruncateToInt64::
New(alloc(), op, instancePointer_,
flags, trapSiteDesc());
curBlock_->add(ins);
return ins;
}
#endif
MDefinition* compare(MDefinition* lhs, MDefinition* rhs, JSOp op,
MCompare::CompareType type) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MCompare::NewWasm(alloc(), lhs, rhs, op, type);
curBlock_->add(ins);
return ins;
}
void assign(
unsigned slot, MDefinition* def) {
if (inDeadCode()) {
return;
}
curBlock_->setSlot(info().localSlot(slot), def);
}
MDefinition* compareIsNull(MDefinition* ref, JSOp compareOp) {
MDefinition* nullVal = constantNullRef(MaybeRefType());
if (!nullVal) {
return nullptr;
}
return compare(ref, nullVal, compareOp, MCompare::Compare_WasmAnyRef);
}
[[nodiscard]] MDefinition* refAsNonNull(MDefinition* ref) {
MOZ_ASSERT(!inDeadCode());
auto* ins = MWasmRefAsNonNull::
New(alloc(), ref, trapSiteDesc());
if (!ins) {
return nullptr;
}
curBlock_->add(ins);
return ins;
}
[[nodiscard]]
bool brOnNull(uint32_t relativeDepth,
const DefVector& values,
const ResultType& type, MDefinition* condition,
BranchHint branchHint) {
if (inDeadCode()) {
return true;
}
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
MDefinition* check = compareIsNull(condition, JSOp::Eq);
if (!check) {
return false;
}
MTest* test = MTest::
New(alloc(), check, nullptr, fallthroughBlock);
if (!test || !addControlFlowPatch(test, relativeDepth,
MTest::TrueBranchIndex, branchHint)) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(test);
curBlock_ = fallthroughBlock;
return true;
}
[[nodiscard]]
bool brOnNonNull(uint32_t relativeDepth,
const DefVector& values,
const ResultType& type, MDefinition* condition,
BranchHint branchHint) {
if (inDeadCode()) {
return true;
}
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
MDefinition* check = compareIsNull(condition, JSOp::Ne);
if (!check) {
return false;
}
MTest* test = MTest::
New(alloc(), check, nullptr, fallthroughBlock);
if (!test || !addControlFlowPatch(test, relativeDepth,
MTest::TrueBranchIndex, branchHint)) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(test);
curBlock_ = fallthroughBlock;
return true;
}
MDefinition* refI31(MDefinition* input) {
auto* ins = MWasmNewI31Ref::
New(alloc(), input);
curBlock_->add(ins);
return ins;
}
MDefinition* i31Get(MDefinition* input, FieldWideningOp wideningOp) {
auto* ins = MWasmI31RefGet::
New(alloc(), input, wideningOp);
curBlock_->add(ins);
return ins;
}
#ifdef ENABLE_WASM_SIMD
// About Wasm SIMD as supported by Ion:
//
// The expectation is that Ion will only ever support SIMD on x86 and x64,
// since ARMv7 will cease to be a tier-1 platform soon, and MIPS64 will never
// implement SIMD.
//
// The division of the operations into MIR nodes reflects that expectation,
// and is a good fit for x86/x64. Should the expectation change we'll
// possibly want to re-architect the SIMD support to be a little more general.
//
// Most SIMD operations map directly to a single MIR node that ultimately ends
// up being expanded in the macroassembler.
//
// Some SIMD operations that do have a complete macroassembler expansion are
// open-coded into multiple MIR nodes here; in some cases that's just
// convenience, in other cases it may also allow them to benefit from Ion
// optimizations. The reason for the expansions will be documented by a
// comment.
// (v128,v128) -> v128 effect-free binary operations
MDefinition* binarySimd128(MDefinition* lhs, MDefinition* rhs,
bool commutative, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(lhs->type() == MIRType::Simd128 &&
rhs->type() == MIRType::Simd128);
auto* ins = MWasmBinarySimd128::
New(alloc(), lhs, rhs, commutative, op);
curBlock_->add(ins);
return ins;
}
// (v128,i32) -> v128 effect-free shift operations
MDefinition* shiftSimd128(MDefinition* lhs, MDefinition* rhs, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(lhs->type() == MIRType::Simd128 &&
rhs->type() == MIRType::Int32);
int32_t maskBits;
if (MacroAssembler::MustMaskShiftCountSimd128(op, &maskBits)) {
MDefinition* mask = constantI32(maskBits);
auto* rhs2 = MBitAnd::
New(alloc(), rhs, mask, MIRType::Int32);
curBlock_->add(rhs2);
rhs = rhs2;
}
auto* ins = MWasmShiftSimd128::
New(alloc(), lhs, rhs, op);
curBlock_->add(ins);
return ins;
}
// (v128,scalar,imm) -> v128
MDefinition* replaceLaneSimd128(MDefinition* lhs, MDefinition* rhs,
uint32_t laneIndex, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(lhs->type() == MIRType::Simd128);
auto* ins = MWasmReplaceLaneSimd128::
New(alloc(), lhs, rhs, laneIndex, op);
curBlock_->add(ins);
return ins;
}
// (scalar) -> v128 effect-free unary operations
MDefinition* scalarToSimd128(MDefinition* src, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MWasmScalarToSimd128::
New(alloc(), src, op);
curBlock_->add(ins);
return ins;
}
// (v128) -> v128 effect-free unary operations
MDefinition* unarySimd128(MDefinition* src, SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(src->type() == MIRType::Simd128);
auto* ins = MWasmUnarySimd128::
New(alloc(), src, op);
curBlock_->add(ins);
return ins;
}
// (v128, imm) -> scalar effect-free unary operations
MDefinition* reduceSimd128(MDefinition* src, SimdOp op, ValType outType,
uint32_t imm =
0) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(src->type() == MIRType::Simd128);
auto* ins =
MWasmReduceSimd128::
New(alloc(), src, op, outType.toMIRType(), imm);
curBlock_->add(ins);
return ins;
}
// (v128, v128, v128) -> v128 effect-free operations
MDefinition* ternarySimd128(MDefinition* v0, MDefinition* v1, MDefinition* v2,
SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(v0->type() == MIRType::Simd128 &&
v1->type() == MIRType::Simd128 &&
v2->type() == MIRType::Simd128);
auto* ins = MWasmTernarySimd128::
New(alloc(), v0, v1, v2, op);
curBlock_->add(ins);
return ins;
}
// (v128, v128, imm_v128) -> v128 effect-free operations
MDefinition* shuffleSimd128(MDefinition* v1, MDefinition* v2, V128 control) {
if (inDeadCode()) {
return nullptr;
}
MOZ_ASSERT(v1->type() == MIRType::Simd128);
MOZ_ASSERT(v2->type() == MIRType::Simd128);
auto* ins = BuildWasmShuffleSimd128(
alloc(),
reinterpret_cast<int8_t*>(control.bytes), v1, v2);
curBlock_->add(ins);
return ins;
}
// Also see below for SIMD memory references
#endif // ENABLE_WASM_SIMD
/************************************************ Linear memory accesses */
// For detailed information about memory accesses, see "Linear memory
// addresses and bounds checking" in WasmMemory.cpp.
private:
// If the platform does not have a HeapReg, load the memory base from
// instance.
MDefinition* maybeLoadMemoryBase(uint32_t memoryIndex) {
#ifdef WASM_HAS_HEAPREG
if (memoryIndex ==
0) {
return nullptr;
}
#endif
return memoryBase(memoryIndex);
}
public:
// A value holding the memory base, whether that's HeapReg or some other
// register.
MDefinition* memoryBase(uint32_t memoryIndex) {
AliasSet aliases = !codeMeta().memories[memoryIndex].canMovingGrow()
? AliasSet::None()
: AliasSet::Load(AliasSet::WasmHeapMeta);
#ifdef WASM_HAS_HEAPREG
if (memoryIndex ==
0) {
MWasmHeapReg* base = MWasmHeapReg::
New(alloc(), aliases);
curBlock_->add(base);
return base;
}
#endif
uint32_t offset =
memoryIndex ==
0
? Instance::offsetOfMemory0Base()
: (Instance::offsetInData(
codeMeta().offsetOfMemoryInstanceData(memoryIndex) +
offsetof(MemoryInstanceData, base)));
MWasmLoadInstance* base = MWasmLoadInstance::
New(
alloc(), instancePointer_, offset, MIRType::Pointer, aliases);
curBlock_->add(base);
return base;
}
private:
// If the bounds checking strategy requires it, load the bounds check limit
// from the instance.
MWasmLoadInstance* maybeLoadBoundsCheckLimit(uint32_t memoryIndex,
MIRType type) {
MOZ_ASSERT(type == MIRType::Int32 || type == MIRType::Int64);
if (codeMeta().hugeMemoryEnabled(memoryIndex)) {
return nullptr;
}
uint32_t offset =
memoryIndex ==
0
? Instance::offsetOfMemory0BoundsCheckLimit()
: (Instance::offsetInData(
codeMeta().offsetOfMemoryInstanceData(memoryIndex) +
offsetof(MemoryInstanceData, boundsCheckLimit)));
AliasSet aliases = !codeMeta().memories[memoryIndex].canMovingGrow()
? AliasSet::None()
: AliasSet::Load(AliasSet::WasmHeapMeta);
auto* load = MWasmLoadInstance::
New(alloc(), instancePointer_, offset, type,
aliases);
curBlock_->add(load);
return load;
}
MDefinition* maybeCanonicalizeNaN(Scalar::Type accessType,
MDefinition* value) {
MOZ_ASSERT(codeMeta().isAsmJS());
// Canonicalize floating point values for differential testing.
if (Scalar::isFloatingType(accessType) &&
js::SupportDifferentialTesting()) {
auto* canonicalize = MCanonicalizeNaN::
New(alloc(), value);
curBlock_->add(canonicalize);
return canonicalize;
}
return value;
}
// Return true if the access requires an alignment check. If so, sets
// *mustAdd to true if the offset must be added to the pointer before
// checking.
bool needAlignmentCheck(MemoryAccessDesc* access, MDefinition* base,
bool* mustAdd) {
MOZ_ASSERT(!*mustAdd);
// asm.js accesses are always aligned and need no checks.
if (codeMeta().isAsmJS() || !access->isAtomic()) {
return false;
}
// If the EA is known and aligned it will need no checks.
if (base->isConstant()) {
// We only care about the low bits, so overflow is OK, as is chopping off
// the high bits of an i64 pointer.
uint32_t ptr =
0;
if (isMem64(access->memoryIndex())) {
ptr = uint32_t(base->toConstant()->toInt64());
}
else {
ptr = base->toConstant()->toInt32();
}
if (((ptr + access->offset64()) & (access->byteSize() -
1)) ==
0) {
return false;
}
}
// If the offset is aligned then the EA is just the pointer, for
// the purposes of this check.
*mustAdd = (access->offset64() & (access->byteSize() -
1)) !=
0;
return true;
}
// Fold a constant base into the offset and make the base 0, provided the
// offset stays below the guard limit. The reason for folding the base into
// the offset rather than vice versa is that a small offset can be ignored
// by both explicit bounds checking and bounds check elimination.
void foldConstantPointer(MemoryAccessDesc* access, MDefinition** base) {
PageSize pageSize = codeMeta().memories[access->memoryIndex()].pageSize();
if (pageSize != PageSize::Standard) {
return;
}
uint64_t offsetGuardLimit = GetMaxOffsetGuardLimit(
codeMeta().hugeMemoryEnabled(access->memoryIndex()), pageSize);
if ((*base)->isConstant()) {
uint64_t basePtr =
0;
if (isMem64(access->memoryIndex())) {
basePtr = uint64_t((*base)->toConstant()->toInt64());
}
else {
basePtr = uint64_t(int64_t((*base)->toConstant()->toInt32()));
}
uint64_t offset = access->offset64();
if (offset < offsetGuardLimit && basePtr < offsetGuardLimit - offset) {
offset += basePtr;
access->setOffset32(uint32_t(offset));
*base = isMem64(access->memoryIndex()) ? constantI64(int64_t(
0))
: constantI32(
0);
}
}
}
// If the offset must be added because it is large or because the true EA must
// be checked, compute the effective address, trapping on overflow.
void maybeComputeEffectiveAddress(MemoryAccessDesc* access,
MDefinition** base,
bool mustAddOffset) {
uint64_t offsetGuardLimit = GetMaxOffsetGuardLimit(
codeMeta().hugeMemoryEnabled(access->memoryIndex()),
codeMeta().memories[access->memoryIndex()].pageSize());
if (access->offset64() >= offsetGuardLimit ||
access->offset64() > UINT32_MAX || mustAddOffset ||
!JitOptions.wasmFoldOffsets) {
*base = computeEffectiveAddress(*base, access);
}
}
MWasmLoadInstance* needBoundsCheck(uint32_t memoryIndex) {
MOZ_RELEASE_ASSERT(codeMeta().memories[memoryIndex].pageSize() ==
PageSize::Standard);
#ifdef JS_64BIT
// For 32-bit base pointers:
//
// If the bounds check uses the full 64 bits of the bounds check limit, then
// the base pointer must be zero-extended to 64 bits before checking and
// wrapped back to 32-bits after Spectre masking. (And it's important that
// the value we end up with has flowed through the Spectre mask.)
//
// If the memory's max size is known to be smaller than 64K pages exactly,
// we can use a 32-bit check and avoid extension and wrapping.
bool mem32LimitIs64Bits =
isMem32(memoryIndex) &&
!codeMeta().memories[memoryIndex].boundsCheckLimitIsAlways32Bits() &&
MaxMemoryBytes(codeMeta().memories[memoryIndex].addressType(),
codeMeta().memories[memoryIndex].pageSize()) >=
0x100000000;
#else
// On 32-bit platforms we have no more than 2GB memory and the limit for a
// 32-bit base pointer is never a 64-bit value.
bool mem32LimitIs64Bits =
false;
#endif
return maybeLoadBoundsCheckLimit(memoryIndex,
mem32LimitIs64Bits || isMem64(memoryIndex)
? MIRType::Int64
: MIRType::Int32);
}
void performBoundsCheck(uint32_t memoryIndex, MDefinition** base,
MWasmLoadInstance* boundsCheckLimit) {
// At the outset, actualBase could be the result of pretty much any integer
// operation, or it could be the load of an integer constant. If its type
// is i32, we may assume the value has a canonical representation for the
// platform, see doc block in MacroAssembler.h.
MDefinition* actualBase = *base;
// Extend an i32 index value to perform a 64-bit bounds check if the memory
// can be 4GB or larger.
bool extendAndWrapIndex =
isMem32(memoryIndex) && boundsCheckLimit->type() == MIRType::Int64;
if (extendAndWrapIndex) {
auto* extended = MWasmExtendU32Index::
New(alloc(), actualBase);
curBlock_->add(extended);
actualBase = extended;
}
auto* ins = MWasmBoundsCheck::
New(alloc(), actualBase, boundsCheckLimit,
trapSiteDesc(), MWasmBoundsCheck::Memory,
memoryIndex);
curBlock_->add(ins);
actualBase = ins;
// If we're masking, then we update *base to create a dependency chain
// through the masked index. But we will first need to wrap the index
// value if it was extended above.
if (JitOptions.spectreIndexMasking) {
if (extendAndWrapIndex) {
auto* wrapped = MWasmWrapU32Index::
New(alloc(), actualBase);
curBlock_->add(wrapped);
actualBase = wrapped;
}
*base = actualBase;
}
}
// Perform all necessary checking before a wasm heap access, based on the
// attributes of the access and base pointer.
//
// For 64-bit indices on platforms that are limited to indices that fit into
// 32 bits (all 32-bit platforms and mips64), this returns a bounds-checked
// `base` that has type Int32. Lowering code depends on this and will assert
// that the base has this type. See the end of this function.
void checkOffsetAndAlignmentAndBounds(MemoryAccessDesc* access,
MDefinition** base) {
MOZ_ASSERT(!inDeadCode());
MOZ_ASSERT(!codeMeta().isAsmJS());
// Attempt to fold a constant base pointer into the offset so as to simplify
// the addressing expression. This may update *base.
foldConstantPointer(access, base);
// Determine whether an alignment check is needed and whether the offset
// must be checked too.
bool mustAddOffsetForAlignmentCheck =
false;
bool alignmentCheck =
needAlignmentCheck(access, *base, &mustAddOffsetForAlignmentCheck);
// If bounds checking or alignment checking requires it, compute the
// effective address: add the offset into the pointer and trap on overflow.
// This may update *base.
maybeComputeEffectiveAddress(access, base, mustAddOffsetForAlignmentCheck);
// Emit the alignment check if necessary; it traps if it fails.
if (alignmentCheck) {
curBlock_->add(MWasmAlignmentCheck::
New(
alloc(), *base, access->byteSize(), trapSiteDesc()));
}
// Emit the bounds check if necessary; it traps if it fails. This may
// update *base.
MWasmLoadInstance* boundsCheckLimit =
needBoundsCheck(access->memoryIndex());
if (boundsCheckLimit) {
performBoundsCheck(access->memoryIndex(), base, boundsCheckLimit);
}
#ifndef JS_64BIT
if (isMem64(access->memoryIndex())) {
// We must have had an explicit bounds check (or one was elided if it was
// proved redundant), and on 32-bit systems the index will for sure fit in
// 32 bits: the max memory is 2GB. So chop the index down to 32-bit to
// simplify the back-end.
MOZ_ASSERT((*base)->type() == MIRType::Int64);
MOZ_ASSERT(!codeMeta().hugeMemoryEnabled(access->memoryIndex()));
auto* chopped = MWasmWrapU32Index::
New(alloc(), *base);
MOZ_ASSERT(chopped->type() == MIRType::Int32);
curBlock_->add(chopped);
*base = chopped;
}
#endif
}
bool isSmallerAccessForI64(ValType result,
const MemoryAccessDesc* access) {
if (result == ValType::I64 && access->byteSize() <=
4) {
// These smaller accesses should all be zero-extending.
MOZ_ASSERT(!isSignedIntType(access->type()));
return true;
}
return false;
}
public:
bool isMem32(uint32_t memoryIndex) {
return codeMeta().memories[memoryIndex].addressType() == AddressType::I32;
}
bool isMem64(uint32_t memoryIndex) {
return codeMeta().memories[memoryIndex].addressType() == AddressType::I64;
}
bool hugeMemoryEnabled(uint32_t memoryIndex) {
return codeMeta().hugeMemoryEnabled(memoryIndex);
}
// Add the offset into the pointer to yield the EA; trap on overflow. Clears
// the offset on the memory access as a result.
MDefinition* computeEffectiveAddress(MDefinition* base,
MemoryAccessDesc* access) {
if (inDeadCode()) {
return nullptr;
}
uint64_t offset = access->offset64();
if (offset ==
0) {
return base;
}
auto* ins = MWasmAddOffset::
New(alloc(), base, offset, trapSiteDesc());
curBlock_->add(ins);
access->clearOffset();
return ins;
}
MDefinition* load(MDefinition* base, MemoryAccessDesc* access,
ValType result) {
if (inDeadCode()) {
return nullptr;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* load = nullptr;
if (codeMeta().isAsmJS()) {
MOZ_ASSERT(access->offset64() ==
0);
MWasmLoadInstance* boundsCheckLimit =
maybeLoadBoundsCheckLimit(access->memoryIndex(), MIRType::Int32);
load = MAsmJSLoadHeap::
New(alloc(), memoryBase, base, boundsCheckLimit,
access->type());
}
else {
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
load = MWasmLoad::
New(alloc(), memoryBase, base, *access,
result.toMIRType());
}
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
void store(MDefinition* base, MemoryAccessDesc* access, MDefinition* v) {
if (inDeadCode()) {
return;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* store = nullptr;
if (codeMeta().isAsmJS()) {
MOZ_ASSERT(access->offset64() ==
0);
MWasmLoadInstance* boundsCheckLimit =
maybeLoadBoundsCheckLimit(access->memoryIndex(), MIRType::Int32);
v = maybeCanonicalizeNaN(access->type(), v);
store = MAsmJSStoreHeap::
New(alloc(), memoryBase, base, boundsCheckLimit,
access->type(), v);
}
else {
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
store = MWasmStore::
New(alloc(), memoryBase, base, *access, v);
}
if (!store) {
return;
}
curBlock_->add(store);
}
MDefinition* atomicCompareExchangeHeap(MDefinition* base,
MemoryAccessDesc* access,
ValType result, MDefinition* oldv,
MDefinition* newv) {
if (inDeadCode()) {
return nullptr;
}
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
if (isSmallerAccessForI64(result, access)) {
auto* cvtOldv =
MWrapInt64ToInt32::
New(alloc(), oldv,
/*bottomHalf=*/true);
curBlock_->add(cvtOldv);
oldv = cvtOldv;
auto* cvtNewv =
MWrapInt64ToInt32::
New(alloc(), newv,
/*bottomHalf=*/true);
curBlock_->add(cvtNewv);
newv = cvtNewv;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* cas = MWasmCompareExchangeHeap::
New(
alloc(), bytecodeOffset(), memoryBase, base, *access, oldv, newv,
instancePointer_);
if (!cas) {
return nullptr;
}
curBlock_->add(cas);
if (isSmallerAccessForI64(result, access)) {
cas = MExtendInt32ToInt64::
New(alloc(), cas,
true);
curBlock_->add(cas);
}
return cas;
}
MDefinition* atomicExchangeHeap(MDefinition* base, MemoryAccessDesc* access,
ValType result, MDefinition* value) {
if (inDeadCode()) {
return nullptr;
}
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
if (isSmallerAccessForI64(result, access)) {
auto* cvtValue =
MWrapInt64ToInt32::
New(alloc(), value,
/*bottomHalf=*/true);
curBlock_->add(cvtValue);
value = cvtValue;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* xchg =
MWasmAtomicExchangeHeap::
New(alloc(), bytecodeOffset(), memoryBase,
base, *access, value, instancePointer_);
if (!xchg) {
return nullptr;
}
curBlock_->add(xchg);
if (isSmallerAccessForI64(result, access)) {
xchg = MExtendInt32ToInt64::
New(alloc(), xchg,
true);
curBlock_->add(xchg);
}
return xchg;
}
MDefinition* atomicBinopHeap(AtomicOp op, MDefinition* base,
MemoryAccessDesc* access, ValType result,
MDefinition* value) {
if (inDeadCode()) {
return nullptr;
}
checkOffsetAndAlignmentAndBounds(access, &base);
#ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
#endif
if (isSmallerAccessForI64(result, access)) {
auto* cvtValue =
MWrapInt64ToInt32::
New(alloc(), value,
/*bottomHalf=*/true);
curBlock_->add(cvtValue);
value = cvtValue;
}
MDefinition* memoryBase = maybeLoadMemoryBase(access->memoryIndex());
MInstruction* binop =
MWasmAtomicBinopHeap::
New(alloc(), bytecodeOffset(), op, memoryBase,
base, *access, value, instancePointer_);
if (!binop) {
return nullptr;
}
curBlock_->add(binop);
if (isSmallerAccessForI64(result, access)) {
binop = MExtendInt32ToInt64::
New(alloc(), binop,
true);
curBlock_->add(binop);
}
return binop;
}
#ifdef ENABLE_WASM_SIMD
MDefinition* loadSplatSimd128(Scalar::Type viewType,
const LinearMemoryAddress<MDefinition*>& addr,
wasm::SimdOp splatOp) {
if (inDeadCode()) {
return nullptr;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(),
hugeMemoryEnabled(addr.memoryIndex));
// Generate better code (on x86)
// If AVX2 is enabled, more broadcast operators are available.
if (viewType == Scalar::Float64
# if defined(JS_CODEGEN_X64) ||
defined(JS_CODEGEN_X86)
|| (js::jit::CPUInfo::IsAVX2Present() &&
(viewType == Scalar::Uint8 || viewType == Scalar::Uint16 ||
viewType == Scalar::Float32))
# endif
) {
access.setSplatSimd128Load();
return load(addr.base, &access, ValType::V128);
}
ValType resultType = ValType::I32;
if (viewType == Scalar::Float32) {
resultType = ValType::F32;
splatOp = wasm::SimdOp::F32x4Splat;
}
auto* scalar = load(addr.base, &access, resultType);
if (!inDeadCode() && !scalar) {
return nullptr;
}
return scalarToSimd128(scalar, splatOp);
}
MDefinition* loadExtendSimd128(
const LinearMemoryAddress<MDefinition*>& addr,
wasm::SimdOp op) {
if (inDeadCode()) {
return nullptr;
}
// Generate better code (on x86) by loading as a double with an
// operation that sign extends directly.
MemoryAccessDesc access(addr.memoryIndex, Scalar::Float64, addr.align,
addr.offset, trapSiteDesc(),
hugeMemoryEnabled(addr.memoryIndex));
access.setWidenSimd128Load(op);
return load(addr.base, &access, ValType::V128);
}
MDefinition* loadZeroSimd128(Scalar::Type viewType, size_t numBytes,
const LinearMemoryAddress<MDefinition*>& addr) {
if (inDeadCode()) {
return nullptr;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(),
hugeMemoryEnabled(addr.memoryIndex));
access.setZeroExtendSimd128Load();
return load(addr.base, &access, ValType::V128);
}
MDefinition* loadLaneSimd128(uint32_t laneSize,
const LinearMemoryAddress<MDefinition*>& addr,
uint32_t laneIndex, MDefinition* src) {
if (inDeadCode()) {
return nullptr;
}
MemoryAccessDesc access(addr.memoryIndex, Scalar::Simd128, addr.align,
addr.offset, trapSiteDesc(),
hugeMemoryEnabled(addr.memoryIndex));
MDefinition* memoryBase = maybeLoadMemoryBase(access.memoryIndex());
MDefinition* base = addr.base;
MOZ_ASSERT(!codeMeta().isAsmJS());
checkOffsetAndAlignmentAndBounds(&access, &base);
# ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
# endif
MInstruction* load = MWasmLoadLaneSimd128::
New(
alloc(), memoryBase, base, access, laneSize, laneIndex, src);
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
void storeLaneSimd128(uint32_t laneSize,
const LinearMemoryAddress<MDefinition*>& addr,
uint32_t laneIndex, MDefinition* src) {
if (inDeadCode()) {
return;
}
MemoryAccessDesc access(addr.memoryIndex, Scalar::Simd128, addr.align,
addr.offset, trapSiteDesc(),
hugeMemoryEnabled(addr.memoryIndex));
MDefinition* memoryBase = maybeLoadMemoryBase(access.memoryIndex());
MDefinition* base = addr.base;
MOZ_ASSERT(!codeMeta().isAsmJS());
checkOffsetAndAlignmentAndBounds(&access, &base);
# ifndef JS_64BIT
MOZ_ASSERT(base->type() == MIRType::Int32);
# endif
MInstruction* store = MWasmStoreLaneSimd128::
New(
alloc(), memoryBase, base, access, laneSize, laneIndex, src);
if (!store) {
return;
}
curBlock_->add(store);
}
#endif // ENABLE_WASM_SIMD
/************************************************ Global variable accesses */
MDefinition* loadGlobalVar(
const GlobalDesc& global) {
if (inDeadCode()) {
return nullptr;
}
MInstruction* load;
if (global.isIndirect()) {
// Pull a pointer to the value out of Instance::globalArea, then
// load from that pointer. Note that the pointer is immutable
// even though the value it points at may change, hence the use of
// |true| for the first node's |isConst| value, irrespective of
// the |isConst| formal parameter to this method. The latter
// applies to the denoted value as a whole.
auto* cellPtr = MWasmLoadInstanceDataField::
New(
alloc(), MIRType::Pointer, global.offset(),
/*isConst=*/true, instancePointer_);
curBlock_->add(cellPtr);
load = MWasmLoadGlobalCell::
New(alloc(), global.type().toMIRType(),
cellPtr, global.type());
}
else {
// Pull the value directly out of Instance::globalArea.
load = MWasmLoadInstanceDataField::
New(
alloc(), global.type().toMIRType(), global.offset(),
!global.isMutable(), instancePointer_,
global.type().toMaybeRefType());
}
curBlock_->add(load);
return load;
}
[[nodiscard]]
bool storeGlobalVar(uint32_t lineOrBytecode,
const GlobalDesc& global, MDefinition* v) {
if (inDeadCode()) {
return true;
}
if (global.isIndirect()) {
// Pull a pointer to the value out of Instance::globalArea, then
// store through that pointer.
auto* valueAddr = MWasmLoadInstanceDataField::
New(
alloc(), MIRType::Pointer, global.offset(),
/*isConstant=*/true, instancePointer_);
curBlock_->add(valueAddr);
// Handle a store to a ref-typed field specially
if (global.type().toMIRType() == MIRType::WasmAnyRef) {
// Load the previous value for the post-write barrier
MOZ_ASSERT(v->type() == MIRType::WasmAnyRef);
auto* prevValue = MWasmLoadGlobalCell::
New(alloc(), MIRType::WasmAnyRef,
valueAddr, global.type());
curBlock_->add(prevValue);
// Store the new value
auto* store =
MWasmStoreRef::
New(alloc(), instancePointer_, valueAddr,
/*valueOffset=*/0, v, AliasSet::WasmGlobalCell,
WasmPreBarrierKind::Normal);
curBlock_->add(store);
// Call the post-write barrier
return postBarrierEdgePrecise(lineOrBytecode, valueAddr, prevValue);
}
auto* store = MWasmStoreGlobalCell::
New(alloc(), v, valueAddr);
curBlock_->add(store);
return true;
}
// Or else store the value directly in Instance::globalArea.
// Handle a store to a ref-typed field specially
if (global.type().toMIRType() == MIRType::WasmAnyRef) {
// Compute the address of the ref-typed global
auto* valueAddr = MWasmDerivedPointer::
New(
alloc(), instancePointer_,
wasm::Instance::offsetInData(global.offset()));
curBlock_->add(valueAddr);
// Load the previous value for the post-write barrier
MOZ_ASSERT(v->type() == MIRType::WasmAnyRef);
auto* prevValue = MWasmLoadGlobalCell::
New(alloc(), MIRType::WasmAnyRef,
valueAddr, global.type());
curBlock_->add(prevValue);
// Store the new value
auto* store =
MWasmStoreRef::
New(alloc(), instancePointer_, valueAddr,
/*valueOffset=*/0, v, AliasSet::WasmInstanceData,
WasmPreBarrierKind::Normal);
curBlock_->add(store);
// Call the post-write barrier
return postBarrierEdgePrecise(lineOrBytecode, valueAddr, prevValue);
}
auto* store = MWasmStoreInstanceDataField::
New(alloc(), global.offset(), v,
instancePointer_);
curBlock_->add(store);
return true;
}
// Load the slot on the instance where the result of `ref.func` is cached.
// This may be null if a function reference for this function has not been
// asked for yet.
MDefinition* loadCachedRefFunc(uint32_t funcIndex) {
uint32_t exportedFuncIndex = codeMeta().findFuncExportIndex(funcIndex);
MWasmLoadInstanceDataField* refFunc = MWasmLoadInstanceDataField::
New(
alloc(), MIRType::WasmAnyRef,
codeMeta().offsetOfFuncExportInstanceData(exportedFuncIndex) +
offsetof(FuncExportInstanceData, func),
true, instancePointer_);
curBlock_->add(refFunc);
return refFunc;
}
MDefinition* loadTableField(uint32_t tableIndex,
unsigned fieldOffset,
MIRType type) {
uint32_t instanceDataOffset = wasm::Instance::offsetInData(
codeMeta().offsetOfTableInstanceData(tableIndex) + fieldOffset);
auto* load =
MWasmLoadInstance::
New(alloc(), instancePointer_, instanceDataOffset,
type, AliasSet::Load(AliasSet::WasmTableMeta));
curBlock_->add(load);
return load;
}
MDefinition* loadTableLength(uint32_t tableIndex) {
const TableDesc& table = codeMeta().tables[tableIndex];
if (table.maximumLength().isSome() &&
table.initialLength() == *table.maximumLength()) {
return table.addressType() == AddressType::I64
? constantI64(int64_t(table.initialLength()))
: constantI32(int32_t(table.initialLength()));
}
return loadTableField(tableIndex, offsetof(TableInstanceData, length),
table.addressType() == AddressType::I64
? MIRType::Int64
: MIRType::Int32);
}
MDefinition* loadTableElements(uint32_t tableIndex) {
return loadTableField(tableIndex, offsetof(TableInstanceData, elements),
MIRType::Pointer);
}
// Clamps a table address into i32 range. If the value is too large to fit in
// an i32, it will be replaced with UINT32_MAX so that it will always fail a
// 32-bit bounds check. Consider using an actual 64-bit bounds check if
// possible.
MDefinition* clampTableAddressToI32(AddressType addressType,
MDefinition* address) {
switch (addressType) {
case AddressType::I32:
return address;
case AddressType::I64:
auto* clamp = MWasmClampTable64Address::
New(alloc(), address);
if (!clamp) {
return nullptr;
}
curBlock_->add(clamp);
return clamp;
}
MOZ_CRASH(
"unknown address type");
}
MDefinition* boundsCheckAndWrapTableAddress(uint32_t tableIndex,
MDefinition* address) {
const TableDesc& table = codeMeta().tables[tableIndex];
// Load the table length and perform a bounds check with spectre index
// masking
auto* length = loadTableLength(tableIndex);
auto* check =
MWasmBoundsCheck::
New(alloc(), address, length, trapSiteDesc(),
MWasmBoundsCheck::Table, tableIndex);
curBlock_->add(check);
if (JitOptions.spectreIndexMasking) {
address = check;
}
if (table.addressType() == AddressType::I64) {
auto* address32 =
MWrapInt64ToInt32::
New(alloc(), address,
/*bottomHalf=*/true);
curBlock_->add(address32);
return address32;
}
return address;
};
MDefinition* tableGetAnyRef(uint32_t tableIndex, MDefinition* address) {
const TableDesc& table = codeMeta().tables[tableIndex];
auto* address32 = boundsCheckAndWrapTableAddress(tableIndex, address);
// Load the table elements and load the element
auto* elements = loadTableElements(tableIndex);
auto* element = MWasmLoadTableElement::
New(alloc(), elements, address32,
table.elemType());
curBlock_->add(element);
return element;
}
[[nodiscard]]
bool tableSetAnyRef(uint32_t tableIndex, MDefinition* address,
MDefinition* value,
uint32_t lineOrBytecode) {
const TableDesc& table = codeMeta().tables[tableIndex];
auto* address32 = boundsCheckAndWrapTableAddress(tableIndex, address);
// Load the table elements
auto* elements = loadTableElements(tableIndex);
// Load the previous value
auto* prevValue = MWasmLoadTableElement::
New(alloc(), elements, address32,
table.elemType());
curBlock_->add(prevValue);
// Compute the value's location for the post barrier
auto* loc = MWasmDerivedIndexPointer::
New(alloc(), elements, address32,
ScalePointer);
curBlock_->add(loc);
// Store the new value
auto* store = MWasmStoreRef::
New(
alloc(), instancePointer_, loc,
/*valueOffset=*/0, value,
AliasSet::WasmTableElement, WasmPreBarrierKind::Normal);
curBlock_->add(store);
// Perform the post barrier
return postBarrierEdgePrecise(lineOrBytecode, loc, prevValue);
}
void addInterruptCheck() {
if (inDeadCode()) {
return;
}
curBlock_->add(
MWasmInterruptCheck::
New(alloc(), instancePointer_, trapSiteDesc()));
}
// Perform a post-write barrier to update the generational store buffer. This
// version stores the entire containing object (e.g. a struct) rather than a
// single edge.
[[nodiscard]]
bool postBarrierWholeCell(uint32_t lineOrBytecode,
MDefinition* object,
MDefinition* newValue) {
auto* barrier = MWasmPostWriteBarrierWholeCell::
New(
alloc(), instancePointer_, object, newValue);
if (!barrier) {
return false;
}
curBlock_->add(barrier);
return true;
}
// Perform a post-write barrier to update the generational store buffer. This
// version tracks a single tenured -> nursery edge, and will remove a previous
// store buffer entry if it is no longer needed.
[[nodiscard]]
bool postBarrierEdgePrecise(uint32_t lineOrBytecode,
MDefinition* valueAddr,
MDefinition* value) {
return emitInstanceCall2(lineOrBytecode, SASigPostBarrierEdgePrecise,
valueAddr, value);
}
// Perform a post-write barrier to update the generational store buffer. This
// version does not remove a previous store buffer entry if it is no longer
// needed.
[[nodiscard]]
bool postBarrierEdgeAtIndex(uint32_t lineOrBytecode,
MDefinition* object,
MDefinition* valueBase,
MDefinition* index, uint32_t scale,
MDefinition* newValue) {
auto* barrier = MWasmPostWriteBarrierEdgeAtIndex::
New(
alloc(), instancePointer_, object, valueBase, index, scale, newValue);
if (!barrier) {
return false;
}
curBlock_->add(barrier);
return true;
}
/***************************************************************** Calls */
// The IonMonkey backend maintains a single stack offset (from the stack
// pointer to the base of the frame) by adding the total amount of spill
// space required plus the maximum stack required for argument passing.
// Since we do not use IonMonkey's MPrepareCall/MPassArg/MCall, we must
// manually accumulate, for the entire function, the maximum required stack
// space for argument passing. (This is passed to the CodeGenerator via
// MIRGenerator::maxWasmStackArgBytes.) This is just be the maximum of the
// stack space required for each individual call (as determined by the call
// ABI).
[[nodiscard]]
bool passInstanceCallArg(MIRType instanceType, CallCompileState* callState) {
if (inDeadCode()) {
return true;
}
// Should only pass an instance once. And it must be a non-GC pointer.
MOZ_ASSERT(callState->instanceArg == ABIArg());
MOZ_ASSERT(instanceType == MIRType::Pointer);
callState->instanceArg = callState->abi.next(MIRType::Pointer);
return true;
}
// Do not call this directly. Call one of the passCallArg() variants instead.
[[nodiscard]]
bool passCallArgWorker(MDefinition* argDef, MIRType type,
CallCompileState* callState) {
MOZ_ASSERT(argDef->type() == type);
// Calling a softFP function requires moving our floats into GPRs.
if (!callState->hardFP &&
(type == MIRType::
Double || type == MIRType::Float32)) {
MIRType softType =
(type == MIRType::
Double) ? MIRType::Int64 : MIRType::Int32;
auto* softDef = MReinterpretCast::
New(alloc(), argDef, softType);
if (!softDef) {
return false;
}
curBlock_->add(softDef);
argDef = softDef;
}
ABIArg arg = callState->abi.next(type);
switch (arg.kind()) {
#ifdef JS_CODEGEN_REGISTER_PAIR
case ABIArg::GPR_PAIR: {
auto mirLow =
MWrapInt64ToInt32::
New(alloc(), argDef,
/* bottomHalf = */ true);
curBlock_->add(mirLow);
auto mirHigh =
MWrapInt64ToInt32::
New(alloc(), argDef,
/* bottomHalf = */ false);
curBlock_->add(mirHigh);
return callState->regArgs.append(
MWasmCallBase::Arg(AnyRegister(arg.gpr64().low), mirLow)) &&
callState->regArgs.append(
MWasmCallBase::Arg(AnyRegister(arg.gpr64().high), mirHigh));
}
#endif
case ABIArg::GPR:
case ABIArg::FPU:
return callState->regArgs.append(MWasmCallBase::Arg(arg.reg(), argDef));
case ABIArg::Stack: {
auto* mir =
MWasmStackArg::
New(alloc(), arg.offsetFromArgBase(), argDef);
curBlock_->add(mir);
return true;
}
case ABIArg::Uninitialized:
MOZ_ASSERT_UNREACHABLE(
"Uninitialized ABIArg kind");
}
MOZ_CRASH(
"Unknown ABIArg kind.");
}
template <
typename VecT>
[[nodiscard]]
bool passCallArgs(
const DefVector& argDefs,
const VecT& types,
CallCompileState* callState) {
MOZ_ASSERT(argDefs.length() == types.length());
for (uint32_t i =
0; i < argDefs.length(); i++) {
MDefinition* def = argDefs[i];
ValType type = types[i];
if (!passCallArg(def, type, callState)) {
return false;
}
}
return true;
}
[[nodiscard]]
bool passCallArg(MDefinition* argDef, MIRType type,
CallCompileState* callState) {
if (inDeadCode()) {
return true;
}
return passCallArgWorker(argDef, type, callState);
}
[[nodiscard]]
bool passCallArg(MDefinition* argDef, ValType type,
CallCompileState* callState) {
if (inDeadCode()) {
return true;
}
return passCallArgWorker(argDef, type.toMIRType(), callState);
}
// If the call returns results on the stack, prepare a stack area to receive
// them, and pass the address of the stack area to the callee as an additional
// argument.
[[nodiscard]]
bool passStackResultAreaCallArg(
const ResultType& resultType,
CallCompileState* callState) {
if (inDeadCode()) {
return true;
}
ABIResultIter iter(resultType);
while (!iter.done() && iter.cur().inRegister()) {
iter.next();
}
if (iter.done()) {
// No stack results.
return true;
}
// The builtin ABI only supports a single result value, so it doesn't
// use stack results.
MOZ_ASSERT(callState->abiKind == ABIKind::Wasm);
auto* stackResultArea = MWasmStackResultArea::
New(alloc());
if (!stackResultArea) {
return false;
}
if (!stackResultArea->init(alloc(), iter.remaining())) {
return false;
}
for (uint32_t base = iter.index(); !iter.done(); iter.next()) {
MWasmStackResultArea::StackResult loc(iter.cur().stackOffset(),
iter.cur().type().toMIRType());
stackResultArea->initResult(iter.index() - base, loc);
}
curBlock_->add(stackResultArea);
MDefinition* def = callState->returnCall ? (MDefinition*)stackResultPointer_
: (MDefinition*)stackResultArea;
if (!passCallArg(def, MIRType::StackResults, callState)) {
return false;
}
callState->stackResultArea = stackResultArea;
return true;
}
[[nodiscard]]
bool finishCallArgs(CallCompileState* callState) {
if (inDeadCode()) {
return true;
}
if (!callState->regArgs.append(
MWasmCallBase::Arg(AnyRegister(InstanceReg), instancePointer_))) {
return false;
}
uint32_t stackBytes = callState->abi.stackBytesConsumedSoFar();
maxStackArgBytes_ = std::max(maxStackArgBytes_, stackBytes);
return true;
}
[[nodiscard]]
bool emitCallArgs(
const FuncType& funcType,
const DefVector& args,
CallCompileState* callState) {
for (size_t i =
0, n = funcType.args().length(); i < n; ++i) {
if (!mirGen().ensureBallast()) {
return false;
}
if (!passCallArg(args[i], funcType.args()[i], callState)) {
return false;
}
}
ResultType resultType = ResultType::Vector(funcType.results());
if (!passStackResultAreaCallArg(resultType, callState)) {
return false;
}
return finishCallArgs(callState);
}
[[nodiscard]]
bool collectBuiltinCallResult(MIRType type, MDefinition** result,
CallCompileState* callState) {
MInstruction* def;
switch (type) {
case MIRType::Int32:
def = MWasmRegisterResult::
New(alloc(), MIRType::Int32, ReturnReg);
break;
case MIRType::Int64:
def = MWasmRegister64Result::
New(alloc(), ReturnReg64);
break;
case MIRType::Float32: {
if (callState->abiKind == ABIKind::System) {
def = MWasmSystemFloatRegisterResult::
New(
alloc(), type, ReturnFloat32Reg, callState->hardFP);
}
else {
def = MWasmFloatRegisterResult::
New(alloc(), MIRType::Float32,
ReturnFloat32Reg);
}
break;
}
case MIRType::
Double: {
if (callState->abiKind == ABIKind::System) {
def = MWasmSystemFloatRegisterResult::
New(
alloc(), type, ReturnDoubleReg, callState->hardFP);
}
else {
def = MWasmFloatRegisterResult::
New(alloc(), MIRType::
Double,
ReturnDoubleReg);
}
break;
}
#ifdef ENABLE_WASM_SIMD
case MIRType::Simd128:
MOZ_CRASH(
"SIMD128 not supported in builtin ABI");
#endif
case MIRType::WasmAnyRef:
def = MWasmRegisterResult::
New(alloc(), MIRType::WasmAnyRef, ReturnReg);
break;
case MIRType::None:
MOZ_ASSERT(result == nullptr,
"Not expecting any results created");
return true;
default:
MOZ_CRASH(
"unexpected MIRType result for builtin call");
}
if (!def) {
return false;
}
curBlock_->add(def);
*result = def;
return true;
}
[[nodiscard]]
bool collectWasmCallResults(
const ResultType& type,
CallCompileState* callState, DefVector* results) {
// This function uses wasm::ABIResultIter which does not handle the system
// ABI. Use collectBuiltinCallResult instead for builtin calls.
MOZ_ASSERT(callState->abiKind == ABIKind::Wasm);
MOZ_ASSERT(callState->hardFP);
if (!results->reserve(type.length())) {
return false;
}
// The result iterator goes in the order in which results would be popped
// off; we want the order in which they would be pushed.
ABIResultIter iter(type);
uint32_t stackResultCount =
0;
while (!iter.done()) {
if (iter.cur().onStack()) {
stackResultCount++;
}
iter.next();
}
for (iter.switchToPrev(); !iter.done(); iter.prev()) {
if (!mirGen().ensureBallast()) {
return false;
}
const ABIResult& result = iter.cur();
MInstruction* def;
if (result.inRegister()) {
switch (result.type().kind()) {
case wasm::ValType::I32:
def =
MWasmRegisterResult::
New(alloc(), MIRType::Int32, result.gpr());
break;
case wasm::ValType::I64:
def = MWasmRegister64Result::
New(alloc(), result.gpr64());
break;
case wasm::ValType::F32:
def = MWasmFloatRegisterResult::
New(alloc(), MIRType::Float32,
result.fpr());
break;
case wasm::ValType::F64:
def = MWasmFloatRegisterResult::
New(alloc(), MIRType::
Double,
result.fpr());
break;
case wasm::ValType::Ref:
def = MWasmRegisterResult::
New(alloc(), MIRType::WasmAnyRef,
result.gpr(),
result.type().toMaybeRefType());
break;
case wasm::ValType::V128:
#ifdef ENABLE_WASM_SIMD
def = MWasmFloatRegisterResult::
New(alloc(), MIRType::Simd128,
result.fpr());
#else
return this->iter().fail(
"Ion has no SIMD support yet");
#endif
}
}
else {
MOZ_ASSERT(callState->stackResultArea);
MOZ_ASSERT(stackResultCount);
uint32_t idx = --stackResultCount;
def = MWasmStackResult::
New(alloc(), callState->stackResultArea, idx);
}
if (!def) {
return false;
}
curBlock_->add(def);
results->infallibleAppend(def);
}
MOZ_ASSERT(results->length() == type.length());
return true;
}
[[nodiscard]]
bool call(CallCompileState* callState,
const CallSiteDesc& desc,
const CalleeDesc& callee,
const ArgTypeVector& argTypes,
MDefinition* addressOrRef = nullptr) {
if (!beginCatchableCall(callState)) {
return false;
}
MInstruction* ins;
if (callState->isCatchable()) {
ins = MWasmCallCatchable::
New(
alloc(), desc, callee, callState->regArgs,
StackArgAreaSizeUnaligned(argTypes, callState->abiKind),
callState->tryNoteIndex, callState->fallthroughBlock,
callState->prePadBlock, addressOrRef);
}
else {
ins = MWasmCallUncatchable::
New(
alloc(), desc, callee, callState->regArgs,
StackArgAreaSizeUnaligned(argTypes, callState->abiKind),
addressOrRef);
}
if (!ins) {
return false;
}
curBlock_->add(ins);
return finishCatchableCall(callState);
}
[[nodiscard]]
CallRefHint auditInlineableCallees(InliningHeuristics::CallKind kind,
CallRefHint hints) {
// Takes candidates for inlining as provided in `hints`, and returns a
// subset (or all) of them for which inlining is approved. To indicate
// that they are all disallowed, return an empty CallRefHint.
MOZ_ASSERT_IF(kind == InliningHeuristics::CallKind::Direct,
hints.length() ==
1);
// We only support inlining when lazy tiering. This is currently a
// requirement because we need the full module bytecode and function
// definition ranges, which are not available in other modes.
if (compilerEnv().mode() != CompileMode::LazyTiering) {
return CallRefHint();
}
// We don't support asm.js and inlining. asm.js also doesn't support
// baseline, which is required for lazy tiering, so we should never get
// here. The biggest complication for asm.js is getting correct stack
// traces with inlining.
MOZ_ASSERT(!codeMeta().isAsmJS());
// If we were given no candidates, give up now.
if (hints.empty()) {
return CallRefHint();
}
// We can't inline if we've exceeded our per-root-function inlining
// budget.
//
// This logic will cause `availableBudget` to be driven slightly negative
// if a budget overshoot happens, so we will have performed slightly more
// inlining than allowed by the initial setting of `availableBudget`. The
// size of this overshoot is however very limited -- it can't exceed the
// size of three function bodies that are inlined (3 because that's what
// CallRefHint can hold). And the max size of an inlineable function body
// is limited by InliningHeuristics::isSmallEnoughToInline.
if (rootCompiler_.inliningBudget() <
0) {
return CallRefHint();
}
// Check each candidate in turn, and add all acceptable ones to `filtered`.
// It is important that `filtered` retains the same ordering as `hints`.
CallRefHint filtered;
for (uint32_t i =
0; i < hints.length(); i++) {
uint32_t funcIndex = hints.get(i);
// We can't inline an imported function.
if (codeMeta().funcIsImport(funcIndex)) {
continue;
}
// We do not support inlining a callee which uses tail calls
FeatureUsage funcFeatureUsage =
codeTailMeta()->funcDefFeatureUsage(funcIndex);
if (funcFeatureUsage & FeatureUsage::ReturnCall) {
continue;
}
// Ask the heuristics system if we're allowed to inline a function of
// this size and kind at the current inlining depth.
uint32_t inlineeBodySize = codeTailMeta()->funcDefRange(funcIndex).size();
uint32_t rootFunctionBodySize = rootCompiler_.func().bytecodeSize();
bool largeFunctionBackoff;
bool smallEnough = InliningHeuristics::isSmallEnoughToInline(
kind, inliningDepth(), inlineeBodySize, rootFunctionBodySize,
&largeFunctionBackoff);
if (largeFunctionBackoff) {
rootCompiler_.noteLargeFunctionBackoffWasApplied();
}
if (!smallEnough) {
continue;
}
filtered.append(funcIndex);
}
// Whatever ends up in `filtered` is approved for inlining.
return filtered;
}
[[nodiscard]]
bool finishInlinedCallDirect(FunctionCompiler& calleeCompiler,
DefVector* results) {
const PendingInlineReturnVector& calleeReturns =
calleeCompiler.pendingInlineReturns_;
MBasicBlock* calleeCatchBlock = calleeCompiler.pendingInlineCatchBlock_;
const FuncType& calleeFuncType = calleeCompiler.funcType();
MBasicBlock* lastBlockBeforeCall = curBlock_;
// Add the observed features from the inlined function to this function
iter_.addFeatureUsage(calleeCompiler.featureUsage());
// Create a block, if needed, to handle exceptions from the callee function
if (calleeCatchBlock) {
ControlInstructionVector* tryLandingPadPatches;
bool inTryCode = inTryBlock(&tryLandingPadPatches);
// The callee compiler should never create a catch block unless we have
// a landing pad for it
MOZ_RELEASE_ASSERT(inTryCode);
// Create a block in our function to jump to the nearest try block. We
// cannot just use the callee's catch block for this, as the slots on it
// are set up for all the locals from that function. We need to create a
// new block in our function with the slots for this function, that then
// does the jump to the landing pad. Ion should be able to optimize this
// away using jump threading.
MBasicBlock* callerCatchBlock = nullptr;
if (!newBlock(nullptr, &callerCatchBlock)) {
return false;
}
// Our catch block inherits all of the locals state from immediately
// before the inlined call
callerCatchBlock->inheritSlots(lastBlockBeforeCall);
// The callee catch block jumps to our catch block
calleeCatchBlock->end(MGoto::
New(alloc(), callerCatchBlock));
// Our catch block has the callee rethrow block as a predecessor, but
// ignores all phi's, because we use our own locals state.
if (!callerCatchBlock->addPredecessorWithoutPhis(calleeCatchBlock)) {
return false;
}
// Our catch block ends with a patch to jump to the enclosing try block.
MBasicBlock* prevBlock = curBlock_;
curBlock_ = callerCatchBlock;
if (!endWithPadPatch(tryLandingPadPatches)) {
return false;
}
curBlock_ = prevBlock;
}
// If there were no returns, then we are now in dead code
if (calleeReturns.empty()) {
curBlock_ = nullptr;
return true;
}
// Create a block to join all of the returns from the inlined function
MBasicBlock* joinAfterCall = nullptr;
if (!newBlock(nullptr, &joinAfterCall)) {
return false;
}
// The join block inherits all of the locals state from immediately before
// the inlined call
joinAfterCall->inheritSlots(lastBlockBeforeCall);
// The join block has a phi node for every result of the inlined function
// type. Each phi node has an operand for each of the returns of the
// inlined function.
for (uint32_t i =
0; i < calleeFuncType.results().length(); i++) {
MPhi* phi = MPhi::
New(alloc(), calleeFuncType.results()[i].toMIRType());
if (!phi || !phi->reserveLength(calleeReturns.length())) {
return false;
}
joinAfterCall->addPhi(phi);
if (!results->append(phi)) {
return false;
}
}
// Bind every return from the inlined function to go to the join block, and
// add the results for the return to the phi nodes.
for (size_t i =
0; i < calleeReturns.length(); i++) {
const PendingInlineReturn& calleeReturn = calleeReturns[i];
// Setup the predecessor and successor relationship
MBasicBlock* pred = calleeReturn.jump->block();
if (!joinAfterCall->addPredecessorWithoutPhis(pred)) {
return false;
}
calleeReturn.jump->replaceSuccessor(MGoto::TargetIndex, joinAfterCall);
// For each result in this return, add it to the corresponding phi node
for (uint32_t resultIndex =
0;
resultIndex < calleeFuncType.results().length(); resultIndex++) {
MDefinition* result = (*results)[resultIndex];
((MPhi*)(result))->addInput(calleeReturn.results[resultIndex]);
}
}
// Continue MIR generation starting in the join block
curBlock_ = joinAfterCall;
return true;
}
[[nodiscard]]
bool callDirect(
const FuncType& funcType, uint32_t funcIndex,
uint32_t lineOrBytecode,
const DefVector& args,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CallCompileState callState(ABIKind::Wasm);
CallSiteDesc desc(lineOrBytecode, rootCompiler_.inlinedCallerOffsetsIndex(),
CallSiteKind::Func);
ResultType resultType = ResultType::Vector(funcType.results());
auto callee = CalleeDesc::function(funcIndex);
ArgTypeVector argTypes(funcType);
return emitCallArgs(funcType, args, &callState) &&
call(&callState, desc, callee, argTypes) &&
collectWasmCallResults(resultType, &callState, results);
}
[[nodiscard]]
bool returnCallDirect(
const FuncType& funcType, uint32_t funcIndex,
uint32_t lineOrBytecode,
const DefVector& args,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
// We do not support tail calls in inlined functions.
MOZ_RELEASE_ASSERT(!isInlined());
CallCompileState callState(ABIKind::Wasm);
callState.returnCall =
true;
CallSiteDesc desc(lineOrBytecode, CallSiteKind::ReturnFunc);
auto callee = CalleeDesc::function(funcIndex);
ArgTypeVector argTypes(funcType);
if (!emitCallArgs(funcType, args, &callState)) {
return false;
}
auto ins = MWasmReturnCall::
New(
alloc(), desc, callee, callState.regArgs,
StackArgAreaSizeUnaligned(argTypes, callState.abiKind), nullptr);
if (!ins) {
return false;
}
curBlock_->end(ins);
curBlock_ = nullptr;
return true;
}
[[nodiscard]]
bool returnCallImport(
unsigned globalDataOffset, uint32_t lineOrBytecode,
const FuncType& funcType,
const DefVector& args,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
// We do not support tail calls in inlined functions.
MOZ_RELEASE_ASSERT(!isInlined());
CallCompileState callState(ABIKind::Wasm);
callState.returnCall =
true;
CallSiteDesc desc(lineOrBytecode, CallSiteKind::Import);
auto callee = CalleeDesc::import(globalDataOffset);
ArgTypeVector argTypes(funcType);
if (!emitCallArgs(funcType, args, &callState)) {
return false;
}
auto* ins = MWasmReturnCall::
New(
alloc(), desc, callee, callState.regArgs,
StackArgAreaSizeUnaligned(argTypes, callState.abiKind), nullptr);
if (!ins) {
return false;
}
curBlock_->end(ins);
curBlock_ = nullptr;
return true;
}
[[nodiscard]]
bool returnCallIndirect(uint32_t funcTypeIndex, uint32_t tableIndex,
MDefinition* address, uint32_t lineOrBytecode,
const DefVector& args, DefVector* results) {
MOZ_ASSERT(!inDeadCode());
// We do not support tail calls in inlined functions.
MOZ_RELEASE_ASSERT(!isInlined());
const FuncType& funcType = (*codeMeta().types)[funcTypeIndex].funcType();
CallIndirectId callIndirectId =
CallIndirectId::forFuncType(codeMeta(), funcTypeIndex);
CallCompileState callState(ABIKind::Wasm);
callState.returnCall =
true;
CalleeDesc callee;
MOZ_ASSERT(callIndirectId.kind() != CallIndirectIdKind::AsmJS);
const TableDesc& table = codeMeta().tables[tableIndex];
callee =
CalleeDesc::wasmTable(codeMeta(), table, tableIndex, callIndirectId);
CallSiteDesc desc(lineOrBytecode, CallSiteKind::Indirect);
ArgTypeVector argTypes(funcType);
if (!emitCallArgs(funcType, args, &callState)) {
return false;
}
auto* address32 = boundsCheckAndWrapTableAddress(tableIndex, address);
if (!address32) {
return false;
}
auto* ins = MWasmReturnCall::
New(
alloc(), desc, callee, callState.regArgs,
StackArgAreaSizeUnaligned(argTypes, callState.abiKind), address32);
if (!ins) {
return false;
}
curBlock_->end(ins);
curBlock_ = nullptr;
return true;
}
[[nodiscard]]
bool callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex,
MDefinition* address, uint32_t lineOrBytecode,
const DefVector& args, DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CallCompileState callState(ABIKind::Wasm);
const FuncType& funcType = (*codeMeta().types)[funcTypeIndex].funcType();
CallIndirectId callIndirectId =
CallIndirectId::forFuncType(codeMeta(), funcTypeIndex);
CalleeDesc callee;
if (codeMeta().isAsmJS()) {
MOZ_ASSERT(tableIndex ==
0);
MOZ_ASSERT(callIndirectId.kind() == CallIndirectIdKind::AsmJS);
uint32_t tableIndex = codeMeta().asmJSSigToTableIndex[funcTypeIndex];
const TableDesc& table = codeMeta().tables[tableIndex];
// ensured by asm.js validation
MOZ_ASSERT(table.initialLength() <= UINT32_MAX);
MOZ_ASSERT(std::has_single_bit(table.initialLength()));
MDefinition* mask = constantI32(int32_t(table.initialLength() -
1));
MBitAnd* maskedAddress =
MBitAnd::
New(alloc(), address, mask, MIRType::Int32);
curBlock_->add(maskedAddress);
address = maskedAddress;
callee = CalleeDesc::asmJSTable(codeMeta(), tableIndex);
}
else {
MOZ_ASSERT(callIndirectId.kind() != CallIndirectIdKind::AsmJS);
const TableDesc& table = codeMeta().tables[tableIndex];
callee =
CalleeDesc::wasmTable(codeMeta(), table, tableIndex, callIndirectId);
address = boundsCheckAndWrapTableAddress(tableIndex, address);
if (!address) {
return false;
}
}
CallSiteDesc desc(lineOrBytecode, rootCompiler_.inlinedCallerOffsetsIndex(),
CallSiteKind::Indirect);
ArgTypeVector argTypes(funcType);
ResultType resultType = ResultType::Vector(funcType.results());
return emitCallArgs(funcType, args, &callState) &&
call(&callState, desc, callee, argTypes, address) &&
collectWasmCallResults(resultType, &callState, results);
}
[[nodiscard]]
bool callImport(
unsigned instanceDataOffset, uint32_t lineOrBytecode,
const FuncType& funcType,
const DefVector& args,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CallCompileState callState(ABIKind::Wasm);
CallSiteDesc desc(lineOrBytecode, rootCompiler_.inlinedCallerOffsetsIndex(),
CallSiteKind::Import);
auto callee = CalleeDesc::import(instanceDataOffset);
ArgTypeVector argTypes(funcType);
ResultType resultType = ResultType::Vector(funcType.results());
return emitCallArgs(funcType, args, &callState) &&
call(&callState, desc, callee, argTypes) &&
collectWasmCallResults(resultType, &callState, results);
}
[[nodiscard]]
bool builtinCall(CallCompileState* callState,
const SymbolicAddressSignature& builtin,
uint32_t lineOrBytecode, MDefinition** result) {
if (inDeadCode()) {
*result = nullptr;
return true;
}
MOZ_ASSERT(builtin.failureMode == FailureMode::Infallible);
CallSiteDesc desc(lineOrBytecode, rootCompiler_.inlinedCallerOffsetsIndex(),
CallSiteKind::Symbolic);
auto callee = CalleeDesc::builtin(builtin.identity);
auto* ins = MWasmCallUncatchable::
New(
alloc(), desc, callee, callState->regArgs,
StackArgAreaSizeUnaligned(builtin, callState->abiKind));
if (!ins) {
return false;
}
curBlock_->add(ins);
return collectBuiltinCallResult(builtin.retType, result, callState);
}
[[nodiscard]]
bool builtinCall1(
const SymbolicAddressSignature& builtin,
uint32_t lineOrBytecode, MDefinition* arg,
MDefinition** result) {
CallCompileState callState(ABIForBuiltin(builtin.identity));
return passCallArg(arg, builtin.argTypes[
0], &callState) &&
finishCallArgs(&callState) &&
builtinCall(&callState, builtin, lineOrBytecode, result);
}
[[nodiscard]]
bool builtinCall2(
const SymbolicAddressSignature& builtin,
uint32_t lineOrBytecode, MDefinition* arg1,
MDefinition* arg2, MDefinition** result) {
CallCompileState callState(ABIForBuiltin(builtin.identity));
return passCallArg(arg1, builtin.argTypes[
0], &callState) &&
passCallArg(arg2, builtin.argTypes[
1], &callState) &&
finishCallArgs(&callState) &&
builtinCall(&callState, builtin, lineOrBytecode, result);
}
[[nodiscard]]
bool builtinCall5(
const SymbolicAddressSignature& builtin,
uint32_t lineOrBytecode, MDefinition* arg1,
MDefinition* arg2, MDefinition* arg3, MDefinition* arg4,
MDefinition* arg5, MDefinition** result) {
CallCompileState callState(ABIForBuiltin(builtin.identity));
return passCallArg(arg1, builtin.argTypes[
0], &callState) &&
passCallArg(arg2, builtin.argTypes[
1], &callState) &&
passCallArg(arg3, builtin.argTypes[
2], &callState) &&
passCallArg(arg4, builtin.argTypes[
3], &callState) &&
passCallArg(arg5, builtin.argTypes[
4], &callState) &&
finishCallArgs(&callState) &&
builtinCall(&callState, builtin, lineOrBytecode, result);
}
[[nodiscard]]
bool builtinCall6(
const SymbolicAddressSignature& builtin,
uint32_t lineOrBytecode, MDefinition* arg1,
MDefinition* arg2, MDefinition* arg3, MDefinition* arg4,
MDefinition* arg5, MDefinition* arg6,
MDefinition** result) {
CallCompileState callState(ABIForBuiltin(builtin.identity));
return passCallArg(arg1, builtin.argTypes[
0], &callState) &&
passCallArg(arg2, builtin.argTypes[
1], &callState) &&
passCallArg(arg3, builtin.argTypes[
2], &callState) &&
passCallArg(arg4, builtin.argTypes[
3], &callState) &&
passCallArg(arg5, builtin.argTypes[
4], &callState) &&
passCallArg(arg6, builtin.argTypes[
5], &callState) &&
finishCallArgs(&callState) &&
builtinCall(&callState, builtin, lineOrBytecode, result);
}
[[nodiscard]]
bool instanceCall(CallCompileState* callState,
const SymbolicAddressSignature& builtin,
uint32_t lineOrBytecode, MDefinition** result = nullptr) {
MOZ_ASSERT_IF(!result, builtin.retType == MIRType::None);
if (inDeadCode()) {
if (result) {
*result = nullptr;
}
return true;
}
CallSiteDesc desc(lineOrBytecode, rootCompiler_.inlinedCallerOffsetsIndex(),
CallSiteKind::Symbolic);
if (builtin.failureMode != FailureMode::Infallible &&
!beginCatchableCall(callState)) {
return false;
}
MInstruction* ins;
if (callState->isCatchable()) {
ins = MWasmCallCatchable::NewBuiltinInstanceMethodCall(
alloc(), desc, builtin.identity, builtin.failureMode,
builtin.failureTrap, callState->instanceArg, callState->regArgs,
StackArgAreaSizeUnaligned(builtin, callState->abiKind),
callState->tryNoteIndex, callState->fallthroughBlock,
callState->prePadBlock);
}
else {
ins = MWasmCallUncatchable::NewBuiltinInstanceMethodCall(
alloc(), desc, builtin.identity, builtin.failureMode,
builtin.failureTrap, callState->instanceArg, callState->regArgs,
StackArgAreaSizeUnaligned(builtin, callState->abiKind));
}
if (!ins) {
return false;
}
curBlock_->add(ins);
if (!finishCatchableCall(callState)) {
return false;
}
if (!result) {
return true;
}
return collectBuiltinCallResult(builtin.retType, result, callState);
}
/*********************************************** Instance call helpers ***/
// Do not call this function directly -- it offers no protection against
// mis-counting of arguments. Instead call one of
// ::emitInstanceCall{0,1,2,3,4,5,6}.
//
// Emits a call to the Instance function indicated by `callee`. This is
// assumed to take an Instance pointer as its first argument. The remaining
// args are taken from `args`, which is assumed to hold `numArgs` entries.
// If `result` is non-null, the MDefinition* holding the return value is
// written to `*result`.
[[nodiscard]]
bool emitInstanceCallN(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition** args, size_t numArgs,
MDefinition** result = nullptr) {
// Check that the first formal parameter is plausibly an Instance pointer.
MOZ_ASSERT(callee.numArgs >
0);
MOZ_ASSERT(callee.argTypes[
0] == MIRType::Pointer);
// Check we agree on the number of args.
MOZ_ASSERT(numArgs +
1 /* the instance pointer */ == callee.numArgs);
// Check we agree on whether a value is returned.
MOZ_ASSERT((result == nullptr) == (callee.retType == MIRType::None));
// If we are in dead code, it can happen that some of the `args` entries
// are nullptr, which will look like an OOM to the logic below. So exit
// at this point. `passInstanceCallArg`, `passCallArg`, `finishCall` and
// `instanceCall` all do nothing in dead code, so it's valid
// to exit here.
if (inDeadCode()) {
if (result) {
*result = nullptr;
}
return true;
}
// Check all args for signs of OOMness before attempting to allocating any
// more memory.
for (size_t i =
0; i < numArgs; i++) {
if (!args[i]) {
if (result) {
*result = nullptr;
}
return false;
}
}
// Finally, construct the call.
CallCompileState callState(ABIForBuiltin(callee.identity));
if (!passInstanceCallArg(callee.argTypes[
0], &callState)) {
return false;
}
for (size_t i =
0; i < numArgs; i++) {
if (!passCallArg(args[i], callee.argTypes[i +
1], &callState)) {
return false;
}
}
if (!finishCallArgs(&callState)) {
return false;
}
return instanceCall(&callState, callee, lineOrBytecode, result);
}
[[nodiscard]]
bool emitInstanceCall0(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition** result = nullptr) {
MDefinition* args[
0] = {};
return emitInstanceCallN(lineOrBytecode, callee, args,
0, result);
}
[[nodiscard]]
bool emitInstanceCall1(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition** result = nullptr) {
MDefinition* args[
1] = {arg1};
return emitInstanceCallN(lineOrBytecode, callee, args,
1, result);
}
[[nodiscard]]
bool emitInstanceCall2(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition** result = nullptr) {
MDefinition* args[
2] = {arg1, arg2};
return emitInstanceCallN(lineOrBytecode, callee, args,
2, result);
}
[[nodiscard]]
bool emitInstanceCall3(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition* arg3, MDefinition** result = nullptr) {
MDefinition* args[
3] = {arg1, arg2, arg3};
return emitInstanceCallN(lineOrBytecode, callee, args,
3, result);
}
[[nodiscard]]
bool emitInstanceCall4(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition* arg3, MDefinition* arg4,
MDefinition** result = nullptr) {
MDefinition* args[
4] = {arg1, arg2, arg3, arg4};
return emitInstanceCallN(lineOrBytecode, callee, args,
4, result);
}
[[nodiscard]]
bool emitInstanceCall5(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition* arg3, MDefinition* arg4,
MDefinition* arg5, MDefinition** result = nullptr) {
MDefinition* args[
5] = {arg1, arg2, arg3, arg4, arg5};
return emitInstanceCallN(lineOrBytecode, callee, args,
5, result);
}
[[nodiscard]]
bool emitInstanceCall6(uint32_t lineOrBytecode,
const SymbolicAddressSignature& callee,
MDefinition* arg1, MDefinition* arg2,
MDefinition* arg3, MDefinition* arg4,
MDefinition* arg5, MDefinition* arg6,
MDefinition** result = nullptr) {
MDefinition* args[
6] = {arg1, arg2, arg3, arg4, arg5, arg6};
return emitInstanceCallN(lineOrBytecode, callee, args,
6, result);
}
[[nodiscard]]
bool callRef(
const FuncType& funcType, MDefinition* ref,
uint32_t lineOrBytecode,
const DefVector& args,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
CallCompileState callState(ABIKind::Wasm);
CalleeDesc callee = CalleeDesc::wasmFuncRef();
CallSiteDesc desc(lineOrBytecode, rootCompiler_.inlinedCallerOffsetsIndex(),
CallSiteKind::FuncRef);
ArgTypeVector argTypes(funcType);
ResultType resultType = ResultType::Vector(funcType.results());
return emitCallArgs(funcType, args, &callState) &&
call(&callState, desc, callee, argTypes, ref) &&
collectWasmCallResults(resultType, &callState, results);
}
[[nodiscard]]
bool returnCallRef(
const FuncType& funcType, MDefinition* ref,
uint32_t lineOrBytecode,
const DefVector& args,
DefVector* results) {
MOZ_ASSERT(!inDeadCode());
MOZ_ASSERT(!isInlined());
CallCompileState callState(ABIKind::Wasm);
callState.returnCall =
true;
CalleeDesc callee = CalleeDesc::wasmFuncRef();
CallSiteDesc desc(lineOrBytecode, CallSiteKind::FuncRef);
ArgTypeVector argTypes(funcType);
if (!emitCallArgs(funcType, args, &callState)) {
return false;
}
auto* ins = MWasmReturnCall::
New(
alloc(), desc, callee, callState.regArgs,
StackArgAreaSizeUnaligned(argTypes, callState.abiKind), ref);
if (!ins) {
return false;
}
curBlock_->end(ins);
curBlock_ = nullptr;
return true;
}
[[nodiscard]] MDefinition* stringCast(MDefinition* string) {
auto* ins = MWasmTrapIfAnyRefIsNotJSString::
New(
alloc(), string, wasm::Trap::BadCast, trapSiteDesc());
if (!ins) {
return ins;
}
curBlock_->add(ins);
return ins;
}
[[nodiscard]] MDefinition* stringTest(MDefinition* string) {
auto* ins = MWasmAnyRefIsJSString::
New(alloc(), string);
if (!ins) {
return nullptr;
}
curBlock_->add(ins);
return ins;
}
[[nodiscard]] MDefinition* stringLength(MDefinition* string) {
auto* ins = MWasmAnyRefJSStringLength::
New(
alloc(), string, wasm::Trap::BadCast, trapSiteDesc());
if (!ins) {
return nullptr;
}
curBlock_->add(ins);
return ins;
}
[[nodiscard]]
bool dispatchInlineBuiltinModuleFunc(
const BuiltinModuleFunc& builtinModuleFunc,
const DefVector& params) {
BuiltinInlineOp inlineOp = builtinModuleFunc.inlineOp();
MOZ_ASSERT(inlineOp != BuiltinInlineOp::None);
switch (inlineOp) {
case BuiltinInlineOp::StringCast: {
MOZ_ASSERT(params.length() ==
1);
MDefinition* string = params[
0];
MDefinition* cast = stringCast(string);
if (!cast) {
return false;
}
iter().setResult(string);
return true;
}
case BuiltinInlineOp::StringTest: {
MOZ_ASSERT(params.length() ==
1);
MDefinition* string = params[
0];
MDefinition* test = stringTest(string);
if (!test) {
return false;
}
iter().setResult(test);
return true;
}
case BuiltinInlineOp::StringLength: {
MOZ_ASSERT(params.length() ==
1);
MDefinition* string = params[
0];
MDefinition* length = stringLength(string);
if (!length) {
return false;
}
iter().setResult(length);
return true;
}
case BuiltinInlineOp::None:
case BuiltinInlineOp::Limit:
break;
}
MOZ_CRASH();
}
[[nodiscard]]
bool callBuiltinModuleFunc(
const BuiltinModuleFunc& builtinModuleFunc,
const DefVector& params) {
MOZ_ASSERT(!inDeadCode());
BuiltinInlineOp inlineOp = builtinModuleFunc.inlineOp();
if (inlineOp != BuiltinInlineOp::None) {
return dispatchInlineBuiltinModuleFunc(builtinModuleFunc, params);
}
// It's almost possible to use FunctionCompiler::emitInstanceCallN here.
// Unfortunately not currently possible though, since ::emitInstanceCallN
// expects an array of arguments along with a size, and that's not what is
// available here. It would be possible if we were prepared to copy
// `builtinModuleFunc->params` into a fixed-sized (16 element?) array, add
// `memoryBase`, and make the call.
const SymbolicAddressSignature& callee = *builtinModuleFunc.sig();
CallCompileState callState(ABIForBuiltin(callee.identity));
if (!passInstanceCallArg(callee.argTypes[
0], &callState) ||
!passCallArgs(params, builtinModuleFunc.funcType()->args(),
&callState)) {
return false;
}
if (builtinModuleFunc.usesMemory()) {
if (!passCallArg(memoryBase(
0), MIRType::Pointer, &callState)) {
return false;
}
}
if (!finishCallArgs(&callState)) {
return false;
}
bool hasResult = !builtinModuleFunc.funcType()->results().empty();
MDefinition* result = nullptr;
MDefinition** resultOutParam = hasResult ? &result : nullptr;
if (!instanceCall(&callState, callee, readBytecodeOffset(),
resultOutParam)) {
return false;
}
if (hasResult) {
iter().setResult(result);
}
return true;
}
/*********************************************** Control flow generation */
inline bool inDeadCode()
const {
return curBlock_ == nullptr; }
[[nodiscard]]
bool returnValues(DefVector&& values) {
if (inDeadCode()) {
return true;
}
// If we're inlined into another function, we must accumulate the returns
// so that they can be patched into the caller function.
if (isInlined()) {
MGoto* jump = MGoto::
New(alloc());
if (!jump) {
return false;
}
curBlock_->end(jump);
curBlock_ = nullptr;
return pendingInlineReturns_.emplaceBack(
PendingInlineReturn(jump, std::move(values)));
}
if (values.empty()) {
curBlock_->end(MWasmReturnVoid::
New(alloc(), instancePointer_));
}
else {
ResultType resultType = ResultType::Vector(funcType().results());
ABIResultIter iter(resultType);
// Switch to iterate in FIFO order instead of the default LIFO.
while (!iter.done()) {
iter.next();
}
iter.switchToPrev();
for (uint32_t i =
0; !iter.done(); iter.prev(), i++) {
if (!mirGen().ensureBallast()) {
return false;
}
const ABIResult& result = iter.cur();
if (result.onStack()) {
MOZ_ASSERT(iter.remaining() >
1);
auto* store = MWasmStoreStackResult::
New(
alloc(), stackResultPointer_, result.stackOffset(), values[i]);
curBlock_->add(store);
}
else {
MOZ_ASSERT(iter.remaining() ==
1);
MOZ_ASSERT(i +
1 == values.length());
curBlock_->end(
MWasmReturn::
New(alloc(), values[i], instancePointer_));
}
}
}
curBlock_ = nullptr;
return true;
}
void unreachableTrap() {
if (inDeadCode()) {
return;
}
auto* ins =
MWasmTrap::
New(alloc(), wasm::Trap::Unreachable, trapSiteDesc());
curBlock_->end(ins);
curBlock_ = nullptr;
}
void unimplementedTrap() {
if (inDeadCode()) {
return;
}
auto* ins =
MWasmTrap::
New(alloc(), wasm::Trap::Unimplemented, trapSiteDesc());
curBlock_->end(ins);
curBlock_ = nullptr;
}
private:
static uint32_t numPushed(MBasicBlock* block) {
return block->stackDepth() - block->info().firstStackSlot();
}
public:
[[nodiscard]]
bool pushDefs(
const DefVector& defs) {
if (inDeadCode()) {
return true;
}
MOZ_ASSERT(numPushed(curBlock_) ==
0);
if (!curBlock_->ensureHasSlots(defs.length())) {
return false;
}
for (MDefinition* def : defs) {
MOZ_ASSERT(def->type() != MIRType::None);
curBlock_->push(def);
}
return true;
}
[[nodiscard]]
bool popPushedDefs(DefVector* defs) {
size_t n = numPushed(curBlock_);
if (!defs->resizeUninitialized(n)) {
return false;
}
for (; n >
0; n--) {
MDefinition* def = curBlock_->pop();
MOZ_ASSERT(def->type() != MIRType::Value);
(*defs)[n -
1] = def;
}
return true;
}
private:
[[nodiscard]]
bool addJoinPredecessor(
const DefVector& defs,
MBasicBlock** joinPred) {
*joinPred = curBlock_;
if (inDeadCode()) {
return true;
}
return pushDefs(defs);
}
public:
[[nodiscard]]
bool branchAndStartThen(MDefinition* cond,
MBasicBlock** elseBlock) {
if (inDeadCode()) {
*elseBlock = nullptr;
}
else {
MBasicBlock* thenBlock;
if (!newBlock(curBlock_, &thenBlock)) {
return false;
}
if (!newBlock(curBlock_, elseBlock)) {
return false;
}
curBlock_->end(MTest::
New(alloc(), cond, thenBlock, *elseBlock));
curBlock_ = thenBlock;
mirGraph().moveBlockToEnd(curBlock_);
}
return startBlock();
}
[[nodiscard]]
bool switchToElse(MBasicBlock* elseBlock,
MBasicBlock** thenJoinPred) {
DefVector values;
if (!finishBlock(&values)) {
return false;
}
if (!elseBlock) {
*thenJoinPred = nullptr;
}
else {
if (!addJoinPredecessor(values, thenJoinPred)) {
return false;
}
curBlock_ = elseBlock;
mirGraph().moveBlockToEnd(curBlock_);
}
return startBlock();
}
[[nodiscard]]
bool joinIfElse(MBasicBlock* thenJoinPred, DefVector* defs) {
DefVector values;
if (!finishBlock(&values)) {
return false;
}
if (!thenJoinPred && inDeadCode()) {
return true;
}
MBasicBlock* elseJoinPred;
if (!addJoinPredecessor(values, &elseJoinPred)) {
return false;
}
mozilla::Array<MBasicBlock*,
2> blocks;
size_t numJoinPreds =
0;
if (thenJoinPred) {
blocks[numJoinPreds++] = thenJoinPred;
}
if (elseJoinPred) {
blocks[numJoinPreds++] = elseJoinPred;
}
if (numJoinPreds ==
0) {
return true;
}
MBasicBlock* join;
if (!goToNewBlock(blocks[
0], &join)) {
return false;
}
for (size_t i =
1; i < numJoinPreds; ++i) {
if (!goToExistingBlock(blocks[i], join)) {
return false;
}
}
curBlock_ = join;
return popPushedDefs(defs);
}
[[nodiscard]]
bool startBlock() {
MOZ_ASSERT_IF(pendingBlockDepth_ < pendingBlocks_.length(),
pendingBlocks_[pendingBlockDepth_].patches.empty());
pendingBlockDepth_++;
return true;
}
[[nodiscard]]
bool finishBlock(DefVector* defs) {
MOZ_ASSERT(pendingBlockDepth_);
uint32_t topLabel = --pendingBlockDepth_;
return bindBranches(topLabel, defs);
}
[[nodiscard]]
bool startLoop(MBasicBlock** loopHeader, size_t paramCount) {
*loopHeader = nullptr;
pendingBlockDepth_++;
rootCompiler_.startLoop();
if (inDeadCode()) {
return true;
}
// Create the loop header.
MOZ_ASSERT(curBlock_->loopDepth() == rootCompiler_.loopDepth() -
1);
*loopHeader = MBasicBlock::
New(mirGraph(), info(), curBlock_,
MBasicBlock::PENDING_LOOP_HEADER);
if (!*loopHeader) {
return false;
}
(*loopHeader)->setLoopDepth(rootCompiler_.loopDepth());
mirGraph().addBlock(*loopHeader);
curBlock_->end(MGoto::
New(alloc(), *loopHeader));
DefVector loopParams;
if (!iter().getResults(paramCount, &loopParams)) {
return false;
}
// Eagerly create a phi for all loop params. setLoopBackedge will remove
// any that were not necessary.
for (size_t i =
0; i < paramCount; i++) {
MPhi* phi = MPhi::
New(alloc(), loopParams[i]->type());
if (!phi) {
return false;
}
if (!phi->reserveLength(
2)) {
return false;
}
(*loopHeader)->addPhi(phi);
phi->addInput(loopParams[i]);
loopParams[i] = phi;
}
iter().setResults(paramCount, loopParams);
MBasicBlock* body;
if (!goToNewBlock(*loopHeader, &body)) {
return false;
}
curBlock_ = body;
return true;
}
private:
void fixupRedundantPhis(MBasicBlock* b) {
for (size_t i =
0, depth = b->stackDepth(); i < depth; i++) {
MDefinition* def = b->getSlot(i);
if (def->isUnused()) {
b->setSlot(i, def->toPhi()->getOperand(
0));
}
}
}
[[nodiscard]]
bool setLoopBackedge(MBasicBlock* loopEntry,
MBasicBlock* loopBody,
MBasicBlock* backedge, size_t paramCount) {
if (!loopEntry->setBackedgeWasm(backedge, paramCount)) {
return false;
}
// Entering a loop will eagerly create a phi node for all locals and loop
// params. Now that we've closed the loop we can check which phi nodes
// were actually needed by checking if the SSA definition flowing into the
// loop header (operand 0) is different than the SSA definition coming from
// the loop backedge (operand 1). If they are the same definition, the phi
// is redundant and can be removed.
//
// To do this we mark all redundant phis as 'unused', then remove the phi's
// from places in ourself the phis may have flowed into, then replace all
// uses of the phi's in the MIR graph with the original SSA definition.
for (MPhiIterator phi = loopEntry->phisBegin(); phi != loopEntry->phisEnd();
phi++) {
MOZ_ASSERT(phi->numOperands() ==
2);
if (phi->getOperand(
0) == phi->getOperand(
1)) {
phi->setUnused();
}
}
// Fix up phis stored in the slots Vector of pending blocks.
for (PendingBlockTarget& pendingBlockTarget : pendingBlocks_) {
for (ControlFlowPatch& p : pendingBlockTarget.patches) {
MBasicBlock* block = p.ins->block();
if (block->loopDepth() >= loopEntry->loopDepth()) {
fixupRedundantPhis(block);
}
}
}
// The loop body, if any, might be referencing recycled phis too.
if (loopBody) {
fixupRedundantPhis(loopBody);
}
// Pending jumps to an enclosing try-catch may reference the recycled phis.
// We have to search above all enclosing try blocks, as a delegate may move
// patches around.
for (uint32_t depth =
0; depth < iter().controlStackDepth(); depth++) {
LabelKind kind = iter().controlKind(depth);
if (kind != LabelKind::
Try && kind != LabelKind::TryTable &&
kind != LabelKind::Body) {
continue;
}
Control& control = iter().controlItem(depth);
if (!control.tryControl) {
continue;
}
for (MControlInstruction* patch : control.tryControl->landingPadPatches) {
MBasicBlock* block = patch->block();
if (block->loopDepth() >= loopEntry->loopDepth()) {
fixupRedundantPhis(block);
}
}
}
for (MControlInstruction* patch : bodyRethrowPadPatches_) {
MBasicBlock* block = patch->block();
if (block->loopDepth() >= loopEntry->loopDepth()) {
fixupRedundantPhis(block);
}
}
// If we're inlined into another function we are accumulating return values
// in a vector, search through the results to see if any refer to a
// redundant phi.
for (PendingInlineReturn& pendingReturn : pendingInlineReturns_) {
for (uint32_t resultIndex =
0;
resultIndex < pendingReturn.results.length(); resultIndex++) {
MDefinition** pendingResult = &pendingReturn.results[resultIndex];
if ((*pendingResult)->isUnused()) {
*pendingResult = (*pendingResult)->toPhi()->getOperand(
0);
}
}
}
// Discard redundant phis and add to the free list.
for (MPhiIterator phi = loopEntry->phisBegin();
phi != loopEntry->phisEnd();) {
MPhi* entryDef = *phi++;
if (!entryDef->isUnused()) {
continue;
}
entryDef->justReplaceAllUsesWith(entryDef->getOperand(
0));
loopEntry->discardPhi(entryDef);
mirGraph().addPhiToFreeList(entryDef);
}
return true;
}
public:
[[nodiscard]]
bool closeLoop(MBasicBlock* loopHeader,
DefVector* loopResults) {
MOZ_ASSERT(pendingBlockDepth_ >=
1);
MOZ_ASSERT(rootCompiler_.loopDepth());
uint32_t headerLabel = pendingBlockDepth_ -
1;
if (!loopHeader) {
MOZ_ASSERT(inDeadCode());
MOZ_ASSERT(headerLabel >= pendingBlocks_.length() ||
pendingBlocks_[headerLabel].patches.empty());
pendingBlockDepth_--;
rootCompiler_.closeLoop();
return true;
}
// Op::Loop doesn't have an implicit backedge so temporarily set
// aside the end of the loop body to bind backedges.
MBasicBlock* loopBody = curBlock_;
curBlock_ = nullptr;
// As explained in bug 1253544, Ion apparently has an invariant that
// there is only one backedge to loop headers. To handle wasm's ability
// to have multiple backedges to the same loop header, we bind all those
// branches as forward jumps to a single backward jump. This is
// unfortunate but the optimizer is able to fold these into single jumps
// to backedges.
DefVector backedgeValues;
if (!bindBranches(headerLabel, &backedgeValues)) {
return false;
}
MOZ_ASSERT(loopHeader->loopDepth() == rootCompiler_.loopDepth());
if (curBlock_) {
// We're on the loop backedge block, created by bindBranches.
for (size_t i =
0, n = numPushed(curBlock_); i != n; i++) {
curBlock_->pop();
}
if (!pushDefs(backedgeValues)) {
return false;
}
MOZ_ASSERT(curBlock_->loopDepth() == rootCompiler_.loopDepth());
curBlock_->end(MGoto::
New(alloc(), loopHeader));
if (!setLoopBackedge(loopHeader, loopBody, curBlock_,
backedgeValues.length())) {
return false;
}
}
curBlock_ = loopBody;
rootCompiler_.closeLoop();
// If the loop depth still at the inner loop body, correct it.
if (curBlock_ && curBlock_->loopDepth() != rootCompiler_.loopDepth()) {
MBasicBlock* out;
if (!goToNewBlock(curBlock_, &out)) {
return false;
}
curBlock_ = out;
}
pendingBlockDepth_ -=
1;
return inDeadCode() || popPushedDefs(loopResults);
}
[[nodiscard]]
bool addControlFlowPatch(
MControlInstruction* ins, uint32_t relative, uint32_t index,
BranchHint branchHint = BranchHint::Invalid) {
MOZ_ASSERT(relative < pendingBlockDepth_);
uint32_t absolute = pendingBlockDepth_ -
1 - relative;
if (absolute >= pendingBlocks_.length() &&
!pendingBlocks_.resize(absolute +
1)) {
return false;
}
pendingBlocks_[absolute].hint = branchHint;
return pendingBlocks_[absolute].patches.append(
ControlFlowPatch(ins, index));
}
[[nodiscard]]
bool br(uint32_t relativeDepth,
const DefVector& values) {
if (inDeadCode()) {
return true;
}
MGoto* jump = MGoto::
New(alloc());
if (!addControlFlowPatch(jump, relativeDepth, MGoto::TargetIndex)) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(jump);
curBlock_ = nullptr;
return true;
}
[[nodiscard]]
bool brIf(uint32_t relativeDepth,
const DefVector& values,
MDefinition* condition, BranchHint branchHint) {
if (inDeadCode()) {
return true;
}
MBasicBlock* joinBlock = nullptr;
if (!newBlock(curBlock_, &joinBlock)) {
return false;
}
MTest* test = MTest::
New(alloc(), condition, nullptr, joinBlock);
if (!addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex,
branchHint)) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(test);
curBlock_ = joinBlock;
return true;
}
[[nodiscard]]
bool brTable(MDefinition* operand, uint32_t defaultDepth,
const Uint32Vector& depths,
const DefVector& values) {
if (inDeadCode()) {
return true;
}
size_t numCases = depths.length();
MOZ_ASSERT(numCases <= INT32_MAX);
MOZ_ASSERT(numCases);
MTableSwitch* table =
MTableSwitch::
New(alloc(), operand,
0, int32_t(numCases -
1));
size_t defaultIndex;
if (!table->addDefault(nullptr, &defaultIndex)) {
return false;
}
if (!addControlFlowPatch(table, defaultDepth, defaultIndex)) {
return false;
}
using IndexToCaseMap =
HashMap<uint32_t, uint32_t, DefaultHasher<uint32_t>, SystemAllocPolicy>;
IndexToCaseMap indexToCase;
if (!indexToCase.put(defaultDepth, defaultIndex)) {
return false;
}
for (size_t i =
0; i < numCases; i++) {
if (!mirGen().ensureBallast()) {
return false;
}
uint32_t depth = depths[i];
size_t caseIndex;
IndexToCaseMap::AddPtr p = indexToCase.lookupForAdd(depth);
if (!p) {
if (!table->addSuccessor(nullptr, &caseIndex)) {
return false;
}
if (!addControlFlowPatch(table, depth, caseIndex)) {
return false;
}
if (!indexToCase.add(p, depth, caseIndex)) {
return false;
}
}
else {
caseIndex = p->value();
}
if (!table->addCase(caseIndex)) {
return false;
}
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(table);
curBlock_ = nullptr;
return true;
}
/********************************************************** Exceptions ***/
bool inTryBlockFrom(uint32_t fromRelativeDepth,
uint32_t* tryRelativeDepth)
const {
uint32_t relativeDepth;
if (iter_.controlFindInnermostFrom(
[](LabelKind kind,
const Control& control) {
return control.tryControl != nullptr &&
control.tryControl->inBody;
},
fromRelativeDepth, &relativeDepth)) {
*tryRelativeDepth = relativeDepth;
return true;
}
if (callerCompiler_ && callerCompiler_->inTryCode()) {
*tryRelativeDepth = iter_.controlStackDepth() -
1;
return true;
}
return false;
}
bool inTryBlockFrom(uint32_t fromRelativeDepth,
ControlInstructionVector** landingPadPatches) {
uint32_t tryRelativeDepth;
if (!inTryBlockFrom(fromRelativeDepth, &tryRelativeDepth)) {
return false;
}
if (tryRelativeDepth == iter().controlStackDepth() -
1) {
*landingPadPatches = &bodyRethrowPadPatches_;
}
else {
*landingPadPatches =
&iter().controlItem(tryRelativeDepth).tryControl->landingPadPatches;
}
return true;
}
bool inTryBlock(ControlInstructionVector** landingPadPatches) {
return inTryBlockFrom(
0, landingPadPatches);
}
bool inTryCode()
const {
uint32_t tryRelativeDepth;
return inTryBlockFrom(
0, &tryRelativeDepth);
}
MDefinition* loadTag(uint32_t tagIndex) {
MWasmLoadInstanceDataField* tag = MWasmLoadInstanceDataField::
New(
alloc(), MIRType::WasmAnyRef,
codeMeta().offsetOfTagInstanceData(tagIndex),
true, instancePointer_);
curBlock_->add(tag);
return tag;
}
void loadPendingExceptionState(MDefinition** pendingException,
MDefinition** pendingExceptionTag) {
auto* exception = MWasmLoadInstance::
New(
alloc(), instancePointer_, wasm::Instance::offsetOfPendingException(),
MIRType::WasmAnyRef, AliasSet::Load(AliasSet::WasmPendingException));
curBlock_->add(exception);
*pendingException = exception;
auto* tag = MWasmLoadInstance::
New(
alloc(), instancePointer_,
wasm::Instance::offsetOfPendingExceptionTag(), MIRType::WasmAnyRef,
AliasSet::Load(AliasSet::WasmPendingException));
curBlock_->add(tag);
*pendingExceptionTag = tag;
}
[[nodiscard]]
bool setPendingExceptionState(MDefinition* exception,
MDefinition* tag) {
// Set the pending exception object
auto* exceptionAddr = MWasmDerivedPointer::
New(
alloc(), instancePointer_, Instance::offsetOfPendingException());
curBlock_->add(exceptionAddr);
auto* setException = MWasmStoreRef::
New(
alloc(), instancePointer_, exceptionAddr,
/*valueOffset=*/0, exception,
AliasSet::WasmPendingException, WasmPreBarrierKind::Normal);
curBlock_->add(setException);
if (!postBarrierEdgePrecise(
/*lineOrBytecode=*/0, exceptionAddr,
exception)) {
return false;
}
// Set the pending exception tag object
auto* exceptionTagAddr = MWasmDerivedPointer::
New(
alloc(), instancePointer_, Instance::offsetOfPendingExceptionTag());
curBlock_->add(exceptionTagAddr);
auto* setExceptionTag = MWasmStoreRef::
New(
alloc(), instancePointer_, exceptionTagAddr,
/*valueOffset=*/0, tag,
AliasSet::WasmPendingException, WasmPreBarrierKind::Normal);
curBlock_->add(setExceptionTag);
return postBarrierEdgePrecise(
/*lineOrBytecode=*/0, exceptionTagAddr, tag);
}
[[nodiscard]]
bool endWithPadPatch(
ControlInstructionVector* tryLandingPadPatches) {
MGoto* jumpToLandingPad = MGoto::
New(alloc());
curBlock_->end(jumpToLandingPad);
return tryLandingPadPatches->emplaceBack(jumpToLandingPad);
}
[[nodiscard]]
bool delegatePadPatches(
const ControlInstructionVector& patches,
uint32_t relativeDepth) {
if (patches.empty()) {
return true;
}
// Find where we are delegating the pad patches to.
ControlInstructionVector* targetPatches;
if (!inTryBlockFrom(relativeDepth, &targetPatches)) {
MOZ_ASSERT(relativeDepth <= pendingBlockDepth_ -
1);
targetPatches = &bodyRethrowPadPatches_;
}
// Append the delegate's pad patches to the target's.
for (MControlInstruction* ins : patches) {
if (!targetPatches->emplaceBack(ins)) {
return false;
}
}
return true;
}
[[nodiscard]]
bool beginCatchableCall(CallCompileState* callState) {
if (!inTryBlock(&callState->tryLandingPadPatches)) {
MOZ_ASSERT(!callState->isCatchable());
return true;
}
MOZ_ASSERT(callState->isCatchable());
// Allocate a try note
if (!rootCompiler_.addTryNote(&callState->tryNoteIndex)) {
return false;
}
// Allocate blocks for fallthrough and exceptions
return newBlock(curBlock_, &callState->fallthroughBlock) &&
newBlock(curBlock_, &callState->prePadBlock);
}
[[nodiscard]]
bool finishCatchableCall(CallCompileState* callState) {
if (!callState->tryLandingPadPatches) {
return true;
}
// Switch to the prePadBlock
MBasicBlock* callBlock = curBlock_;
curBlock_ = callState->prePadBlock;
// Mark this as the landing pad for the call
curBlock_->add(MWasmCallLandingPrePad::
New(alloc(), callBlock,
callState->tryNoteIndex));
// End with a pending jump to the landing pad
if (!endWithPadPatch(callState->tryLandingPadPatches)) {
return false;
}
// Compilation continues in the fallthroughBlock.
curBlock_ = callState->fallthroughBlock;
return true;
}
// Create a landing pad for a try block. This is also used for the implicit
// rethrow landing pad used for delegate instructions that target the
// outermost label.
[[nodiscard]]
bool createTryLandingPad(ControlInstructionVector& landingPadPatches,
MBasicBlock** landingPad) {
MOZ_ASSERT(!landingPadPatches.empty());
// Bind the branches from exception throwing code to a new landing pad
// block. This is done similarly to what is done in bindBranches.
MControlInstruction* ins = landingPadPatches[
0];
MBasicBlock* pred = ins->block();
if (!newBlock(pred, landingPad)) {
return false;
}
ins->replaceSuccessor(MGoto::TargetIndex, *landingPad);
for (size_t i =
1; i < landingPadPatches.length(); i++) {
ins = landingPadPatches[i];
pred = ins->block();
if (!(*landingPad)->addPredecessor(alloc(), pred)) {
return false;
}
ins->replaceSuccessor(MGoto::TargetIndex, *landingPad);
}
// Clear the now bound pad patches.
landingPadPatches.clear();
return true;
}
[[nodiscard]]
bool createTryTableLandingPad(TryControl* tryControl) {
// If there were no patches, then there were no throwing instructions and
// we don't need to do anything.
if (tryControl->landingPadPatches.empty()) {
return true;
}
// Create the landing pad block and bind all the throwing instructions
MBasicBlock* landingPad;
if (!createTryLandingPad(tryControl->landingPadPatches, &landingPad)) {
return false;
}
// Get the pending exception from the instance
MDefinition* pendingException;
MDefinition* pendingExceptionTag;
if (!consumePendingException(&landingPad, &pendingException,
&pendingExceptionTag)) {
return false;
}
MBasicBlock* originalBlock = curBlock_;
curBlock_ = landingPad;
bool hadCatchAll =
false;
for (
const TryTableCatch& tryTableCatch : tryControl->catches) {
// Handle a catch_all by jumping to the target block
if (tryTableCatch.tagIndex == CatchAllIndex) {
// Capture the exnref value if we need to
DefVector values;
if (tryTableCatch.captureExnRef && !values.append(pendingException)) {
return false;
}
// Branch to the catch_all code
if (!br(tryTableCatch.labelRelativeDepth, values)) {
return false;
}
// Break from the loop and skip the implicit rethrow that's needed
// if we didn't have a catch_all
hadCatchAll =
true;
break;
}
// Handle a tagged catch by doing a compare and branch on the tag index,
// jumping to a catch block if they match, or else to a fallthrough block
// to continue the landing pad.
MBasicBlock* catchBlock = nullptr;
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &catchBlock) ||
!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
// Branch to the catch block if the exception's tag matches this catch
// block's tag.
MDefinition* catchTag = loadTag(tryTableCatch.tagIndex);
MDefinition* matchesCatchTag =
compare(pendingExceptionTag, catchTag, JSOp::Eq,
MCompare::Compare_WasmAnyRef);
curBlock_->end(
MTest::
New(alloc(), matchesCatchTag, catchBlock, fallthroughBlock));
// Set up the catch block by extracting the values from the exception
// object.
curBlock_ = catchBlock;
// Extract the exception values for the catch block
DefVector values;
if (!loadExceptionValues(pendingException, tryTableCatch.tagIndex,
&values)) {
return false;
}
if (tryTableCatch.captureExnRef && !values.append(pendingException)) {
return false;
}
if (!br(tryTableCatch.labelRelativeDepth, values)) {
return false;
}
curBlock_ = fallthroughBlock;
}
// If there was no catch_all, we must rethrow this exception.
if (!hadCatchAll) {
if (!throwFrom(pendingException, pendingExceptionTag)) {
return false;
}
}
curBlock_ = originalBlock;
return true;
}
// Consume the pending exception state from instance. This will clear out the
// previous value.
[[nodiscard]]
bool consumePendingException(MBasicBlock** landingPad,
MDefinition** pendingException,
MDefinition** pendingExceptionTag) {
MBasicBlock* prevBlock = curBlock_;
curBlock_ = *landingPad;
// Load the pending exception and tag
loadPendingExceptionState(pendingException, pendingExceptionTag);
// Clear the pending exception and tag
auto* null = constantNullRef(MaybeRefType());
if (!setPendingExceptionState(null, null)) {
return false;
}
// The landing pad may have changed from loading and clearing the pending
// exception state.
*landingPad = curBlock_;
curBlock_ = prevBlock;
return true;
}
[[nodiscard]]
bool startTry() {
Control& control = iter().controlItem();
control.block = curBlock_;
control.tryControl = rootCompiler_.newTryControl();
if (!control.tryControl) {
return false;
}
control.tryControl->inBody =
true;
return startBlock();
}
[[nodiscard]]
bool startTryTable(TryTableCatchVector&& catches) {
Control& control = iter().controlItem();
control.block = curBlock_;
control.tryControl = rootCompiler_.newTryControl();
if (!control.tryControl) {
return false;
}
control.tryControl->inBody =
true;
control.tryControl->catches = std::move(catches);
return startBlock();
}
[[nodiscard]]
bool joinTryOrCatchBlock(Control& control) {
// If the try or catch block ended with dead code, there is no need to
// do any control flow join.
if (inDeadCode()) {
return true;
}
// This is a split path which we'll need to join later, using a control
// flow patch.
MOZ_ASSERT(!curBlock_->hasLastIns());
MGoto* jump = MGoto::
New(alloc());
if (!addControlFlowPatch(jump,
0, MGoto::TargetIndex)) {
return false;
}
// Finish the current block with the control flow patch instruction.
curBlock_->end(jump);
return true;
}
// Finish the previous block (either a try or catch block) and then setup a
// new catch block.
[[nodiscard]]
bool switchToCatch(Control& control, LabelKind fromKind,
uint32_t tagIndex) {
// Mark this control node as being no longer in the body of the try
control.tryControl->inBody =
false;
// If there is no control block, then either:
// - the entry of the try block is dead code, or
// - there is no landing pad for the try-catch.
// In either case, any catch will be dead code.
if (!control.block) {
MOZ_ASSERT(inDeadCode());
return true;
}
// Join the previous try or catch block with a patch to the future join of
// the whole try-catch block.
if (!joinTryOrCatchBlock(control)) {
return false;
}
// If we are switching from the try block, create the landing pad. This is
// guaranteed to happen once and only once before processing catch blocks.
if (fromKind == LabelKind::
Try) {
if (!control.tryControl->landingPadPatches.empty()) {
// Create the landing pad block and bind all the throwing instructions
MBasicBlock* padBlock = nullptr;
if (!createTryLandingPad(control.tryControl->landingPadPatches,
&padBlock)) {
return false;
}
// Store the pending exception and tag on the control item for future
// use in catch handlers.
if (!consumePendingException(
&padBlock, &control.tryControl->pendingException,
&control.tryControl->pendingExceptionTag)) {
return false;
}
// Set the control block for this try-catch to the landing pad.
control.block = padBlock;
}
else {
control.block = nullptr;
}
}
// If there is no landing pad, then this and following catches are dead
// code.
if (!control.block) {
curBlock_ = nullptr;
return true;
}
// Switch to the landing pad.
curBlock_ = control.block;
// We should have a pending exception and tag if we were able to create a
// landing pad.
MOZ_ASSERT(control.tryControl->pendingException);
MOZ_ASSERT(control.tryControl->pendingExceptionTag);
// Handle a catch_all by immediately jumping to a new block. We require a
// new block (as opposed to just emitting the catch_all code in the current
// block) because rethrow requires the exception/tag to be present in the
// landing pad's slots, while the catch_all block must not have the
// exception/tag in slots.
if (tagIndex == CatchAllIndex) {
MBasicBlock* catchAllBlock = nullptr;
if (!goToNewBlock(curBlock_, &catchAllBlock)) {
return false;
}
// Compilation will continue in the catch_all block.
curBlock_ = catchAllBlock;
return true;
}
// Handle a tagged catch by doing a compare and branch on the tag index,
// jumping to a catch block if they match, or else to a fallthrough block
// to continue the landing pad.
MBasicBlock* catchBlock = nullptr;
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &catchBlock) ||
!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
// Branch to the catch block if the exception's tag matches this catch
// block's tag.
MDefinition* catchTag = loadTag(tagIndex);
MDefinition* matchesCatchTag =
compare(control.tryControl->pendingExceptionTag, catchTag, JSOp::Eq,
MCompare::Compare_WasmAnyRef);
curBlock_->end(
MTest::
New(alloc(), matchesCatchTag, catchBlock, fallthroughBlock));
// The landing pad will continue in the fallthrough block
control.block = fallthroughBlock;
// Set up the catch block by extracting the values from the exception
// object.
curBlock_ = catchBlock;
// Extract the exception values for the catch block
DefVector values;
if (!loadExceptionValues(control.tryControl->pendingException, tagIndex,
&values)) {
return false;
}
iter().setResults(values.length(), values);
return true;
}
[[nodiscard]]
bool loadExceptionValues(MDefinition* exception,
uint32_t tagIndex, DefVector* values) {
SharedTagType tagType = codeMeta().tags[tagIndex].type;
const ValTypeVector& params = tagType->argTypes();
const TagOffsetVector& offsets = tagType->exceptionArgOffsets();
// Get the data pointer from the exception object
auto* data = MWasmLoadField::
New(
alloc(), exception, nullptr, WasmExceptionObject::offsetOfData(),
mozilla::Nothing(), MIRType::Pointer, MWideningOp::None,
AliasSet::Load(AliasSet::Any));
if (!data) {
return false;
}
curBlock_->add(data);
// Presize the values vector to the number of params
if (!values->reserve(params.length())) {
return false;
}
// Load each value from the data pointer
for (size_t i =
0; i < params.length(); i++) {
if (!mirGen().ensureBallast()) {
return false;
}
auto* load =
MWasmLoadField::
New(alloc(), data, exception, offsets[i],
mozilla::Nothing(), params[i].toMIRType(),
MWideningOp::None, AliasSet::Load(AliasSet::Any),
mozilla::Nothing(), params[i].toMaybeRefType());
if (!load || !values->append(load)) {
return false;
}
curBlock_->add(load);
}
return true;
}
[[nodiscard]]
bool finishTryCatch(LabelKind kind, Control& control,
DefVector* defs) {
switch (kind) {
case LabelKind::
Try: {
// This is a catchless try, we must delegate all throwing instructions
// to the nearest enclosing try block if one exists, or else to the
// body block which will handle it in emitBodyRethrowPad. We
// specify a relativeDepth of '1' to delegate outside of the still
// active try block.
uint32_t relativeDepth =
1;
if (!delegatePadPatches(control.tryControl->landingPadPatches,
relativeDepth)) {
return false;
}
break;
}
case LabelKind::
Catch: {
MOZ_ASSERT(!control.tryControl->inBody);
// This is a try without a catch_all, we must have a rethrow at the end
// of the landing pad (if any).
MBasicBlock* padBlock = control.block;
if (padBlock) {
MBasicBlock* prevBlock = curBlock_;
curBlock_ = padBlock;
if (!throwFrom(control.tryControl->pendingException,
control.tryControl->pendingExceptionTag)) {
return false;
}
curBlock_ = prevBlock;
}
break;
}
case LabelKind::CatchAll: {
MOZ_ASSERT(!control.tryControl->inBody);
// This is a try with a catch_all, and requires no special handling.
break;
}
default:
MOZ_CRASH();
}
// Finish the block, joining the try and catch blocks
return finishBlock(defs);
}
[[nodiscard]]
bool finishTryTable(Control& control, DefVector* defs) {
// Mark this control as no longer in the body of the try
control.tryControl->inBody =
false;
// Create a landing pad for all of the catches
if (!createTryTableLandingPad(control.tryControl.get())) {
return false;
}
// Finish the block, joining the try and catch blocks
return finishBlock(defs);
}
[[nodiscard]]
bool emitBodyRethrowPad(Control& control) {
// If there are no throwing instructions pending, we don't need to do
// anything
if (bodyRethrowPadPatches_.empty()) {
return true;
}
// Create a landing pad for any throwing instructions
MBasicBlock* padBlock;
if (!createTryLandingPad(bodyRethrowPadPatches_, &padBlock)) {
return false;
}
// If we're inlined into another function, we save the landing pad to be
// linked later directly to our caller's landing pad. See
// `finishedInlinedCallDirect`.
if (callerCompiler_ && callerCompiler_->inTryCode()) {
pendingInlineCatchBlock_ = padBlock;
return true;
}
// Otherwise we need to grab the pending exception and rethrow it.
MDefinition* pendingException;
MDefinition* pendingExceptionTag;
if (!consumePendingException(&padBlock, &pendingException,
&pendingExceptionTag)) {
return false;
}
// Switch to the landing pad and rethrow the exception
MBasicBlock* prevBlock = curBlock_;
curBlock_ = padBlock;
if (!throwFrom(pendingException, pendingExceptionTag)) {
return false;
}
curBlock_ = prevBlock;
MOZ_ASSERT(bodyRethrowPadPatches_.empty());
return true;
}
[[nodiscard]]
bool emitNewException(MDefinition* tag,
MDefinition** exception) {
return emitInstanceCall1(readBytecodeOffset(), SASigExceptionNew, tag,
exception);
}
[[nodiscard]]
bool emitThrow(uint32_t tagIndex,
const DefVector& argValues) {
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
// Load the tag
MDefinition* tag = loadTag(tagIndex);
if (!tag) {
return false;
}
// Allocate an exception object
MDefinition* exception;
if (!emitNewException(tag, &exception)) {
return false;
}
// Load the data pointer from the object
auto* data = MWasmLoadField::
New(
alloc(), exception, nullptr, WasmExceptionObject::offsetOfData(),
mozilla::Nothing(), MIRType::Pointer, MWideningOp::None,
AliasSet::Load(AliasSet::Any));
if (!data) {
return false;
}
curBlock_->add(data);
// Store the params into the data pointer
SharedTagType tagType = codeMeta().tags[tagIndex].type;
for (size_t i =
0; i < tagType->exceptionArgOffsets().length(); i++) {
if (!mirGen().ensureBallast()) {
return false;
}
ValType type = tagType->argTypes()[i];
uint32_t offset = tagType->exceptionArgOffsets()[i];
if (!type.isRefRepr()) {
auto* store = MWasmStoreField::
New(
alloc(), data, exception, offset, mozilla::Nothing(), argValues[i],
MNarrowingOp::None, AliasSet::Store(AliasSet::Any));
if (!store) {
return false;
}
curBlock_->add(store);
continue;
}
// Store the new value
auto* store = MWasmStoreFieldRef::
New(
alloc(), instancePointer_, data, exception, offset,
mozilla::Nothing(), argValues[i], AliasSet::Store(AliasSet::Any),
Nothing(), WasmPreBarrierKind::None);
if (!store) {
return false;
}
curBlock_->add(store);
// Call the post-write barrier
if (!postBarrierWholeCell(bytecodeOffset, exception, argValues[i])) {
return false;
}
}
// Throw the exception
return throwFrom(exception, tag);
}
[[nodiscard]]
bool emitThrowRef(MDefinition* exnRef) {
if (inDeadCode()) {
return true;
}
// The exception must be non-null
exnRef = refAsNonNull(exnRef);
if (!exnRef) {
return false;
}
// Call Instance::throwException to perform tag unpacking and throw the
// exception
if (!emitInstanceCall1(readBytecodeOffset(), SASigThrowException, exnRef)) {
return false;
}
unreachableTrap();
curBlock_ = nullptr;
return true;
}
[[nodiscard]]
bool throwFrom(MDefinition* exn, MDefinition* tag) {
if (inDeadCode()) {
return true;
}
// Check if there is a local catching try control, and if so, then add a
// pad-patch to its tryPadPatches.
ControlInstructionVector* tryLandingPadPatches;
if (inTryBlock(&tryLandingPadPatches)) {
// Set the pending exception state, the landing pad will read from this
if (!setPendingExceptionState(exn, tag)) {
return false;
}
// End with a pending jump to the landing pad
if (!endWithPadPatch(tryLandingPadPatches)) {
return false;
}
curBlock_ = nullptr;
return true;
}
// If there is no surrounding catching block, call an instance method to
// throw the exception.
if (!emitInstanceCall1(readBytecodeOffset(), SASigThrowException, exn)) {
return false;
}
unreachableTrap();
curBlock_ = nullptr;
return true;
}
[[nodiscard]]
bool emitRethrow(uint32_t relativeDepth) {
if (inDeadCode()) {
return true;
}
Control& control = iter().controlItem(relativeDepth);
MOZ_ASSERT(iter().controlKind(relativeDepth) == LabelKind::
Catch ||
iter().controlKind(relativeDepth) == LabelKind::CatchAll);
return throwFrom(control.tryControl->pendingException,
control.tryControl->pendingExceptionTag);
}
/******************************** WasmGC: low level load/store helpers ***/
// Given a (StorageType, FieldExtension) pair, produce the (MIRType,
// MWideningOp) pair that will give the correct operation for reading the
// value from memory.
static void fieldLoadInfoToMIR(StorageType type, FieldWideningOp wideningOp,
MIRType* mirType, MWideningOp* mirWideningOp) {
switch (type.kind()) {
case StorageType::I8: {
switch (wideningOp) {
case FieldWideningOp::
Signed:
*mirType = MIRType::Int32;
*mirWideningOp = MWideningOp::FromS8;
return;
case FieldWideningOp::
Unsigned:
*mirType = MIRType::Int32;
*mirWideningOp = MWideningOp::FromU8;
return;
default:
MOZ_CRASH();
}
}
case StorageType::I16: {
switch (wideningOp) {
case FieldWideningOp::
Signed:
*mirType = MIRType::Int32;
*mirWideningOp = MWideningOp::FromS16;
return;
case FieldWideningOp::
Unsigned:
*mirType = MIRType::Int32;
*mirWideningOp = MWideningOp::FromU16;
return;
default:
MOZ_CRASH();
}
}
default: {
switch (wideningOp) {
case FieldWideningOp::None:
*mirType = type.toMIRType();
*mirWideningOp = MWideningOp::None;
return;
default:
MOZ_CRASH();
}
}
}
}
// Given a StorageType, return the Scale required when accessing array
// elements of this type.
static Scale scaleFromFieldType(StorageType type) {
if (type.kind() == StorageType::V128) {
// V128 is accessed differently, so this scale will not be used.
return Scale::Invalid;
}
return ShiftToScale(type.indexingShift());
}
// Given a StorageType, produce the MNarrowingOp required for writing the
// value to memory.
static MNarrowingOp fieldStoreInfoToMIR(StorageType type) {
switch (type.kind()) {
case StorageType::I8:
return MNarrowingOp::To8;
case StorageType::I16:
return MNarrowingOp::To16;
default:
return MNarrowingOp::None;
}
}
// Generate a write of `value` at address `base + offset`, where `offset` is
// known at JIT time. If the written value is a reftype, the previous value
// at `base + offset` will be retrieved and handed off to the post-write
// barrier. `keepAlive` will be referenced by the instruction so as to hold
// it live (from the GC's point of view).
[[nodiscard]]
bool writeGcValueAtBasePlusOffset(
uint32_t lineOrBytecode, StorageType type, MDefinition* keepAlive,
AliasSet::Flag aliasBitset, MDefinition* value, MDefinition* base,
uint32_t offset, uint32_t fieldIndex,
bool needsTrapInfo,
WasmPreBarrierKind preBarrierKind, WasmPostBarrierKind postBarrierKind) {
MOZ_ASSERT(aliasBitset !=
0);
MOZ_ASSERT(keepAlive->type() == MIRType::WasmAnyRef);
MOZ_ASSERT(type.widenToValType().toMIRType() == value->type());
MNarrowingOp narrowingOp = fieldStoreInfoToMIR(type);
if (!type.isRefRepr()) {
MaybeTrapSiteDesc maybeTrap;
if (needsTrapInfo) {
maybeTrap.emplace(trapSiteDesc());
}
auto* store = MWasmStoreField::
New(
alloc(), base, keepAlive, offset, mozilla::Some(fieldIndex), value,
narrowingOp, AliasSet::Store(aliasBitset), maybeTrap);
if (!store) {
return false;
}
curBlock_->add(store);
return true;
}
// Otherwise it's a ref store. Load the previous value so we can show it
// to the post-write barrier.
//
// Optimisation opportunity: for the case where this field write results
// from struct.new, the old value is always zero. So we should synthesise
// a suitable zero constant rather than reading it from the object. See
// also bug 1799999.
MOZ_ASSERT(narrowingOp == MNarrowingOp::None);
MOZ_ASSERT(type.widenToValType() == type.valType());
// Store the new value
auto* store = MWasmStoreFieldRef::
New(
alloc(), instancePointer_, base, keepAlive, offset,
mozilla::Some(fieldIndex), value, AliasSet::Store(aliasBitset),
mozilla::Some(trapSiteDesc()), preBarrierKind);
if (!store) {
return false;
}
curBlock_->add(store);
// Call the post-write barrier
switch (postBarrierKind) {
case WasmPostBarrierKind::WholeCell:
return postBarrierWholeCell(lineOrBytecode, keepAlive, value);
case WasmPostBarrierKind::Edge:
MOZ_CRASH(
"WasmPostBarrierKind::Edge not supported");
case WasmPostBarrierKind::None:
return true;
default:
MOZ_CRASH(
"Bad postBarrierKind");
}
}
// Generate a write of `value` at address `base + index * scale`, where
// `scale` is known at JIT-time. If the written value is a reftype, the
// previous value at `base + index * scale` will be retrieved and handed off
// to the post-write barrier. `keepAlive` will be referenced by the
// instruction so as to hold it live (from the GC's point of view).
[[nodiscard]]
bool writeGcValueAtBasePlusScaledIndex(
uint32_t lineOrBytecode, StorageType type, MDefinition* keepAlive,
AliasSet::Flag aliasBitset, MDefinition* value, MDefinition* base,
uint32_t scale, MDefinition* index, WasmPreBarrierKind preBarrierKind,
WasmPostBarrierKind postBarrierKind) {
MOZ_ASSERT(aliasBitset !=
0);
MOZ_ASSERT(keepAlive->type() == MIRType::WasmAnyRef);
MOZ_ASSERT(type.widenToValType().toMIRType() == value->type());
MOZ_ASSERT(scale ==
1 || scale ==
2 || scale ==
4 || scale ==
8 ||
scale ==
16);
MNarrowingOp narrowingOp = fieldStoreInfoToMIR(type);
if (!type.isRefRepr()) {
MaybeTrapSiteDesc maybeTrap;
Scale scale = scaleFromFieldType(type);
auto* store = MWasmStoreElement::
New(
alloc(), base, index, value, keepAlive, narrowingOp, scale,
AliasSet::Store(aliasBitset), maybeTrap);
if (!store) {
return false;
}
curBlock_->add(store);
return true;
}
// Otherwise it's a ref store.
MOZ_ASSERT(narrowingOp == MNarrowingOp::None);
MOZ_ASSERT(type.widenToValType() == type.valType());
// Store the new value
auto* store = MWasmStoreElementRef::
New(
alloc(), instancePointer_, base, index, value, keepAlive,
AliasSet::Store(aliasBitset), mozilla::Some(trapSiteDesc()),
preBarrierKind);
if (!store) {
return false;
}
curBlock_->add(store);
switch (postBarrierKind) {
case WasmPostBarrierKind::WholeCell:
return postBarrierWholeCell(lineOrBytecode, keepAlive, value);
case WasmPostBarrierKind::Edge:
return postBarrierEdgeAtIndex(lineOrBytecode, keepAlive, base, index,
sizeof(
void*), value);
case WasmPostBarrierKind::None:
return true;
default:
MOZ_CRASH(
"Bad postBarrierKind");
}
}
// Generate a read from address `base + offset`, where `offset` is known at
// JIT time. The loaded value will be widened as described by `type` and
// `fieldWideningOp`. `keepAlive` will be referenced by the instruction so as
// to hold it live (from the GC's point of view).
[[nodiscard]] MDefinition* readGcValueAtBasePlusOffset(
StorageType type, FieldWideningOp fieldWideningOp, MDefinition* keepAlive,
AliasSet::Flag aliasBitset, MDefinition* base, uint32_t offset,
uint32_t fieldIndex,
bool needsTrapInfo) {
MOZ_ASSERT(aliasBitset !=
0);
MOZ_ASSERT(keepAlive->type() == MIRType::WasmAnyRef);
MIRType mirType;
MWideningOp mirWideningOp;
fieldLoadInfoToMIR(type, fieldWideningOp, &mirType, &mirWideningOp);
MaybeTrapSiteDesc maybeTrap;
if (needsTrapInfo) {
maybeTrap.emplace(trapSiteDesc());
}
auto* load = MWasmLoadField::
New(alloc(), base, keepAlive, offset,
mozilla::Some(fieldIndex), mirType,
mirWideningOp, AliasSet::Load(aliasBitset),
maybeTrap, type.toMaybeRefType());
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
// Generate a read from address `base + index * scale`, where `scale` is
// known at JIT-time. The loaded value will be widened as described by
// `type` and `fieldWideningOp`. `keepAlive` will be referenced by the
// instruction so as to hold it live (from the GC's point of view).
[[nodiscard]] MDefinition* readGcArrayValueAtIndex(
StorageType type, FieldWideningOp fieldWideningOp, MDefinition* keepAlive,
AliasSet::Flag aliasBitset, MDefinition* base, MDefinition* index) {
MOZ_ASSERT(aliasBitset !=
0);
MOZ_ASSERT(keepAlive->type() == MIRType::WasmAnyRef);
MIRType mirType;
MWideningOp mirWideningOp;
fieldLoadInfoToMIR(type, fieldWideningOp, &mirType, &mirWideningOp);
Scale scale = scaleFromFieldType(type);
auto* load = MWasmLoadElement::
New(
alloc(), base, keepAlive, index, mirType, mirWideningOp, scale,
AliasSet::Load(aliasBitset), mozilla::Some(trapSiteDesc()),
type.toMaybeRefType());
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
/************************************************ WasmGC: type helpers ***/
// Returns an MDefinition holding the supertype vector for `typeIndex`.
[[nodiscard]] MDefinition* loadSuperTypeVector(uint32_t typeIndex) {
uint32_t stvOffset = codeMeta().offsetOfSuperTypeVector(typeIndex);
auto* load =
MWasmLoadInstanceDataField::
New(alloc(), MIRType::Pointer, stvOffset,
/*isConst=*/true, instancePointer_);
if (!load) {
return nullptr;
}
curBlock_->add(load);
return load;
}
uint32_t readAllocSiteIndex(uint32_t typeIndex) {
if (!codeTailMeta() || !codeTailMeta()->hasFuncDefAllocSites()) {
// For single tier of optimized compilation, there are no assigned alloc
// sites, using type index as alloc site.
return typeIndex;
}
AllocSitesRange rangeInModule =
codeTailMeta()->getFuncDefAllocSites(funcIndex());
uint32_t localIndex = numAllocSites_++;
MOZ_RELEASE_ASSERT(localIndex < rangeInModule.length);
return rangeInModule.begin + localIndex;
}
[[nodiscard]] MDefinition* loadAllocSiteInstanceData(
uint32_t allocSiteIndex) {
auto* allocSites = MWasmLoadInstance::
New(
alloc(), instancePointer_, Instance::offsetOfAllocSites(),
MIRType::Pointer, AliasSet::None());
if (!allocSites) {
return nullptr;
}
curBlock_->add(allocSites);
auto* result = MWasmDerivedPointer::
New(
alloc(), allocSites, allocSiteIndex *
sizeof(gc::AllocSite));
if (!result) {
return nullptr;
}
curBlock_->add(result);
return result;
}
/********************************************** WasmGC: struct helpers ***/
[[nodiscard]] MDefinition* createStructObject(uint32_t typeIndex,
uint32_t allocSiteIndex,
bool zeroFields) {
// Allocate an uninitialized struct.
MDefinition* allocSite = loadAllocSiteInstanceData(allocSiteIndex);
if (!allocSite) {
return nullptr;
}
const TypeDef*
typeDef = &(*codeMeta().types)[typeIndex];
auto* structObject =
MWasmNewStructObject::
New(alloc(), instancePointer_, allocSite,
typeDef,
zeroFields, trapSiteDesc());
if (!structObject) {
return nullptr;
}
curBlock_->add(structObject);
return structObject;
}
// Helper function for EmitStruct{New,Set}: given a MIR pointer to a
// WasmStructObject, a MIR pointer to a value, and a field descriptor,
// generate MIR to write the value to the relevant field in the object.
[[nodiscard]]
bool writeValueToStructField(
uint32_t lineOrBytecode,
const StructType& structType,
uint32_t fieldIndex, MDefinition* structObject, MDefinition* value,
WasmPreBarrierKind preBarrierKind) {
StorageType fieldType = structType.fields_[fieldIndex].type;
FieldAccessPath path = structType.fieldAccessPaths_[fieldIndex];
uint32_t areaOffset = path.hasOOL() ? path.oolOffset() : path.ilOffset();
// Make `base` point at the first byte of either the struct object as a
// whole or of the out-of-line data area.
MDefinition* base;
bool needsTrapInfo;
if (path.hasOOL()) {
// The path has two components, of which the first (the IL component) is
// the offset where the OOL pointer is stored. Hence `path.ilOffset()`.
auto* loadDataPointer = MWasmLoadField::
New(
alloc(), structObject, nullptr, path.ilOffset(), mozilla::Nothing(),
MIRType::WasmStructData, MWideningOp::None,
AliasSet::Load(AliasSet::WasmStructOutlineDataPointer),
mozilla::Some(trapSiteDesc()));
if (!loadDataPointer) {
return false;
}
curBlock_->add(loadDataPointer);
base = loadDataPointer;
needsTrapInfo =
false;
}
else {
base = structObject;
needsTrapInfo =
true;
}
// The transaction is to happen at `base + areaOffset`, so to speak.
// The alias set denoting the field's location, although lacking a
// Load-vs-Store indication at this point.
AliasSet::Flag fieldAliasSet = path.hasOOL()
? AliasSet::WasmStructOutlineDataArea
: AliasSet::WasmStructInlineDataArea;
return writeGcValueAtBasePlusOffset(
lineOrBytecode, fieldType, structObject, fieldAliasSet, value, base,
areaOffset, fieldIndex, needsTrapInfo, preBarrierKind,
WasmPostBarrierKind::WholeCell);
}
// Helper function for EmitStructGet: given a MIR pointer to a
// WasmStructObject, a field descriptor and a field widening operation,
// generate MIR to read the value from the relevant field in the object.
[[nodiscard]] MDefinition* readValueFromStructField(
const StructType& structType, uint32_t fieldIndex,
FieldWideningOp wideningOp, MDefinition* structObject) {
StorageType fieldType = structType.fields_[fieldIndex].type;
FieldAccessPath path = structType.fieldAccessPaths_[fieldIndex];
uint32_t areaOffset = path.hasOOL() ? path.oolOffset() : path.ilOffset();
// Make `base` point at the first byte of either the struct object as a
// whole or of the out-of-line data area.
MDefinition* base;
bool needsTrapInfo;
if (path.hasOOL()) {
// The path has two components, of which the first (the IL component) is
// the offset where the OOL pointer is stored. Hence `path.ilOffset()`.
auto* loadDataPointer = MWasmLoadField::
New(
alloc(), structObject, nullptr, path.ilOffset(), mozilla::Nothing(),
MIRType::WasmStructData, MWideningOp::None,
AliasSet::Load(AliasSet::WasmStructOutlineDataPointer),
mozilla::Some(trapSiteDesc()));
if (!loadDataPointer) {
return nullptr;
}
curBlock_->add(loadDataPointer);
base = loadDataPointer;
needsTrapInfo =
false;
}
else {
base = structObject;
needsTrapInfo =
true;
}
// The transaction is to happen at `base + areaOffset`, so to speak.
// The alias set denoting the field's location, although lacking a
// Load-vs-Store indication at this point.
AliasSet::Flag fieldAliasSet = path.hasOOL()
? AliasSet::WasmStructOutlineDataArea
: AliasSet::WasmStructInlineDataArea;
return readGcValueAtBasePlusOffset(fieldType, wideningOp, structObject,
fieldAliasSet, base, areaOffset,
fieldIndex, needsTrapInfo);
}
/********************************* WasmGC: address-arithmetic helpers ***/
inline bool targetIs64Bit()
const {
#ifdef JS_64BIT
return true;
#else
return false;
#endif
}
// Generate MIR to unsigned widen `val` out to the target word size. If
// `val` is already at the target word size, this is a no-op. The only
// other allowed case is where `val` is Int32 and we're compiling for a
// 64-bit target, in which case a widen is generated.
[[nodiscard]] MDefinition* unsignedWidenToTargetWord(MDefinition* val) {
if (targetIs64Bit()) {
if (val->type() == MIRType::Int32) {
auto* ext = MExtendInt32ToInt64::
New(alloc(), val,
/*isUnsigned=*/true);
if (!ext) {
return nullptr;
}
curBlock_->add(ext);
return ext;
}
MOZ_ASSERT(val->type() == MIRType::Int64);
return val;
}
MOZ_ASSERT(val->type() == MIRType::Int32);
return val;
}
/********************************************** WasmGC: array helpers ***/
// Given `arrayObject`, the address of a WasmArrayObject, generate MIR to
// return the contents of the WasmArrayObject::numElements_ field.
// Adds trap site info for the null check.
[[nodiscard]] MDefinition* getWasmArrayObjectNumElements(
MDefinition* arrayObject) {
MOZ_ASSERT(arrayObject->type() == MIRType::WasmAnyRef);
auto* numElements = MWasmLoadField::
New(
alloc(), arrayObject, nullptr, WasmArrayObject::offsetOfNumElements(),
mozilla::Nothing(), MIRType::Int32, MWideningOp::None,
AliasSet::Load(AliasSet::WasmArrayNumElements),
mozilla::Some(trapSiteDesc()));
if (!numElements) {
return nullptr;
}
curBlock_->add(numElements);
return numElements;
}
// Given `arrayObject`, the address of a WasmArrayObject, generate MIR to
// return the contents of the WasmArrayObject::data_ field.
[[nodiscard]] MDefinition* getWasmArrayObjectData(MDefinition* arrayObject) {
MOZ_ASSERT(arrayObject->type() == MIRType::WasmAnyRef);
auto* data = MWasmLoadField::
New(
alloc(), arrayObject, nullptr, WasmArrayObject::offsetOfData(),
mozilla::Nothing(), MIRType::WasmArrayData, MWideningOp::None,
AliasSet::Load(AliasSet::WasmArrayDataPointer),
mozilla::Some(trapSiteDesc()));
if (!data) {
return nullptr;
}
curBlock_->add(data);
return data;
}
// Given a JIT-time-known type index `typeIndex` and a run-time known number
// of elements `numElements`, create MIR to allocate a new wasm array,
// possibly initialized with `typeIndex`s default value.
[[nodiscard]] MDefinition* createArrayObject(uint32_t typeIndex,
uint32_t allocSiteIndex,
MDefinition* numElements,
bool zeroFields) {
MDefinition* allocSite = loadAllocSiteInstanceData(allocSiteIndex);
if (!allocSite) {
return nullptr;
}
const TypeDef*
typeDef = &(*codeMeta().types)[typeIndex];
auto* arrayObject = MWasmNewArrayObject::
New(
alloc(), instancePointer_, numElements, allocSite,
typeDef, zeroFields,
trapSiteDesc());
if (!arrayObject) {
return nullptr;
}
curBlock_->add(arrayObject);
return arrayObject;
}
// This emits MIR to perform several actions common to array loads and
// stores. Given `arrayObject`, that points to a WasmArrayObject, and an
// index value `index`, it:
//
// * Generates a trap if the array pointer is null
// * Gets the size of the array
// * Emits a bounds check of `index` against the array size
// * Retrieves the OOL object pointer from the array
// * Includes check for null via signal handler.
//
// The returned value is for the OOL object pointer.
[[nodiscard]] MDefinition* setupForArrayAccess(MDefinition* arrayObject,
MDefinition* index) {
MOZ_ASSERT(arrayObject->type() == MIRType::WasmAnyRef);
MOZ_ASSERT(index->type() == MIRType::Int32);
// Check for null is done in getWasmArrayObjectNumElements.
// Get the size value for the array.
MDefinition* numElements = getWasmArrayObjectNumElements(arrayObject);
if (!numElements) {
return nullptr;
}
// Create a bounds check.
auto* boundsCheck =
MWasmBoundsCheck::
New(alloc(), index, numElements, trapSiteDesc(),
MWasmBoundsCheck::Target::Other);
if (!boundsCheck) {
return nullptr;
}
curBlock_->add(boundsCheck);
// Get the address of the first byte of the (OOL) data area.
return getWasmArrayObjectData(arrayObject);
}
[[nodiscard]]
bool fillArray(uint32_t lineOrBytecode,
const ArrayType& arrayType,
MDefinition* arrayObject, MDefinition* index,
MDefinition* numElements, MDefinition* val,
WasmPreBarrierKind preBarrierKind,
WasmPostBarrierKind postBarrierKind) {
mozilla::DebugOnly<MIRType> valMIRType = val->type();
StorageType elemType = arrayType.elementType();
MOZ_ASSERT(elemType.widenToValType().toMIRType() == valMIRType);
uint32_t elemSize = elemType.size();
MOZ_ASSERT(elemSize >=
1 && elemSize <=
16);
// Make `arrayBase` point at the first byte of the (OOL) data area.
MDefinition* arrayBase = getWasmArrayObjectData(arrayObject);
if (!arrayBase) {
return false;
}
// We have:
// arrayBase : TargetWord
// index : Int32
// numElements : Int32
// val : <any StorageType>
// $elemSize = arrayType.elementType_.size(); 1, 2, 4, 8 or 16
//
// Generate MIR:
// <in current block>
// limit : Int32 = index + numElements
// if (limit == index) goto after; // skip loop if trip count == 0
// loop:
// indexPhi = phi(index, indexNext)
// arrayBase[index * $elemSize] = val
// indexNext = indexPhi + 1
// if (indexNext <u limit) goto loop;
// after:
//
// We construct the loop "manually" rather than using
// FunctionCompiler::{startLoop,closeLoop} as the latter have awareness of
// the wasm view of loops, whereas the loop we're building here is not a
// wasm-level loop.
// ==== Create the "loop" and "after" blocks ====
MBasicBlock* loopBlock;
if (!newBlock(curBlock_, &loopBlock, MBasicBlock::LOOP_HEADER)) {
return false;
}
MBasicBlock* afterBlock;
if (!newBlock(loopBlock, &afterBlock)) {
return false;
}
// ==== Fill in the remainder of the block preceding the loop ====
MAdd* limit = MAdd::NewWasm(alloc(), index, numElements, MIRType::Int32);
if (!limit) {
return false;
}
curBlock_->add(limit);
// Note: the comparison (and eventually the entire initialisation loop) will
// be folded out in the case where the number of elements is zero.
// See MCompare::tryFoldEqualOperands.
MDefinition* limitEqualsBase =
compare(limit, index, JSOp::StrictEq, MCompare::Compare_UInt32);
if (!limitEqualsBase) {
return false;
}
MTest* skipIfLimitEqualsBase =
MTest::
New(alloc(), limitEqualsBase, afterBlock, loopBlock);
if (!skipIfLimitEqualsBase) {
return false;
}
curBlock_->end(skipIfLimitEqualsBase);
if (!afterBlock->addPredecessor(alloc(), curBlock_)) {
return false;
}
// ==== Fill in the loop block as best we can ====
curBlock_ = loopBlock;
MPhi* indexPhi = MPhi::
New(alloc(), MIRType::Int32);
if (!indexPhi) {
return false;
}
if (!indexPhi->reserveLength(
2)) {
return false;
}
indexPhi->addInput(index);
curBlock_->addPhi(indexPhi);
curBlock_->setLoopDepth(rootCompiler_.loopDepth() +
1);
if (!writeGcValueAtBasePlusScaledIndex(
lineOrBytecode, elemType, arrayObject, AliasSet::WasmArrayDataArea,
val, arrayBase, elemSize, indexPhi, preBarrierKind,
postBarrierKind)) {
return false;
}
auto* indexNext =
MAdd::NewWasm(alloc(), indexPhi, constantI32(
1), MIRType::Int32);
if (!indexNext) {
return false;
}
curBlock_->add(indexNext);
indexPhi->addInput(indexNext);
MDefinition* indexNextLtuLimit =
compare(indexNext, limit, JSOp::Lt, MCompare::Compare_UInt32);
if (!indexNextLtuLimit) {
return false;
}
auto* continueIfIndexNextLtuLimit =
MTest::
New(alloc(), indexNextLtuLimit, loopBlock, afterBlock);
if (!continueIfIndexNextLtuLimit) {
return false;
}
curBlock_->end(continueIfIndexNextLtuLimit);
if (!loopBlock->addPredecessor(alloc(), loopBlock)) {
return false;
}
// ==== Loop block completed ====
curBlock_ = afterBlock;
return true;
}
[[nodiscard]]
bool createArrayCopy(uint32_t lineOrBytecode,
MDefinition* dstArrayObject,
MDefinition* dstArrayIndex,
MDefinition* srcArrayObject,
MDefinition* srcArrayIndex,
MDefinition* numElements, int32_t elemSize,
bool elemsAreRefTyped) {
// Check for null is done in getWasmArrayObjectNumElements.
// Get the arrays' actual sizes.
MDefinition* dstNumElements = getWasmArrayObjectNumElements(dstArrayObject);
if (!dstNumElements) {
return false;
}
MDefinition* srcNumElements = getWasmArrayObjectNumElements(srcArrayObject);
if (!srcNumElements) {
return false;
}
// Create the bounds checks.
MInstruction* dstBoundsCheck = MWasmBoundsCheckRange32::
New(
alloc(), dstArrayIndex, numElements, dstNumElements, trapSiteDesc());
if (!dstBoundsCheck) {
return false;
}
curBlock_->add(dstBoundsCheck);
MInstruction* srcBoundsCheck = MWasmBoundsCheckRange32::
New(
alloc(), srcArrayIndex, numElements, srcNumElements, trapSiteDesc());
if (!srcBoundsCheck) {
return false;
}
curBlock_->add(srcBoundsCheck);
// Check if numElements != 0 -- optimization to not invoke builtins.
MBasicBlock* copyBlock = nullptr;
if (!newBlock(curBlock_, ©Block)) {
return false;
}
MBasicBlock* joinBlock = nullptr;
if (!newBlock(curBlock_, &joinBlock)) {
return false;
}
MInstruction* condition =
MCompare::NewWasm(alloc(), numElements, constantI32(
0), JSOp::StrictEq,
MCompare::Compare_UInt32);
curBlock_->add(condition);
MTest* test = MTest::
New(alloc(), condition, joinBlock, copyBlock);
if (!test) {
return false;
}
curBlock_->end(test);
curBlock_ = copyBlock;
if (elemsAreRefTyped) {
MOZ_RELEASE_ASSERT(elemSize ==
sizeof(
void*));
if (!builtinCall5(SASigArrayRefsMove, lineOrBytecode, dstArrayObject,
dstArrayIndex, srcArrayObject, srcArrayIndex,
numElements, nullptr)) {
return false;
}
}
else {
MDefinition* elemSizeDef = constantI32(elemSize);
if (!elemSizeDef) {
return false;
}
if (!builtinCall6(SASigArrayMemMove, lineOrBytecode, dstArrayObject,
dstArrayIndex, srcArrayObject, srcArrayIndex,
elemSizeDef, numElements, nullptr)) {
return false;
}
}
MGoto* fallthrough = MGoto::
New(alloc(), joinBlock);
if (!fallthrough) {
return false;
}
curBlock_->end(fallthrough);
if (!joinBlock->addPredecessor(alloc(), curBlock_)) {
return false;
}
curBlock_ = joinBlock;
return true;
}
[[nodiscard]]
bool createArrayFill(uint32_t lineOrBytecode,
uint32_t typeIndex,
MDefinition* arrayObject,
MDefinition* index, MDefinition* val,
MDefinition* numElements) {
MOZ_ASSERT(arrayObject->type() == MIRType::WasmAnyRef);
MOZ_ASSERT(index->type() == MIRType::Int32);
MOZ_ASSERT(numElements->type() == MIRType::Int32);
const ArrayType& arrayType = (*codeMeta().types)[typeIndex].arrayType();
// Check for null is done in getWasmArrayObjectNumElements.
// Get the array's actual size.
MDefinition* actualNumElements = getWasmArrayObjectNumElements(arrayObject);
if (!actualNumElements) {
return false;
}
// Create a bounds check.
auto* boundsCheck = MWasmBoundsCheckRange32::
New(
alloc(), index, numElements, actualNumElements, trapSiteDesc());
if (!boundsCheck) {
return false;
}
curBlock_->add(boundsCheck);
return fillArray(lineOrBytecode, arrayType, arrayObject, index, numElements,
val, WasmPreBarrierKind::Normal,
WasmPostBarrierKind::Edge);
}
/*********************************************** WasmGC: other helpers ***/
// Generate MIR that attempts to cast `ref` to `castToTypeDef`. If the
// cast fails, we trap. If it succeeds, then `ref` can be assumed to
// have a type that is a subtype of (or the same as) `castToTypeDef` after
// this point.
[[nodiscard]] MDefinition* refCast(MDefinition* ref, RefType destType) {
MInstruction* cast = nullptr;
if (destType.isTypeRef()) {
uint32_t typeIndex = codeMeta().types->indexOf(*destType.
typeDef());
MDefinition* superSTV = loadSuperTypeVector(typeIndex);
if (!superSTV) {
return nullptr;
}
cast = MWasmRefCastConcrete::
New(alloc(), ref, superSTV, destType,
trapSiteDesc());
}
else {
cast = MWasmRefCastAbstract::
New(alloc(), ref, destType, trapSiteDesc());
}
if (!cast) {
return nullptr;
}
curBlock_->add(cast);
return cast;
}
// Generate MIR that computes a boolean value indicating whether or not it
// is possible to cast `ref` to `destType`.
[[nodiscard]] MDefinition* refTest(MDefinition* ref, RefType destType) {
MInstruction* isSubTypeOf = nullptr;
if (destType.isTypeRef()) {
uint32_t typeIndex = codeMeta().types->indexOf(*destType.
typeDef());
MDefinition* superSTV = loadSuperTypeVector(typeIndex);
if (!superSTV) {
return nullptr;
}
isSubTypeOf = MWasmRefTestConcrete::
New(alloc(), ref, superSTV, destType);
}
else {
isSubTypeOf = MWasmRefTestAbstract::
New(alloc(), ref, destType);
}
MOZ_ASSERT(isSubTypeOf);
curBlock_->add(isSubTypeOf);
return isSubTypeOf;
}
// Generates MIR for br_on_cast and br_on_cast_fail.
[[nodiscard]]
bool brOnCastCommon(
bool onSuccess, uint32_t labelRelativeDepth,
RefType sourceType, RefType destType,
const ResultType& labelType,
const DefVector& values,
BranchHint branchHint) {
if (inDeadCode()) {
return true;
}
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
// `values` are the values in the top block-value on the stack. Since the
// argument to `br_on_cast{_fail}` is at the top of the stack, it is the
// last element in `values`.
//
// For both br_on_cast and br_on_cast_fail, the OpIter validation routines
// ensure that `values` is non-empty (by rejecting the case
// `labelType->length() < 1`) and that the last value in `values` is
// reftyped.
MOZ_RELEASE_ASSERT(values.length() >
0);
MDefinition* ref = values.back();
MOZ_ASSERT(ref->type() == MIRType::WasmAnyRef);
MDefinition* success = refTest(ref, destType);
if (!success) {
return false;
}
MTest* test;
if (onSuccess) {
test = MTest::
New(alloc(), success, nullptr, fallthroughBlock);
if (!test || !addControlFlowPatch(test, labelRelativeDepth,
MTest::TrueBranchIndex, branchHint)) {
return false;
}
}
else {
test = MTest::
New(alloc(), success, fallthroughBlock, nullptr);
if (!test || !addControlFlowPatch(test, labelRelativeDepth,
MTest::FalseBranchIndex, branchHint)) {
return false;
}
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(test);
curBlock_ = fallthroughBlock;
return true;
}
[[nodiscard]]
bool brOnNonStruct(
const DefVector& values) {
if (inDeadCode()) {
return true;
}
MBasicBlock* fallthroughBlock = nullptr;
if (!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
MOZ_ASSERT(values.length() >
0);
MOZ_ASSERT(values.back()->type() == MIRType::WasmAnyRef);
MGoto* jump = MGoto::
New(alloc(), fallthroughBlock);
if (!jump) {
return false;
}
if (!pushDefs(values)) {
return false;
}
curBlock_->end(jump);
curBlock_ = fallthroughBlock;
return true;
}
[[nodiscard]] MDefinition* convertAnyExtern(MDefinition* ref,
wasm::RefType::Kind kind) {
auto* converted = MWasmRefConvertAnyExtern::
New(alloc(), ref, kind);
if (!converted) {
return nullptr;
}
curBlock_->add(converted);
return converted;
}
/************************************************************ DECODING ***/
// AsmJS adds a line number to `callSiteLineNums` for certain operations that
// are represented by a JS call, such as math builtins. We use these line
// numbers when calling builtins. This method will read from
// `callSiteLineNums` when we are using AsmJS, or else return the current
// bytecode offset.
//
// This method MUST be called from opcodes that AsmJS will emit a call site
// line number for, or else the arrays will get out of sync. Other opcodes
// must use `readBytecodeOffset` below.
uint32_t readCallSiteLineOrBytecode() {
if (!func_.callSiteLineNums.empty()) {
return func_.callSiteLineNums[lastReadCallSite_++];
}
return iter_.lastOpcodeOffset();
}
// Return the current bytecode offset.
uint32_t readBytecodeOffset() {
return iter_.lastOpcodeOffset(); }
CallRefHint readCallRefHint() {
// We don't track anything if we're not using lazy tiering
if (compilerEnv().mode() != CompileMode::LazyTiering) {
return CallRefHint();
}
CallRefMetricsRange rangeInModule =
codeTailMeta()->getFuncDefCallRefs(funcIndex());
uint32_t localIndex = numCallRefs_++;
MOZ_RELEASE_ASSERT(localIndex < rangeInModule.length);
uint32_t moduleIndex = rangeInModule.begin + localIndex;
return codeTailMeta()->getCallRefHint(moduleIndex);
}
#if DEBUG
bool done()
const {
return iter_.done(); }
#endif
/*************************************************************************/
private:
[[nodiscard]]
bool newBlock(MBasicBlock* pred, MBasicBlock** block,
MBasicBlock::Kind kind = MBasicBlock::NORMAL) {
*block = MBasicBlock::
New(mirGraph(), info(), pred, kind);
if (!*block) {
return false;
}
mirGraph().addBlock(*block);
(*block)->setLoopDepth(rootCompiler_.loopDepth());
return true;
}
[[nodiscard]]
bool goToNewBlock(MBasicBlock* pred, MBasicBlock** block) {
if (!newBlock(pred, block)) {
return false;
}
pred->end(MGoto::
New(alloc(), *block));
return true;
}
[[nodiscard]]
bool goToExistingBlock(MBasicBlock* prev, MBasicBlock* next) {
MOZ_ASSERT(prev);
MOZ_ASSERT(next);
prev->end(MGoto::
New(alloc(), next));
return next->addPredecessor(alloc(), prev);
}
[[nodiscard]]
bool bindBranches(uint32_t absolute, DefVector* defs) {
if (absolute >= pendingBlocks_.length() ||
pendingBlocks_[absolute].patches.empty()) {
return inDeadCode() || popPushedDefs(defs);
}
ControlFlowPatchVector& patches = pendingBlocks_[absolute].patches;
MControlInstruction* ins = patches[
0].ins;
MBasicBlock* pred = ins->block();
MBasicBlock* join = nullptr;
if (!newBlock(pred, &join)) {
return false;
}
// Use branch hinting information if any.
if (pendingBlocks_[absolute].hint != BranchHint::Invalid) {
BranchHint hint = pendingBlocks_[absolute].hint;
if (hint == BranchHint::Likely) {
join->setFrequency(Frequency::Likely);
}
else if (hint == BranchHint::Unlikely) {
join->setFrequency(Frequency::Unlikely);
}
}
pred->mark();
ins->replaceSuccessor(patches[
0].index, join);
for (size_t i =
1; i < patches.length(); i++) {
ins = patches[i].ins;
pred = ins->block();
if (!pred->isMarked()) {
if (!join->addPredecessor(alloc(), pred)) {
return false;
}
pred->mark();
}
ins->replaceSuccessor(patches[i].index, join);
}
MOZ_ASSERT_IF(curBlock_, !curBlock_->isMarked());
for (uint32_t i =
0; i < join->numPredecessors(); i++) {
join->getPredecessor(i)->unmark();
}
if (curBlock_ && !goToExistingBlock(curBlock_, join)) {
return false;
}
curBlock_ = join;
if (!popPushedDefs(defs)) {
return false;
}
patches.clear();
return true;
}
bool emitI32Const();
bool emitI64Const();
bool emitF32Const();
bool emitF64Const();
bool emitBlock();
bool emitLoop();
bool emitIf();
bool emitElse();
bool emitEnd();
bool emitBr();
bool emitBrIf();
bool emitBrTable();
bool emitReturn();
bool emitUnreachable();
bool emitTry();
bool emitCatch();
bool emitCatchAll();
bool emitTryTable();
bool emitDelegate();
bool emitThrow();
bool emitThrowRef();
bool emitRethrow();
bool emitInlineCall(
const FuncType& funcType, uint32_t funcIndex,
InliningHeuristics::CallKind callKind,
const DefVector& args, DefVector* results);
bool emitCall(
bool asmJSFuncDef);
bool emitCallIndirect(
bool oldStyle);
bool emitReturnCall();
bool emitReturnCallIndirect();
bool emitReturnCallRef();
#ifdef ENABLE_WASM_JSPI
bool emitContNew();
bool emitContBind();
bool emitStoreSuspendParams(MDefinition* paramsArea,
const ValTypeVector& suspendTagParams,
const DefVector& suspendParams,
MDefinition* suspendedCont);
bool emitSuspend();
bool emitResume();
bool emitResumeThrow();
bool emitResumeThrowRef();
bool emitSwitch();
bool emitGuardSuspending();
#endif
bool emitGetLocal();
bool emitSetLocal();
bool emitTeeLocal();
bool emitGetGlobal();
bool emitSetGlobal();
bool emitTeeGlobal();
template <
typename MIRClass>
bool emitUnary(ValType operandType);
template <
typename MIRClass>
bool emitConversion(ValType operandType, ValType resultType);
template <
typename MIRClass>
bool emitUnaryWithType(ValType operandType, MIRType mirType);
template <
typename MIRClass>
bool emitConversionWithType(ValType operandType, ValType resultType,
MIRType mirType);
bool emitTruncate(ValType operandType, ValType resultType,
bool isUnsigned,
bool isSaturating);
bool emitSignExtend(uint32_t srcSize, uint32_t targetSize);
bool emitExtendI32(
bool isUnsigned);
bool emitWrapI32();
bool emitConvertI64ToFloatingPoint(ValType resultType, MIRType mirType,
bool isUnsigned);
bool emitReinterpret(ValType resultType, ValType operandType,
MIRType mirType);
bool emitAdd(ValType type, MIRType mirType);
bool emitSub(ValType type, MIRType mirType);
bool emitRotate(ValType type,
bool isLeftRotation);
bool emitBitNot(ValType operandType, MIRType mirType);
bool emitBitwiseAndOrXor(ValType operandType, MIRType mirType,
MWasmBinaryBitwise::SubOpcode subOpc);
template <
typename MIRClass>
bool emitShift(ValType operandType, MIRType mirType);
bool emitUrsh(ValType operandType, MIRType mirType);
bool emitMul(ValType operandType, MIRType mirType);
bool emitDiv(ValType operandType, MIRType mirType,
bool isUnsigned);
bool emitRem(ValType operandType, MIRType mirType,
bool isUnsigned);
bool emitMinMax(ValType operandType, MIRType mirType,
bool isMax);
bool emitCopySign(ValType operandType);
bool emitComparison(ValType operandType, JSOp compareOp,
MCompare::CompareType compareType);
bool emitSelect(
bool typed);
bool emitLoad(ValType type, Scalar::Type viewType);
bool emitStore(ValType resultType, Scalar::Type viewType);
bool emitTeeStore(ValType resultType, Scalar::Type viewType);
bool emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType);
bool tryInlineUnaryBuiltin(SymbolicAddress callee, MDefinition* input);
bool emitUnaryMathBuiltinCall(
const SymbolicAddressSignature& callee);
bool emitBinaryMathBuiltinCall(
const SymbolicAddressSignature& callee);
bool emitMemoryGrow();
bool emitMemorySize();
bool emitAtomicCmpXchg(ValType type, Scalar::Type viewType);
bool emitAtomicLoad(ValType type, Scalar::Type viewType);
bool emitAtomicRMW(ValType type, Scalar::Type viewType, jit::AtomicOp op);
bool emitAtomicStore(ValType type, Scalar::Type viewType);
bool emitWait(ValType type, uint32_t byteSize);
bool emitFence();
bool emitNotify();
bool emitAtomicXchg(ValType type, Scalar::Type viewType);
bool emitMemCopyCall(uint32_t dstMemIndex, uint32_t srcMemIndex,
MDefinition* dst, MDefinition* src, MDefinition* len);
bool emitMemCopyInline(uint32_t memoryIndex, MDefinition* dst,
MDefinition* src, uint32_t length);
bool emitMemCopy();
bool emitTableCopy();
bool emitDataOrElemDrop(
bool isData);
bool emitMemFillCall(uint32_t memoryIndex, MDefinition* start,
MDefinition* val, MDefinition* len);
bool emitMemFillInline(uint32_t memoryIndex, MDefinition* start,
MDefinition* val, uint32_t length);
bool emitMemFill();
bool emitMemInit();
bool emitTableInit();
bool emitTableFill();
bool emitMemDiscard();
bool emitTableGet();
bool emitTableGrow();
bool emitTableSet();
bool emitTableSize();
bool emitRefFunc();
bool emitRefNull();
bool emitRefIsNull();
bool emitI64AddSub128(
bool isAdd);
bool emitI64MulWide(
bool isSigned);
bool emitConstSimd128();
bool emitBinarySimd128(
bool commutative, SimdOp op);
bool emitTernarySimd128(wasm::SimdOp op);
bool emitShiftSimd128(SimdOp op);
bool emitSplatSimd128(ValType inType, SimdOp op);
bool emitUnarySimd128(SimdOp op);
bool emitReduceSimd128(SimdOp op);
bool emitExtractLaneSimd128(ValType outType, uint32_t laneLimit, SimdOp op);
bool emitReplaceLaneSimd128(ValType laneType, uint32_t laneLimit, SimdOp op);
bool emitShuffleSimd128();
bool emitLoadSplatSimd128(Scalar::Type viewType, wasm::SimdOp splatOp);
bool emitLoadExtendSimd128(wasm::SimdOp op);
bool emitLoadZeroSimd128(Scalar::Type viewType, size_t numBytes);
bool emitLoadLaneSimd128(uint32_t laneSize);
bool emitStoreLaneSimd128(uint32_t laneSize);
bool emitRefAsNonNull();
bool emitBrOnNull();
bool emitBrOnNonNull();
bool emitSpeculativeInlineCallRef(uint32_t bytecodeOffset,
const FuncType& funcType,
CallRefHint expectedFuncIndices,
MDefinition* actualCalleeFunc,
const DefVector& args, DefVector* results);
bool emitCallRef();
bool emitStructNew();
bool emitStructNewDefault();
bool emitStructSet();
bool emitStructGet(FieldWideningOp wideningOp);
bool emitArrayNew();
bool emitArrayNewDefault();
bool emitArrayNewFixed();
bool emitArrayNewData();
bool emitArrayNewElem();
bool emitArrayInitData();
bool emitArrayInitElem();
bool emitArraySet();
bool emitArrayGet(FieldWideningOp wideningOp);
bool emitArrayLen();
bool emitArrayCopy();
bool emitArrayFill();
bool emitRefI31();
bool emitI31Get(FieldWideningOp wideningOp);
bool emitRefTest(
bool nullable);
bool emitRefCast(
bool nullable);
bool emitBrOnCast(
bool onSuccess);
bool emitAnyConvertExtern();
bool emitExternConvertAny();
bool emitCallBuiltinModuleFunc();
public:
bool emitBodyExprs();
};
template <>
MDefinition* FunctionCompiler::unary<MToFloat32>(MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MToFloat32::
New(alloc(), op, mustPreserveNaN(op->type()));
curBlock_->add(ins);
return ins;
}
template <>
MDefinition* FunctionCompiler::unary<MWasmBuiltinTruncateToInt32>(
MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MWasmBuiltinTruncateToInt32::
New(
alloc(), op, instancePointer_, trapSiteDescWithCallSiteLineNumber());
curBlock_->add(ins);
return ins;
}
template <>
MDefinition* FunctionCompiler::unary<MNot>(MDefinition* op) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MNot::NewInt32(alloc(), op);
curBlock_->add(ins);
return ins;
}
template <>
MDefinition* FunctionCompiler::unary<MAbs>(MDefinition* op, MIRType type) {
if (inDeadCode()) {
return nullptr;
}
auto* ins = MAbs::NewWasm(alloc(), op, type);
curBlock_->add(ins);
return ins;
}
bool FunctionCompiler::emitI32Const() {
int32_t i32;
if (!iter().readI32Const(&i32)) {
return false;
}
iter().setResult(constantI32(i32));
return true;
}
bool FunctionCompiler::emitI64Const() {
int64_t i64;
if (!iter().readI64Const(&i64)) {
return false;
}
iter().setResult(constantI64(i64));
return true;
}
bool FunctionCompiler::emitF32Const() {
float f32;
if (!iter().readF32Const(&f32)) {
return false;
}
iter().setResult(constantF32(f32));
return true;
}
bool FunctionCompiler::emitF64Const() {
double f64;
if (!iter().readF64Const(&f64)) {
return false;
}
iter().setResult(constantF64(f64));
return true;
}
bool FunctionCompiler::emitBlock() {
BlockType type;
return iter().readBlock(&type) && startBlock();
}
bool FunctionCompiler::emitLoop() {
BlockType type;
if (!iter().readLoop(&type)) {
return false;
}
MBasicBlock* loopHeader;
if (!startLoop(&loopHeader, type.params().length())) {
return false;
}
addInterruptCheck();
iter().controlItem().block = loopHeader;
return true;
}
bool FunctionCompiler::emitIf() {
BranchHint branchHint =
iter().getBranchHint(funcIndex(), relativeBytecodeOffset());
BlockType type;
MDefinition* condition = nullptr;
if (!iter().readIf(&type, &condition)) {
return false;
}
MBasicBlock* elseBlock;
if (!branchAndStartThen(condition, &elseBlock)) {
return false;
}
// Store the branch hint in the basic block.
if (!inDeadCode() && branchHint != BranchHint::Invalid) {
if (branchHint == BranchHint::Likely) {
getCurBlock()->setFrequency(Frequency::Likely);
}
else if (branchHint == BranchHint::Unlikely) {
getCurBlock()->setFrequency(Frequency::Unlikely);
}
}
iter().controlItem().block = elseBlock;
return true;
}
bool FunctionCompiler::emitElse() {
ResultType paramType;
ResultType resultType;
DefVector thenValues;
if (!iter().readElse(¶mType, &resultType, &thenValues)) {
return false;
}
if (!pushDefs(thenValues)) {
return false;
}
Control& control = iter().controlItem();
return switchToElse(control.block, &control.block);
}
bool FunctionCompiler::emitEnd() {
LabelKind kind;
ResultType type;
DefVector preJoinDefs;
DefVector resultsForEmptyElse;
if (!iter().readEnd(&kind, &type, &preJoinDefs, &resultsForEmptyElse)) {
return false;
}
Control& control = iter().controlItem();
MBasicBlock* block = control.block;
if (!pushDefs(preJoinDefs)) {
return false;
}
// Every label case is responsible to pop the control item at the appropriate
// time for the label case
DefVector postJoinDefs;
switch (kind) {
case LabelKind::Body: {
MOZ_ASSERT(!control.tryControl);
if (!emitBodyRethrowPad(control)) {
return false;
}
if (!finishBlock(&postJoinDefs)) {
return false;
}
if (!returnValues(std::move(postJoinDefs))) {
return false;
}
iter().popEnd();
MOZ_ASSERT(iter().controlStackEmpty());
return iter().endFunction(iter().end());
}
case LabelKind::Block:
MOZ_ASSERT(!control.tryControl);
if (!finishBlock(&postJoinDefs)) {
return false;
}
iter().popEnd();
break;
case LabelKind::Loop:
MOZ_ASSERT(!control.tryControl);
if (!closeLoop(block, &postJoinDefs)) {
return false;
}
iter().popEnd();
break;
case LabelKind::Then: {
MOZ_ASSERT(!control.tryControl);
// If we didn't see an Else, create a trivial else block so that we create
// a diamond anyway, to preserve Ion invariants.
if (!switchToElse(block, &block)) {
return false;
}
if (!pushDefs(resultsForEmptyElse)) {
return false;
}
if (!joinIfElse(block, &postJoinDefs)) {
return false;
}
iter().popEnd();
break;
}
case LabelKind::
Else:
MOZ_ASSERT(!control.tryControl);
if (!joinIfElse(block, &postJoinDefs)) {
return false;
}
iter().popEnd();
break;
case LabelKind::
Try:
case LabelKind::
Catch:
case LabelKind::CatchAll:
MOZ_ASSERT(control.tryControl);
if (!finishTryCatch(kind, control, &postJoinDefs)) {
return false;
}
rootCompiler().freeTryControl(std::move(control.tryControl));
iter().popEnd();
break;
case LabelKind::TryTable:
MOZ_ASSERT(control.tryControl);
if (!finishTryTable(control, &postJoinDefs)) {
return false;
}
rootCompiler().freeTryControl(std::move(control.tryControl));
iter().popEnd();
break;
}
MOZ_ASSERT_IF(!inDeadCode(), postJoinDefs.length() == type.length());
iter().setResults(postJoinDefs.length(), postJoinDefs);
return true;
}
bool FunctionCompiler::emitBr() {
uint32_t relativeDepth;
ResultType type;
DefVector values;
if (!iter().readBr(&relativeDepth, &type, &values)) {
return false;
}
return br(relativeDepth, values);
}
bool FunctionCompiler::emitBrIf() {
uint32_t relativeDepth;
ResultType type;
DefVector values;
MDefinition* condition;
BranchHint branchHint =
iter().getBranchHint(funcIndex(), relativeBytecodeOffset());
if (!iter().readBrIf(&relativeDepth, &type, &values, &condition)) {
return false;
}
return brIf(relativeDepth, values, condition, branchHint);
}
bool FunctionCompiler::emitBrTable() {
Uint32Vector depths;
uint32_t defaultDepth;
ResultType branchValueType;
DefVector branchValues;
MDefinition* index;
if (!iter().readBrTable(&depths, &defaultDepth, &branchValueType,
&branchValues, &index)) {
return false;
}
// If all the targets are the same, or there are no targets, we can just
// use a goto. This is not just an optimization: MaybeFoldConditionBlock
// assumes that tables have more than one successor.
bool allSameDepth =
true;
for (uint32_t depth : depths) {
if (depth != defaultDepth) {
allSameDepth =
false;
break;
}
}
if (allSameDepth) {
return br(defaultDepth, branchValues);
}
return brTable(index, defaultDepth, depths, branchValues);
}
bool FunctionCompiler::emitReturn() {
DefVector values;
if (!iter().readReturn(&values)) {
return false;
}
return returnValues(std::move(values));
}
bool FunctionCompiler::emitUnreachable() {
if (!iter().readUnreachable()) {
return false;
}
unreachableTrap();
return true;
}
bool FunctionCompiler::emitTry() {
BlockType type;
if (!iter().readTry(&type)) {
return false;
}
return startTry();
}
bool FunctionCompiler::emitCatch() {
LabelKind kind;
uint32_t tagIndex;
ResultType paramType, resultType;
DefVector tryValues;
if (!iter().readCatch(&kind, &tagIndex, ¶mType, &resultType,
&tryValues)) {
return false;
}
// Pushing the results of the previous block, to properly join control flow
// after the try and after each handler, as well as potential control flow
// patches from other instrunctions. This is similar to what is done for
// if-then-else control flow and for most other control control flow joins.
if (!pushDefs(tryValues)) {
return false;
}
return switchToCatch(iter().controlItem(), kind, tagIndex);
}
bool FunctionCompiler::emitCatchAll() {
LabelKind kind;
ResultType paramType, resultType;
DefVector tryValues;
if (!iter().readCatchAll(&kind, ¶mType, &resultType, &tryValues)) {
return false;
}
// Pushing the results of the previous block, to properly join control flow
// after the try and after each handler, as well as potential control flow
// patches from other instrunctions.
if (!pushDefs(tryValues)) {
return false;
}
return switchToCatch(iter().controlItem(), kind, CatchAllIndex);
}
bool FunctionCompiler::emitTryTable() {
BlockType type;
TryTableCatchVector catches;
if (!iter().readTryTable(&type, &catches)) {
return false;
}
return startTryTable(std::move(catches));
}
bool FunctionCompiler::emitDelegate() {
uint32_t relativeDepth;
ResultType resultType;
DefVector tryValues;
if (!iter().readDelegate(&relativeDepth, &resultType, &tryValues)) {
return false;
}
Control& control = iter().controlItem();
MBasicBlock* block = control.block;
MOZ_ASSERT(control.tryControl);
// Unless the entire try-delegate is dead code, delegate any pad-patches from
// this try to the next try-block above relativeDepth.
if (block) {
ControlInstructionVector& padPatches =
control.tryControl->landingPadPatches;
if (!delegatePadPatches(padPatches, relativeDepth)) {
return false;
}
}
rootCompiler().freeTryControl(std::move(control.tryControl));
iter().popDelegate();
// Push the results of the previous block, and join control flow with
// potential control flow patches from other instrunctions in the try code.
// This is similar to what is done for EmitEnd.
if (!pushDefs(tryValues)) {
return false;
}
DefVector postJoinDefs;
if (!finishBlock(&postJoinDefs)) {
return false;
}
MOZ_ASSERT_IF(!inDeadCode(), postJoinDefs.length() == resultType.length());
iter().setResults(postJoinDefs.length(), postJoinDefs);
return true;
}
bool FunctionCompiler::emitThrow() {
uint32_t tagIndex;
DefVector argValues;
if (!iter().readThrow(&tagIndex, &argValues)) {
return false;
}
return emitThrow(tagIndex, argValues);
}
bool FunctionCompiler::emitThrowRef() {
MDefinition* exnRef;
if (!iter().readThrowRef(&exnRef)) {
return false;
}
return emitThrowRef(exnRef);
}
bool FunctionCompiler::emitRethrow() {
uint32_t relativeDepth;
if (!iter().readRethrow(&relativeDepth)) {
return false;
}
return emitRethrow(relativeDepth);
}
bool FunctionCompiler::emitInlineCall(
const FuncType& funcType,
uint32_t funcIndex,
InliningHeuristics::CallKind callKind,
const DefVector& args,
DefVector* results) {
UniqueChars error;
const BytecodeRange& funcRange = codeTailMeta()->funcDefRange(funcIndex);
BytecodeSpan funcBytecode = codeTailMeta()->funcDefBody(funcIndex);
FuncCompileInput func(funcIndex, funcRange.start, funcBytecode.data(),
funcBytecode.data() + funcBytecode.size(),
Uint32Vector());
Decoder d(func.begin, func.end, func.lineOrBytecode, &error);
ValTypeVector locals;
if (!DecodeLocalEntriesWithParams(d, codeMeta(), funcIndex, &locals)) {
return false;
}
CompileInfo* compileInfo = rootCompiler().startInlineCall(
this->funcIndex(), bytecodeOffset(), funcIndex, locals.length(),
funcRange.size(), callKind);
if (!compileInfo) {
return false;
}
FunctionCompiler calleeCompiler(
this, d, func, locals, *compileInfo);
if (!calleeCompiler.initInline(args)) {
MOZ_ASSERT(!error);
return false;
}
if (!calleeCompiler.startBlock()) {
MOZ_ASSERT(!error);
return false;
}
if (!calleeCompiler.emitBodyExprs()) {
MOZ_ASSERT(!error);
return false;
}
calleeCompiler.finish();
rootCompiler_.finishInlineCall();
return finishInlinedCallDirect(calleeCompiler, results);
}
bool FunctionCompiler::emitCall(
bool asmJSFuncDef) {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t funcIndex;
DefVector args;
if (asmJSFuncDef) {
if (!iter().readOldCallDirect(codeMeta().numFuncImports, &funcIndex,
&args)) {
return false;
}
}
else {
if (!iter().readCall(&funcIndex, &args)) {
return false;
}
}
if (inDeadCode()) {
return true;
}
const FuncType& funcType = codeMeta().getFuncType(funcIndex);
DefVector results;
if (codeMeta().funcIsImport(funcIndex)) {
BuiltinModuleFuncId knownFuncImport = codeMeta().knownFuncImport(funcIndex);
if (knownFuncImport != BuiltinModuleFuncId::None) {
const BuiltinModuleFunc& builtinModuleFunc =
BuiltinModuleFuncs::getFromId(knownFuncImport);
return callBuiltinModuleFunc(builtinModuleFunc, args);
}
uint32_t instanceDataOffset =
codeMeta().offsetOfFuncImportInstanceData(funcIndex);
if (!callImport(instanceDataOffset, lineOrBytecode, funcType, args,
&results)) {
return false;
}
}
else {
const auto callKind = InliningHeuristics::CallKind::Direct;
// Make up a single-entry CallRefHint and enquire about its inlineability.
CallRefHint hints;
hints.append(funcIndex);
hints = auditInlineableCallees(callKind, hints);
if (!hints.empty()) {
// Inlining of `funcIndex` was approved.
if (!emitInlineCall(funcType, funcIndex, callKind, args, &results)) {
return false;
}
}
else {
if (!callDirect(funcType, funcIndex, lineOrBytecode, args, &results)) {
return false;
}
}
}
iter().setResults(results.length(), results);
return true;
}
bool FunctionCompiler::emitCallIndirect(
bool oldStyle) {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t funcTypeIndex;
uint32_t tableIndex;
MDefinition* callee;
DefVector args;
if (oldStyle) {
tableIndex =
0;
if (!iter().readOldCallIndirect(&funcTypeIndex, &callee, &args)) {
return false;
}
}
else {
if (!iter().readCallIndirect(&funcTypeIndex, &tableIndex, &callee, &args)) {
return false;
}
}
if (inDeadCode()) {
return true;
}
DefVector results;
if (!callIndirect(funcTypeIndex, tableIndex, callee, lineOrBytecode, args,
&results)) {
return false;
}
iter().setResults(results.length(), results);
return true;
}
bool FunctionCompiler::emitReturnCall() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t funcIndex;
DefVector args;
if (!iter().readReturnCall(&funcIndex, &args)) {
return false;
}
if (inDeadCode()) {
return true;
}
const FuncType& funcType = codeMeta().getFuncType(funcIndex);
DefVector results;
if (codeMeta().funcIsImport(funcIndex)) {
uint32_t globalDataOffset =
codeMeta().offsetOfFuncImportInstanceData(funcIndex);
if (!returnCallImport(globalDataOffset, lineOrBytecode, funcType, args,
&results)) {
return false;
}
}
else {
if (!returnCallDirect(funcType, funcIndex, lineOrBytecode, args,
&results)) {
return false;
}
}
return true;
}
bool FunctionCompiler::emitReturnCallIndirect() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t funcTypeIndex;
uint32_t tableIndex;
MDefinition* callee;
DefVector args;
if (!iter().readReturnCallIndirect(&funcTypeIndex, &tableIndex, &callee,
&args)) {
return false;
}
if (inDeadCode()) {
return true;
}
DefVector results;
return returnCallIndirect(funcTypeIndex, tableIndex, callee, lineOrBytecode,
args, &results);
}
bool FunctionCompiler::emitReturnCallRef() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t funcTypeIndex;
MDefinition* callee;
DefVector args;
if (!iter().readReturnCallRef(&funcTypeIndex, &callee, &args)) {
return false;
}
if (inDeadCode()) {
return true;
}
const FuncType& funcType = codeMeta().types->type(funcTypeIndex).funcType();
DefVector results;
return returnCallRef(funcType, callee, lineOrBytecode, args, &results);
}
bool FunctionCompiler::emitGetLocal() {
uint32_t id;
if (!iter().readGetLocal(&id)) {
return false;
}
iter().setResult(getLocalDef(id));
return true;
}
bool FunctionCompiler::emitSetLocal() {
uint32_t id;
MDefinition* value;
if (!iter().readSetLocal(&id, &value)) {
return false;
}
assign(id, value);
return true;
}
bool FunctionCompiler::emitTeeLocal() {
uint32_t id;
MDefinition* value;
if (!iter().readTeeLocal(&id, &value)) {
return false;
}
assign(id, value);
return true;
}
bool FunctionCompiler::emitGetGlobal() {
uint32_t id;
if (!iter().readGetGlobal(&id)) {
return false;
}
const GlobalDesc& global = codeMeta().globals[id];
if (!global.isConstant()) {
iter().setResult(loadGlobalVar(global));
return true;
}
LitVal value = global.constantValue();
MDefinition* result;
switch (value.type().kind()) {
case ValType::I32:
result = constantI32(int32_t(value.i32()));
break;
case ValType::I64:
result = constantI64(int64_t(value.i64()));
break;
case ValType::F32:
result = constantF32(value.f32());
break;
case ValType::F64:
result = constantF64(value.f64());
break;
case ValType::V128:
#ifdef ENABLE_WASM_SIMD
result = constantV128(value.v128());
break;
#else
return iter().fail(
"Ion has no SIMD support yet");
#endif
case ValType::Ref:
MOZ_ASSERT(value.ref().isNull());
result = constantNullRef(MaybeRefType(value.type().refType()));
break;
default:
MOZ_CRASH(
"unexpected type in EmitGetGlobal");
}
iter().setResult(result);
return true;
}
bool FunctionCompiler::emitSetGlobal() {
uint32_t bytecodeOffset = readBytecodeOffset();
uint32_t id;
MDefinition* value;
if (!iter().readSetGlobal(&id, &value)) {
return false;
}
const GlobalDesc& global = codeMeta().globals[id];
MOZ_ASSERT(global.isMutable());
return storeGlobalVar(bytecodeOffset, global, value);
}
bool FunctionCompiler::emitTeeGlobal() {
uint32_t bytecodeOffset = readBytecodeOffset();
uint32_t id;
MDefinition* value;
if (!iter().readTeeGlobal(&id, &value)) {
return false;
}
const GlobalDesc& global = codeMeta().globals[id];
MOZ_ASSERT(global.isMutable());
return storeGlobalVar(bytecodeOffset, global, value);
}
template <
typename MIRClass>
bool FunctionCompiler::emitUnary(ValType operandType) {
MDefinition* input;
if (!iter().readUnary(operandType, &input)) {
return false;
}
iter().setResult(unary<MIRClass>(input));
return true;
}
template <
typename MIRClass>
bool FunctionCompiler::emitConversion(ValType operandType, ValType resultType) {
MDefinition* input;
if (!iter().readConversion(operandType, resultType, &input)) {
return false;
}
iter().setResult(unary<MIRClass>(input));
return true;
}
template <
typename MIRClass>
bool FunctionCompiler::emitUnaryWithType(ValType operandType, MIRType mirType) {
MDefinition* input;
if (!iter().readUnary(operandType, &input)) {
return false;
}
iter().setResult(unary<MIRClass>(input, mirType));
return true;
}
template <
typename MIRClass>
bool FunctionCompiler::emitConversionWithType(ValType operandType,
ValType resultType,
MIRType mirType) {
MDefinition* input;
if (!iter().readConversion(operandType, resultType, &input)) {
return false;
}
iter().setResult(unary<MIRClass>(input, mirType));
return true;
}
bool FunctionCompiler::emitTruncate(ValType operandType, ValType resultType,
bool isUnsigned,
bool isSaturating) {
MDefinition* input = nullptr;
if (!iter().readConversion(operandType, resultType, &input)) {
return false;
}
TruncFlags flags =
0;
if (isUnsigned) {
flags |= TRUNC_UNSIGNED;
}
if (isSaturating) {
flags |= TRUNC_SATURATING;
}
if (resultType == ValType::I32) {
if (codeMeta().isAsmJS()) {
if (inDeadCode()) {
// The read callsite line, produced by prepareCall, has to be
// consumed -- the MWasmBuiltinTruncateToInt32 and MTruncateToInt32
// will not create MIR node.
(
void)readCallSiteLineOrBytecode();
iter().setResult(nullptr);
}
else if (input && (input->type() == MIRType::
Double ||
input->type() == MIRType::Float32)) {
iter().setResult(unary<MWasmBuiltinTruncateToInt32>(input));
}
else {
iter().setResult(unary<MTruncateToInt32>(input));
}
}
else {
iter().setResult(truncate<MWasmTruncateToInt32>(input, flags));
}
}
else {
MOZ_ASSERT(resultType == ValType::I64);
MOZ_ASSERT(!codeMeta().isAsmJS());
#if defined(JS_CODEGEN_ARM)
iter().setResult(truncateWithInstance(input, flags));
#else
iter().setResult(truncate<MWasmTruncateToInt64>(input, flags));
#endif
}
return true;
}
bool FunctionCompiler::emitSignExtend(uint32_t srcSize, uint32_t targetSize) {
MDefinition* input;
ValType type = targetSize ==
4 ? ValType::I32 : ValType::I64;
if (!iter().readConversion(type, type, &input)) {
return false;
}
iter().setResult(signExtend(input, srcSize, targetSize));
return true;
}
bool FunctionCompiler::emitExtendI32(
bool isUnsigned) {
MDefinition* input;
if (!iter().readConversion(ValType::I32, ValType::I64, &input)) {
return false;
}
iter().setResult(extendI32(input, isUnsigned));
return true;
}
bool FunctionCompiler::emitWrapI32() {
MDefinition* input;
if (!iter().readConversion(ValType::I64, ValType::I32, &input)) {
return false;
}
iter().setResult(wrapI32(input));
return true;
}
bool FunctionCompiler::emitConvertI64ToFloatingPoint(ValType resultType,
MIRType mirType,
bool isUnsigned) {
MDefinition* input;
if (!iter().readConversion(ValType::I64, resultType, &input)) {
return false;
}
iter().setResult(convertI64ToFloatingPoint(input, mirType, isUnsigned));
return true;
}
bool FunctionCompiler::emitReinterpret(ValType resultType, ValType operandType,
MIRType mirType) {
MDefinition* input;
if (!iter().readConversion(operandType, resultType, &input)) {
return false;
}
iter().setResult(unary<MReinterpretCast>(input, mirType));
return true;
}
bool FunctionCompiler::emitAdd(ValType type, MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(type, &lhs, &rhs)) {
return false;
}
iter().setResult(add(lhs, rhs, mirType));
return true;
}
bool FunctionCompiler::emitSub(ValType type, MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(type, &lhs, &rhs)) {
return false;
}
iter().setResult(sub(lhs, rhs, mirType));
return true;
}
bool FunctionCompiler::emitRotate(ValType type,
bool isLeftRotation) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(type, &lhs, &rhs)) {
return false;
}
MDefinition* result = rotate(lhs, rhs, type.toMIRType(), isLeftRotation);
iter().setResult(result);
return true;
}
bool FunctionCompiler::emitBitNot(ValType operandType, MIRType mirType) {
MDefinition* input;
if (!iter().readUnary(operandType, &input)) {
return false;
}
iter().setResult(bitnot(input, mirType));
return true;
}
bool FunctionCompiler::emitBitwiseAndOrXor(
ValType operandType, MIRType mirType,
MWasmBinaryBitwise::SubOpcode subOpc) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
iter().setResult(binary<MWasmBinaryBitwise>(lhs, rhs, mirType, subOpc));
return true;
}
template <
typename MIRClass>
bool FunctionCompiler::emitShift(ValType operandType, MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
iter().setResult(binary<MIRClass>(lhs, rhs, mirType));
return true;
}
bool FunctionCompiler::emitUrsh(ValType operandType, MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
iter().setResult(ursh(lhs, rhs, mirType));
return true;
}
bool FunctionCompiler::emitMul(ValType operandType, MIRType mirType) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
iter().setResult(
mul(lhs, rhs, mirType,
mirType == MIRType::Int32 ? MMul::Integer : MMul::Normal));
return true;
}
bool FunctionCompiler::emitDiv(ValType operandType, MIRType mirType,
bool isUnsigned) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
iter().setResult(div(lhs, rhs, mirType, isUnsigned));
return true;
}
bool FunctionCompiler::emitRem(ValType operandType, MIRType mirType,
bool isUnsigned) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
iter().setResult(mod(lhs, rhs, mirType, isUnsigned));
return true;
}
bool FunctionCompiler::emitMinMax(ValType operandType, MIRType mirType,
bool isMax) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
iter().setResult(minMax(lhs, rhs, mirType, isMax));
return true;
}
bool FunctionCompiler::emitCopySign(ValType operandType) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(operandType, &lhs, &rhs)) {
return false;
}
iter().setResult(binary<MCopySign>(lhs, rhs, operandType.toMIRType()));
return true;
}
bool FunctionCompiler::emitComparison(ValType operandType, JSOp compareOp,
MCompare::CompareType compareType) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readComparison(operandType, &lhs, &rhs)) {
return false;
}
iter().setResult(compare(lhs, rhs, compareOp, compareType));
return true;
}
bool FunctionCompiler::emitSelect(
bool typed) {
StackType type;
MDefinition* trueValue;
MDefinition* falseValue;
MDefinition* condition;
if (!iter().readSelect(typed, &type, &trueValue, &falseValue, &condition)) {
return false;
}
iter().setResult(select(trueValue, falseValue, condition));
return true;
}
bool FunctionCompiler::emitLoad(ValType type, Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
if (!iter().readLoad(type, Scalar::byteSize(viewType), &addr)) {
return false;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(), hugeMemoryEnabled(addr.memoryIndex));
auto* ins = load(addr.base, &access, type);
if (!inDeadCode() && !ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitStore(ValType resultType, Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!iter().readStore(resultType, Scalar::byteSize(viewType), &addr,
&value)) {
return false;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(), hugeMemoryEnabled(addr.memoryIndex));
store(addr.base, &access, value);
return true;
}
bool FunctionCompiler::emitTeeStore(ValType resultType, Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!iter().readTeeStore(resultType, Scalar::byteSize(viewType), &addr,
&value)) {
return false;
}
MOZ_ASSERT(isMem32(addr.memoryIndex));
// asm.js opcode
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(), hugeMemoryEnabled(addr.memoryIndex));
store(addr.base, &access, value);
return true;
}
bool FunctionCompiler::emitTeeStoreWithCoercion(ValType resultType,
Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!iter().readTeeStore(resultType, Scalar::byteSize(viewType), &addr,
&value)) {
return false;
}
if (resultType == ValType::F32 && viewType == Scalar::Float64) {
value = unary<MToDouble>(value);
}
else if (resultType == ValType::F64 && viewType == Scalar::Float32) {
value = unary<MToFloat32>(value);
}
else {
MOZ_CRASH(
"unexpected coerced store");
}
MOZ_ASSERT(isMem32(addr.memoryIndex));
// asm.js opcode
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(), hugeMemoryEnabled(addr.memoryIndex));
store(addr.base, &access, value);
return true;
}
bool FunctionCompiler::tryInlineUnaryBuiltin(SymbolicAddress callee,
MDefinition* input) {
if (!input) {
return false;
}
MOZ_ASSERT(IsFloatingPointType(input->type()));
RoundingMode mode;
if (!IsRoundingFunction(callee, &mode)) {
return false;
}
if (!MNearbyInt::HasAssemblerSupport(mode)) {
return false;
}
iter().setResult(nearbyInt(input, mode));
return true;
}
bool FunctionCompiler::emitUnaryMathBuiltinCall(
const SymbolicAddressSignature& callee) {
MOZ_ASSERT(callee.numArgs ==
1);
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
MDefinition* input;
if (!iter().readUnary(ValType::fromMIRType(callee.argTypes[
0]), &input)) {
return false;
}
if (tryInlineUnaryBuiltin(callee.identity, input)) {
return true;
}
MDefinition* def;
if (!builtinCall1(callee, lineOrBytecode, input, &def)) {
return false;
}
iter().setResult(def);
return true;
}
bool FunctionCompiler::emitBinaryMathBuiltinCall(
const SymbolicAddressSignature& callee) {
MOZ_ASSERT(callee.numArgs ==
2);
MOZ_ASSERT(callee.argTypes[
0] == callee.argTypes[
1]);
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
MDefinition* lhs;
MDefinition* rhs;
// This call to readBinary assumes both operands have the same type.
if (!iter().readBinary(ValType::fromMIRType(callee.argTypes[
0]), &lhs,
&rhs)) {
return false;
}
MDefinition* def;
if (!builtinCall2(callee, lineOrBytecode, lhs, rhs, &def)) {
return false;
}
iter().setResult(def);
return true;
}
bool FunctionCompiler::emitMemoryGrow() {
uint32_t bytecodeOffset = readBytecodeOffset();
MDefinition* delta;
uint32_t memoryIndex;
if (!iter().readMemoryGrow(&memoryIndex, &delta)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* memoryIndexValue = constantI32(int32_t(memoryIndex));
if (!memoryIndexValue) {
return false;
}
const SymbolicAddressSignature& callee =
isMem32(memoryIndex) ? SASigMemoryGrowM32 : SASigMemoryGrowM64;
MDefinition* ret;
if (!emitInstanceCall2(bytecodeOffset, callee, delta, memoryIndexValue,
&ret)) {
return false;
}
iter().setResult(ret);
return true;
}
bool FunctionCompiler::emitMemorySize() {
uint32_t bytecodeOffset = readBytecodeOffset();
uint32_t memoryIndex;
if (!iter().readMemorySize(&memoryIndex)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* memoryIndexValue = constantI32(int32_t(memoryIndex));
if (!memoryIndexValue) {
return false;
}
const SymbolicAddressSignature& callee =
isMem32(memoryIndex) ? SASigMemorySizeM32 : SASigMemorySizeM64;
MDefinition* ret;
if (!emitInstanceCall1(bytecodeOffset, callee, memoryIndexValue, &ret)) {
return false;
}
iter().setResult(ret);
return true;
}
bool FunctionCompiler::emitAtomicCmpXchg(ValType type, Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* oldValue;
MDefinition* newValue;
if (!iter().readAtomicCmpXchg(&addr, type, byteSize(viewType), &oldValue,
&newValue)) {
return false;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(), hugeMemoryEnabled(addr.memoryIndex),
Synchronization::Full());
auto* ins =
atomicCompareExchangeHeap(addr.base, &access, type, oldValue, newValue);
if (!inDeadCode() && !ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitAtomicLoad(ValType type, Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
if (!iter().readAtomicLoad(&addr, type, byteSize(viewType))) {
return false;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(), hugeMemoryEnabled(addr.memoryIndex),
Synchronization::Load());
auto* ins = load(addr.base, &access, type);
if (!inDeadCode() && !ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitAtomicRMW(ValType type, Scalar::Type viewType,
jit::AtomicOp op) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!iter().readAtomicRMW(&addr, type, byteSize(viewType), &value)) {
return false;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(), hugeMemoryEnabled(addr.memoryIndex),
Synchronization::Full());
auto* ins = atomicBinopHeap(op, addr.base, &access, type, value);
if (!inDeadCode() && !ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitAtomicStore(ValType type, Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!iter().readAtomicStore(&addr, type, byteSize(viewType), &value)) {
return false;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(), hugeMemoryEnabled(addr.memoryIndex),
Synchronization::Store());
store(addr.base, &access, value);
return true;
}
bool FunctionCompiler::emitWait(ValType type, uint32_t byteSize) {
MOZ_ASSERT(type == ValType::I32 || type == ValType::I64);
MOZ_ASSERT(type.size() == byteSize);
uint32_t bytecodeOffset = readBytecodeOffset();
LinearMemoryAddress<MDefinition*> addr;
MDefinition* expected;
MDefinition* timeout;
if (!iter().readWait(&addr, type, byteSize, &expected, &timeout)) {
return false;
}
if (inDeadCode()) {
return true;
}
MemoryAccessDesc access(addr.memoryIndex,
type == ValType::I32 ? Scalar::Int32 : Scalar::Int64,
addr.align, addr.offset, trapSiteDesc(),
hugeMemoryEnabled(addr.memoryIndex));
MDefinition* ptr = computeEffectiveAddress(addr.base, &access);
if (!ptr) {
return false;
}
MDefinition* memoryIndex = constantI32(int32_t(addr.memoryIndex));
if (!memoryIndex) {
return false;
}
const SymbolicAddressSignature& callee =
isMem32(addr.memoryIndex)
? (type == ValType::I32 ? SASigWaitI32M32 : SASigWaitI64M32)
: (type == ValType::I32 ? SASigWaitI32M64 : SASigWaitI64M64);
MDefinition* ret;
if (!emitInstanceCall4(bytecodeOffset, callee, ptr, expected, timeout,
memoryIndex, &ret)) {
return false;
}
iter().setResult(ret);
return true;
}
bool FunctionCompiler::emitFence() {
if (!iter().readFence()) {
return false;
}
fence();
return true;
}
bool FunctionCompiler::emitNotify() {
uint32_t bytecodeOffset = readBytecodeOffset();
LinearMemoryAddress<MDefinition*> addr;
MDefinition* count;
if (!iter().readNotify(&addr, &count)) {
return false;
}
if (inDeadCode()) {
return true;
}
MemoryAccessDesc access(addr.memoryIndex, Scalar::Int32, addr.align,
addr.offset, trapSiteDesc(),
hugeMemoryEnabled(addr.memoryIndex));
MDefinition* ptr = computeEffectiveAddress(addr.base, &access);
if (!ptr) {
return false;
}
MDefinition* memoryIndex = constantI32(int32_t(addr.memoryIndex));
if (!memoryIndex) {
return false;
}
const SymbolicAddressSignature& callee =
isMem32(addr.memoryIndex) ? SASigWakeM32 : SASigWakeM64;
MDefinition* ret;
if (!emitInstanceCall3(bytecodeOffset, callee, ptr, count, memoryIndex,
&ret)) {
return false;
}
iter().setResult(ret);
return true;
}
bool FunctionCompiler::emitAtomicXchg(ValType type, Scalar::Type viewType) {
LinearMemoryAddress<MDefinition*> addr;
MDefinition* value;
if (!iter().readAtomicRMW(&addr, type, byteSize(viewType), &value)) {
return false;
}
MemoryAccessDesc access(addr.memoryIndex, viewType, addr.align, addr.offset,
trapSiteDesc(), hugeMemoryEnabled(addr.memoryIndex),
Synchronization::Full());
MDefinition* ins = atomicExchangeHeap(addr.base, &access, type, value);
if (!inDeadCode() && !ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitMemCopyCall(uint32_t dstMemIndex,
uint32_t srcMemIndex, MDefinition* dst,
MDefinition* src, MDefinition* len) {
uint32_t bytecodeOffset = readBytecodeOffset();
if (dstMemIndex == srcMemIndex) {
const SymbolicAddressSignature& callee =
(codeMeta().usesSharedMemory(dstMemIndex)
? (isMem32(dstMemIndex) ? SASigMemCopySharedM32
: SASigMemCopySharedM64)
: (isMem32(dstMemIndex) ? SASigMemCopyM32 : SASigMemCopyM64));
MDefinition* base = memoryBase(dstMemIndex);
if (!base) {
return false;
}
return emitInstanceCall4(bytecodeOffset, callee, dst, src, len, base);
}
AddressType dstIndexType = codeMeta().memories[dstMemIndex].addressType();
AddressType srcIndexType = codeMeta().memories[srcMemIndex].addressType();
if (dstIndexType == AddressType::I32) {
dst = extendI32(dst,
/*isUnsigned=*/true);
if (!dst) {
return false;
}
}
if (srcIndexType == AddressType::I32) {
src = extendI32(src,
/*isUnsigned=*/true);
if (!src) {
return false;
}
}
if (dstIndexType == AddressType::I32 || srcIndexType == AddressType::I32) {
len = extendI32(len,
/*isUnsigned=*/true);
if (!len) {
return false;
}
}
MDefinition* dstMemIndexValue = constantI32(int32_t(dstMemIndex));
if (!dstMemIndexValue) {
return false;
}
MDefinition* srcMemIndexValue = constantI32(int32_t(srcMemIndex));
if (!srcMemIndexValue) {
return false;
}
return emitInstanceCall5(bytecodeOffset, SASigMemCopyAny, dst, src, len,
dstMemIndexValue, srcMemIndexValue);
}
bool FunctionCompiler::emitMemCopyInline(uint32_t memoryIndex, MDefinition* dst,
MDefinition* src, uint32_t length) {
MOZ_ASSERT(length !=
0 && length <= MaxInlineMemoryCopyLength);
// Compute the number of copies of each width we will need to do
size_t remainder = length;
#ifdef ENABLE_WASM_SIMD
size_t numCopies16 =
0;
if (MacroAssembler::SupportsFastUnalignedFPAccesses()) {
numCopies16 = remainder /
sizeof(V128);
remainder %=
sizeof(V128);
}
#endif
#ifdef JS_64BIT
size_t numCopies8 = remainder /
sizeof(uint64_t);
remainder %=
sizeof(uint64_t);
#endif
size_t numCopies4 = remainder /
sizeof(uint32_t);
remainder %=
sizeof(uint32_t);
size_t numCopies2 = remainder /
sizeof(uint16_t);
remainder %=
sizeof(uint16_t);
size_t numCopies1 = remainder;
// Load all source bytes from low to high using the widest transfer width we
// can for the system. We will trap without writing anything if any source
// byte is out-of-bounds.
size_t offset =
0;
DefVector loadedValues;
#ifdef ENABLE_WASM_SIMD
for (uint32_t i =
0; i < numCopies16; i++) {
MemoryAccessDesc access(memoryIndex, Scalar::Simd128,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* loadValue = load(src, &access, ValType::V128);
if (!loadValue || !loadedValues.append(loadValue)) {
return false;
}
offset +=
sizeof(V128);
}
#endif
#ifdef JS_64BIT
for (uint32_t i =
0; i < numCopies8; i++) {
MemoryAccessDesc access(memoryIndex, Scalar::Int64,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* loadValue = load(src, &access, ValType::I64);
if (!loadValue || !loadedValues.append(loadValue)) {
return false;
}
offset +=
sizeof(uint64_t);
}
#endif
for (uint32_t i =
0; i < numCopies4; i++) {
MemoryAccessDesc access(memoryIndex, Scalar::Uint32,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* loadValue = load(src, &access, ValType::I32);
if (!loadValue || !loadedValues.append(loadValue)) {
return false;
}
offset +=
sizeof(uint32_t);
}
if (numCopies2) {
MemoryAccessDesc access(memoryIndex, Scalar::Uint16,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* loadValue = load(src, &access, ValType::I32);
if (!loadValue || !loadedValues.append(loadValue)) {
return false;
}
offset +=
sizeof(uint16_t);
}
if (numCopies1) {
MemoryAccessDesc access(memoryIndex, Scalar::Uint8,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* loadValue = load(src, &access, ValType::I32);
if (!loadValue || !loadedValues.append(loadValue)) {
return false;
}
}
// Store all source bytes to the destination from high to low. We will trap
// without writing anything on the first store if any dest byte is
// out-of-bounds.
offset = length;
if (numCopies1) {
offset -=
sizeof(uint8_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint8,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
store(dst, &access, value);
}
if (numCopies2) {
offset -=
sizeof(uint16_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint16,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
store(dst, &access, value);
}
for (uint32_t i =
0; i < numCopies4; i++) {
offset -=
sizeof(uint32_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint32,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
store(dst, &access, value);
}
#ifdef JS_64BIT
for (uint32_t i =
0; i < numCopies8; i++) {
offset -=
sizeof(uint64_t);
MemoryAccessDesc access(memoryIndex, Scalar::Int64,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
store(dst, &access, value);
}
#endif
#ifdef ENABLE_WASM_SIMD
for (uint32_t i =
0; i < numCopies16; i++) {
offset -=
sizeof(V128);
MemoryAccessDesc access(memoryIndex, Scalar::Simd128,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
auto* value = loadedValues.popCopy();
store(dst, &access, value);
}
#endif
return true;
}
bool FunctionCompiler::emitMemCopy() {
MDefinition *dst, *src, *len;
uint32_t dstMemIndex;
uint32_t srcMemIndex;
if (!iter().readMemOrTableCopy(
true, &dstMemIndex, &dst, &srcMemIndex, &src,
&len)) {
return false;
}
if (inDeadCode()) {
return true;
}
if (dstMemIndex == srcMemIndex && len->isConstant()) {
uint64_t length = isMem32(dstMemIndex) ? len->toConstant()->toInt32()
: len->toConstant()->toInt64();
static_assert(MaxInlineMemoryCopyLength <= UINT32_MAX);
if (length !=
0 && length <= MaxInlineMemoryCopyLength) {
return emitMemCopyInline(dstMemIndex, dst, src, uint32_t(length));
}
}
return emitMemCopyCall(dstMemIndex, srcMemIndex, dst, src, len);
}
bool FunctionCompiler::emitTableCopy() {
MDefinition *dst, *src, *len;
uint32_t dstTableIndex;
uint32_t srcTableIndex;
if (!iter().readMemOrTableCopy(
false, &dstTableIndex, &dst, &srcTableIndex,
&src, &len)) {
return false;
}
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
const TableDesc& dstTable = codeMeta().tables[dstTableIndex];
const TableDesc& srcTable = codeMeta().tables[srcTableIndex];
AddressType dstAddressType = dstTable.addressType();
AddressType srcAddressType = srcTable.addressType();
AddressType lenAddressType =
dstAddressType == AddressType::I64 && srcAddressType == AddressType::I64
? AddressType::I64
: AddressType::I32;
MDefinition* dst32 = clampTableAddressToI32(dstAddressType, dst);
if (!dst32) {
return false;
}
MDefinition* src32 = clampTableAddressToI32(srcAddressType, src);
if (!src32) {
return false;
}
MDefinition* len32 = clampTableAddressToI32(lenAddressType, len);
if (!len32) {
return false;
}
MDefinition* dti = constantI32(int32_t(dstTableIndex));
MDefinition* sti = constantI32(int32_t(srcTableIndex));
return emitInstanceCall5(bytecodeOffset, SASigTableCopy, dst32, src32, len32,
dti, sti);
}
bool FunctionCompiler::emitDataOrElemDrop(
bool isData) {
uint32_t segIndexVal =
0;
if (!iter().readDataOrElemDrop(isData, &segIndexVal)) {
return false;
}
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
MDefinition* segIndex = constantI32(int32_t(segIndexVal));
const SymbolicAddressSignature& callee =
isData ? SASigDataDrop : SASigElemDrop;
return emitInstanceCall1(bytecodeOffset, callee, segIndex);
}
bool FunctionCompiler::emitMemFillCall(uint32_t memoryIndex, MDefinition* start,
MDefinition* val, MDefinition* len) {
MDefinition* base = memoryBase(memoryIndex);
uint32_t bytecodeOffset = readBytecodeOffset();
const SymbolicAddressSignature& callee =
(codeMeta().usesSharedMemory(memoryIndex)
? (isMem32(memoryIndex) ? SASigMemFillSharedM32
: SASigMemFillSharedM64)
: (isMem32(memoryIndex) ? SASigMemFillM32 : SASigMemFillM64));
return emitInstanceCall4(bytecodeOffset, callee, start, val, len, base);
}
bool FunctionCompiler::emitMemFillInline(uint32_t memoryIndex,
MDefinition* start, MDefinition* val,
uint32_t length) {
MOZ_ASSERT(length !=
0 && length <= MaxInlineMemoryFillLength);
uint32_t value = val->toConstant()->toInt32();
// Compute the number of copies of each width we will need to do
size_t remainder = length;
#ifdef ENABLE_WASM_SIMD
size_t numCopies16 =
0;
if (MacroAssembler::SupportsFastUnalignedFPAccesses()) {
numCopies16 = remainder /
sizeof(V128);
remainder %=
sizeof(V128);
}
#endif
#ifdef JS_64BIT
size_t numCopies8 = remainder /
sizeof(uint64_t);
remainder %=
sizeof(uint64_t);
#endif
size_t numCopies4 = remainder /
sizeof(uint32_t);
remainder %=
sizeof(uint32_t);
size_t numCopies2 = remainder /
sizeof(uint16_t);
remainder %=
sizeof(uint16_t);
size_t numCopies1 = remainder;
// Generate splatted definitions for wider fills as needed
#ifdef ENABLE_WASM_SIMD
MDefinition* val16 = numCopies16 ? constantV128(V128(value)) : nullptr;
#endif
#ifdef JS_64BIT
MDefinition* val8 =
numCopies8 ? constantI64(int64_t(SplatByteToUInt<uint64_t>(value,
8)))
: nullptr;
#endif
MDefinition* val4 =
numCopies4 ? constantI32(int32_t(SplatByteToUInt<uint32_t>(value,
4)))
: nullptr;
MDefinition* val2 =
numCopies2 ? constantI32(int32_t(SplatByteToUInt<uint32_t>(value,
2)))
: nullptr;
// Store the fill value to the destination from high to low. We will trap
// without writing anything on the first store if any dest byte is
// out-of-bounds.
size_t offset = length;
if (numCopies1) {
offset -=
sizeof(uint8_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint8,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
store(start, &access, val);
}
if (numCopies2) {
offset -=
sizeof(uint16_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint16,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
store(start, &access, val2);
}
for (uint32_t i =
0; i < numCopies4; i++) {
offset -=
sizeof(uint32_t);
MemoryAccessDesc access(memoryIndex, Scalar::Uint32,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
store(start, &access, val4);
}
#ifdef JS_64BIT
for (uint32_t i =
0; i < numCopies8; i++) {
offset -=
sizeof(uint64_t);
MemoryAccessDesc access(memoryIndex, Scalar::Int64,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
store(start, &access, val8);
}
#endif
#ifdef ENABLE_WASM_SIMD
for (uint32_t i =
0; i < numCopies16; i++) {
offset -=
sizeof(V128);
MemoryAccessDesc access(memoryIndex, Scalar::Simd128,
1, offset,
trapSiteDesc(), hugeMemoryEnabled(memoryIndex));
store(start, &access, val16);
}
#endif
return true;
}
bool FunctionCompiler::emitMemFill() {
uint32_t memoryIndex;
MDefinition *start, *val, *len;
if (!iter().readMemFill(&memoryIndex, &start, &val, &len)) {
return false;
}
if (inDeadCode()) {
return true;
}
if (len->isConstant() && val->isConstant()) {
uint64_t length = isMem32(memoryIndex) ? len->toConstant()->toInt32()
: len->toConstant()->toInt64();
static_assert(MaxInlineMemoryFillLength <= UINT32_MAX);
if (length !=
0 && length <= MaxInlineMemoryFillLength) {
return emitMemFillInline(memoryIndex, start, val, uint32_t(length));
}
}
return emitMemFillCall(memoryIndex, start, val, len);
}
bool FunctionCompiler::emitMemInit() {
uint32_t segIndexVal =
0, dstMemIndex =
0;
MDefinition *dstOff, *srcOff, *len;
if (!iter().readMemOrTableInit(
true, &segIndexVal, &dstMemIndex, &dstOff,
&srcOff, &len)) {
return false;
}
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
const SymbolicAddressSignature& callee =
(isMem32(dstMemIndex) ? SASigMemInitM32 : SASigMemInitM64);
MDefinition* segIndex = constantI32(int32_t(segIndexVal));
if (!segIndex) {
return false;
}
MDefinition* dti = constantI32(int32_t(dstMemIndex));
if (!dti) {
return false;
}
return emitInstanceCall5(bytecodeOffset, callee, dstOff, srcOff, len,
segIndex, dti);
}
bool FunctionCompiler::emitTableInit() {
uint32_t segIndexVal =
0, dstTableIndex =
0;
MDefinition *dstOff, *srcOff, *len;
if (!iter().readMemOrTableInit(
false, &segIndexVal, &dstTableIndex, &dstOff,
&srcOff, &len)) {
return false;
}
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
const TableDesc& table = codeMeta().tables[dstTableIndex];
MDefinition* dstOff32 = clampTableAddressToI32(table.addressType(), dstOff);
if (!dstOff32) {
return false;
}
MDefinition* segIndex = constantI32(int32_t(segIndexVal));
if (!segIndex) {
return false;
}
MDefinition* dti = constantI32(int32_t(dstTableIndex));
if (!dti) {
return false;
}
return emitInstanceCall5(bytecodeOffset, SASigTableInit, dstOff32, srcOff,
len, segIndex, dti);
}
bool FunctionCompiler::emitTableFill() {
uint32_t tableIndex;
MDefinition *start, *val, *len;
if (!iter().readTableFill(&tableIndex, &start, &val, &len)) {
return false;
}
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
const TableDesc& table = codeMeta().tables[tableIndex];
MDefinition* start32 = clampTableAddressToI32(table.addressType(), start);
if (!start32) {
return false;
}
MDefinition* len32 = clampTableAddressToI32(table.addressType(), len);
if (!len32) {
return false;
}
MDefinition* tableIndexArg = constantI32(int32_t(tableIndex));
if (!tableIndexArg) {
return false;
}
return emitInstanceCall4(bytecodeOffset, SASigTableFill, start32, val, len32,
tableIndexArg);
}
#if ENABLE_WASM_MEMORY_CONTROL
bool FunctionCompiler::emitMemDiscard() {
uint32_t memoryIndex;
MDefinition *start, *len;
if (!iter().readMemDiscard(&memoryIndex, &start, &len)) {
return false;
}
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
MDefinition* base = memoryBase(memoryIndex);
bool mem32 = isMem32(memoryIndex);
const SymbolicAddressSignature& callee =
(codeMeta().usesSharedMemory(memoryIndex)
? (mem32 ? SASigMemDiscardSharedM32 : SASigMemDiscardSharedM64)
: (mem32 ? SASigMemDiscardM32 : SASigMemDiscardM64));
return emitInstanceCall3(bytecodeOffset, callee, start, len, base);
}
#endif
bool FunctionCompiler::emitTableGet() {
uint32_t tableIndex;
MDefinition* address;
if (!iter().readTableGet(&tableIndex, &address)) {
return false;
}
if (inDeadCode()) {
return true;
}
const TableDesc& table = codeMeta().tables[tableIndex];
if (table.elemType().tableRepr() == TableRepr::Ref) {
MDefinition* ret = tableGetAnyRef(tableIndex, address);
if (!ret) {
return false;
}
iter().setResult(ret);
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
MDefinition* address32 = clampTableAddressToI32(table.addressType(), address);
if (!address32) {
return false;
}
MDefinition* tableIndexArg = constantI32(int32_t(tableIndex));
if (!tableIndexArg) {
return false;
}
// The return value here is either null, denoting an error, or a short-lived
// pointer to a location containing a possibly-null ref.
MDefinition* ret;
if (!emitInstanceCall2(bytecodeOffset, SASigTableGet, address32,
tableIndexArg, &ret)) {
return false;
}
iter().setResult(ret);
return true;
}
bool FunctionCompiler::emitTableGrow() {
uint32_t tableIndex;
MDefinition* initValue;
MDefinition* delta;
if (!iter().readTableGrow(&tableIndex, &initValue, &delta)) {
return false;
}
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
const TableDesc& table = codeMeta().tables[tableIndex];
MDefinition* delta32 = clampTableAddressToI32(table.addressType(), delta);
if (!delta32) {
return false;
}
MDefinition* tableIndexArg = constantI32(int32_t(tableIndex));
if (!tableIndexArg) {
return false;
}
MDefinition* ret;
if (!emitInstanceCall3(bytecodeOffset, SASigTableGrow, initValue, delta32,
tableIndexArg, &ret)) {
return false;
}
if (table.addressType() == AddressType::I64) {
ret = extendI32(ret,
false);
if (!ret) {
return false;
}
}
iter().setResult(ret);
return true;
}
bool FunctionCompiler::emitTableSet() {
uint32_t tableIndex;
MDefinition* address;
MDefinition* value;
if (!iter().readTableSet(&tableIndex, &address, &value)) {
return false;
}
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
const TableDesc& table = codeMeta().tables[tableIndex];
if (table.elemType().tableRepr() == TableRepr::Ref) {
return tableSetAnyRef(tableIndex, address, value, bytecodeOffset);
}
MDefinition* address32 = clampTableAddressToI32(table.addressType(), address);
if (!address32) {
return false;
}
MDefinition* tableIndexArg = constantI32(int32_t(tableIndex));
if (!tableIndexArg) {
return false;
}
return emitInstanceCall3(bytecodeOffset, SASigTableSet, address32, value,
tableIndexArg);
}
bool FunctionCompiler::emitTableSize() {
uint32_t tableIndex;
if (!iter().readTableSize(&tableIndex)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* length = loadTableLength(tableIndex);
if (!length) {
return false;
}
iter().setResult(length);
return true;
}
bool FunctionCompiler::emitRefFunc() {
uint32_t funcIndex;
if (!iter().readRefFunc(&funcIndex)) {
return false;
}
if (inDeadCode()) {
return true;
}
uint32_t bytecodeOffset = readBytecodeOffset();
MDefinition* funcIndexArg = constantI32(int32_t(funcIndex));
if (!funcIndexArg) {
return false;
}
// The return value here is either null, denoting an error, or a short-lived
// pointer to a location containing a possibly-null ref.
MDefinition* ret;
if (!emitInstanceCall1(bytecodeOffset, SASigRefFunc, funcIndexArg, &ret)) {
return false;
}
iter().setResult(ret);
return true;
}
bool FunctionCompiler::emitRefNull() {
RefType type;
if (!iter().readRefNull(&type)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* nullVal = constantNullRef(MaybeRefType(type));
if (!nullVal) {
return false;
}
iter().setResult(nullVal);
return true;
}
bool FunctionCompiler::emitRefIsNull() {
MDefinition* input;
RefType sourceType;
if (!iter().readRefIsNull(&input, &sourceType)) {
return false;
}
if (inDeadCode()) {
return true;
}
// Some types are not castable (right now just continuations). The casting
// machinery cannot handle them. So emit a direct null-check for now.
if (!sourceType.isCastable()) {
MDefinition* isNull = compareIsNull(input, JSOp::Eq);
if (!isNull) {
return false;
}
iter().setResult(isNull);
return true;
}
// ref.is_null is implemented as a ref.test against the bottom type of the
// input ref's hierarchy. This will codegen to a simple null comparison, but
// allows this op to participate in other optimizations surrounding ref.test
// and ref.cast.
MDefinition* test = refTest(input, sourceType.bottomType());
if (!test) {
return false;
}
iter().setResult(test);
return true;
}
//////////////////////////////////////////////////////////////////////////////
//
// Wide Arithmetic support
bool FunctionCompiler::emitI64AddSub128(
bool isAdd) {
MDefinition* xLo;
MDefinition* xHi;
MDefinition* yLo;
MDefinition* yHi;
if (!iter().readBinaryI128(&xLo, &xHi, &yLo, &yHi)) {
return false;
}
if (inDeadCode()) {
return true;
}
// Compute zHi:zLo = xHi:xLo +/- yHi:yLo.
MInstruction* zHi;
MInstruction* zLo;
#ifdef JS_64BIT
// 64 bit implementation. Produce inline code.
if (isAdd) {
zLo = MAdd::NewWasm(alloc(), xLo, yLo, MIRType::Int64);
}
else {
zLo = MSub::NewWasm(alloc(), xLo, yLo, MIRType::Int64,
/*mustPreserveNaN=*/false);
}
if (!zLo) {
return false;
}
curBlock_->add(zLo);
zHi = MWasmAddSubI128HI64::
New(alloc(), xLo, xHi, yLo, yHi, isAdd);
if (!zHi) {
return false;
}
curBlock_->add(zHi);
#else
// 32 bit implementation. Call a helper function. The arguments and return
// value are passed in Instance::baselineScratchWords_[0..7]; see
// Instance::addSubI128 for details.
MDefinition* storeSequence[
4] = {xLo, xHi, yLo, yHi};
for (
int i =
0; i <
4; i++) {
auto* store = MWasmStoreInstanceScratch2xI32::
New(
alloc(),
/*byteOffset=*/i * 8, storeSequence[i], instancePointer_);
if (!store) {
return false;
}
curBlock_->add(store);
}
MDefinition* mIsAdd = constantI32(isAdd ?
1 :
0);
if (!mIsAdd ||
!emitInstanceCall1(readBytecodeOffset(), SASigAddSubI128, mIsAdd)) {
return false;
}
zLo = MWasmLoadInstanceScratch2xI32::
New(alloc(),
/*byteOffset=*/0,
instancePointer_);
if (!zLo) {
return false;
}
curBlock_->add(zLo);
zHi = MWasmLoadInstanceScratch2xI32::
New(alloc(),
/*byteOffset=*/8,
instancePointer_);
if (!zHi) {
return false;
}
curBlock_->add(zHi);
#endif // JS_64BIT
DefVector results;
if (!results.reserve(
2)) {
return false;
}
results.infallibleAppend(zLo);
results.infallibleAppend(zHi);
iter().setResults(results.length(), results);
return true;
}
bool FunctionCompiler::emitI64MulWide(
bool isSigned) {
MDefinition* x;
MDefinition* y;
if (!iter().readBinaryI64Wide(&x, &y)) {
return false;
}
if (inDeadCode()) {
return true;
}
// Compute zHi:zLo = x *widen y
MInstruction* zLo;
MInstruction* zHi;
#ifdef JS_64BIT
// 64 bit implementation. Produce inline code.
// We can compute the low and high halves in either order. However, the
// RiscV specification advises computing the high half first, in the hope
// that microarchitectures can merge it with the immediately following low
// half multiply, hence avoiding the duplicate multiply.
zHi = MWasmMulI64WideHI64::
New(alloc(), x, y, isSigned);
if (!zHi) {
return false;
}
curBlock_->add(zHi);
zLo = MMul::NewWasm(alloc(), x, y, MIRType::Int64,
/*mode=*/MMul::Normal, /*mustPreserveNaN=*/false);
if (!zLo) {
return false;
}
curBlock_->add(zLo);
#else
// 32 bit implementation. Call a helper function. The arguments and return
// value are passed in Instance::baselineScratchWords_[0..4]; see
// Instance::mulI64Wide for details.
auto* store = MWasmStoreInstanceScratch2xI32::
New(alloc(),
/*byteOffset=*/0, x,
instancePointer_);
if (!store) {
return false;
}
curBlock_->add(store);
store = MWasmStoreInstanceScratch2xI32::
New(alloc(),
/*byteOffset=*/8, y,
instancePointer_);
if (!store) {
return false;
}
curBlock_->add(store);
MDefinition* mIsSigned = constantI32(isSigned ?
1 :
0);
if (!mIsSigned ||
!emitInstanceCall1(readBytecodeOffset(), SASigMulI64Wide, mIsSigned)) {
return false;
}
zLo = MWasmLoadInstanceScratch2xI32::
New(alloc(),
/*byteOffset=*/0,
instancePointer_);
if (!zLo) {
return false;
}
curBlock_->add(zLo);
zHi = MWasmLoadInstanceScratch2xI32::
New(alloc(),
/*byteOffset=*/8,
instancePointer_);
if (!zHi) {
return false;
}
curBlock_->add(zHi);
#endif
DefVector results;
if (!results.reserve(
2)) {
return false;
}
results.infallibleAppend(zLo);
results.infallibleAppend(zHi);
iter().setResults(results.length(), results);
return true;
}
//////////////////////////////////////////////////////////////////////////////
//
// SIMD support
#ifdef ENABLE_WASM_SIMD
bool FunctionCompiler::emitConstSimd128() {
V128 v128;
if (!iter().readV128Const(&v128)) {
return false;
}
iter().setResult(constantV128(v128));
return true;
}
bool FunctionCompiler::emitBinarySimd128(
bool commutative, SimdOp op) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readBinary(ValType::V128, &lhs, &rhs)) {
return false;
}
iter().setResult(binarySimd128(lhs, rhs, commutative, op));
return true;
}
bool FunctionCompiler::emitTernarySimd128(wasm::SimdOp op) {
MDefinition* v0;
MDefinition* v1;
MDefinition* v2;
if (!iter().readTernary(ValType::V128, &v0, &v1, &v2)) {
return false;
}
iter().setResult(ternarySimd128(v0, v1, v2, op));
return true;
}
bool FunctionCompiler::emitShiftSimd128(SimdOp op) {
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readVectorShift(&lhs, &rhs)) {
return false;
}
iter().setResult(shiftSimd128(lhs, rhs, op));
return true;
}
bool FunctionCompiler::emitSplatSimd128(ValType inType, SimdOp op) {
MDefinition* src;
if (!iter().readConversion(inType, ValType::V128, &src)) {
return false;
}
iter().setResult(scalarToSimd128(src, op));
return true;
}
bool FunctionCompiler::emitUnarySimd128(SimdOp op) {
MDefinition* src;
if (!iter().readUnary(ValType::V128, &src)) {
return false;
}
iter().setResult(unarySimd128(src, op));
return true;
}
bool FunctionCompiler::emitReduceSimd128(SimdOp op) {
MDefinition* src;
if (!iter().readConversion(ValType::V128, ValType::I32, &src)) {
return false;
}
iter().setResult(reduceSimd128(src, op, ValType::I32));
return true;
}
bool FunctionCompiler::emitExtractLaneSimd128(ValType outType,
uint32_t laneLimit, SimdOp op) {
uint32_t laneIndex;
MDefinition* src;
if (!iter().readExtractLane(outType, laneLimit, &laneIndex, &src)) {
return false;
}
iter().setResult(reduceSimd128(src, op, outType, laneIndex));
return true;
}
bool FunctionCompiler::emitReplaceLaneSimd128(ValType laneType,
uint32_t laneLimit, SimdOp op) {
uint32_t laneIndex;
MDefinition* lhs;
MDefinition* rhs;
if (!iter().readReplaceLane(laneType, laneLimit, &laneIndex, &lhs, &rhs)) {
return false;
}
iter().setResult(replaceLaneSimd128(lhs, rhs, laneIndex, op));
return true;
}
bool FunctionCompiler::emitShuffleSimd128() {
MDefinition* v1;
MDefinition* v2;
V128 control;
if (!iter().readVectorShuffle(&v1, &v2, &control)) {
return false;
}
iter().setResult(shuffleSimd128(v1, v2, control));
return true;
}
bool FunctionCompiler::emitLoadSplatSimd128(Scalar::Type viewType,
wasm::SimdOp splatOp) {
LinearMemoryAddress<MDefinition*> addr;
if (!iter().readLoadSplat(Scalar::byteSize(viewType), &addr)) {
return false;
}
auto* ins = loadSplatSimd128(viewType, addr, splatOp);
if (!inDeadCode() && !ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitLoadExtendSimd128(wasm::SimdOp op) {
LinearMemoryAddress<MDefinition*> addr;
if (!iter().readLoadExtend(&addr)) {
return false;
}
auto* ins = loadExtendSimd128(addr, op);
if (!inDeadCode() && !ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitLoadZeroSimd128(Scalar::Type viewType,
size_t numBytes) {
LinearMemoryAddress<MDefinition*> addr;
if (!iter().readLoadSplat(numBytes, &addr)) {
return false;
}
auto* ins = loadZeroSimd128(viewType, numBytes, addr);
if (!inDeadCode() && !ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitLoadLaneSimd128(uint32_t laneSize) {
uint32_t laneIndex;
MDefinition* src;
LinearMemoryAddress<MDefinition*> addr;
if (!iter().readLoadLane(laneSize, &addr, &laneIndex, &src)) {
return false;
}
auto* ins = loadLaneSimd128(laneSize, addr, laneIndex, src);
if (!inDeadCode() && !ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitStoreLaneSimd128(uint32_t laneSize) {
uint32_t laneIndex;
MDefinition* src;
LinearMemoryAddress<MDefinition*> addr;
if (!iter().readStoreLane(laneSize, &addr, &laneIndex, &src)) {
return false;
}
storeLaneSimd128(laneSize, addr, laneIndex, src);
return true;
}
#endif // ENABLE_WASM_SIMD
bool FunctionCompiler::emitRefAsNonNull() {
MDefinition* ref;
if (!iter().readRefAsNonNull(&ref)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* ins = refAsNonNull(ref);
if (!ins) {
return false;
}
iter().setResult(ins);
return true;
}
bool FunctionCompiler::emitBrOnNull() {
uint32_t relativeDepth;
ResultType type;
DefVector values;
MDefinition* condition;
BranchHint branchHint =
iter().getBranchHint(funcIndex(), relativeBytecodeOffset());
if (branchHint == BranchHint::Invalid) {
branchHint = BranchHint::Unlikely;
}
if (!iter().readBrOnNull(&relativeDepth, &type, &values, &condition)) {
return false;
}
return brOnNull(relativeDepth, values, type, condition, branchHint);
}
bool FunctionCompiler::emitBrOnNonNull() {
uint32_t relativeDepth;
ResultType type;
DefVector values;
MDefinition* condition;
BranchHint branchHint =
iter().getBranchHint(funcIndex(), relativeBytecodeOffset());
if (branchHint == BranchHint::Invalid) {
branchHint = BranchHint::Likely;
}
if (!iter().readBrOnNonNull(&relativeDepth, &type, &values, &condition)) {
return false;
}
return brOnNonNull(relativeDepth, values, type, condition, branchHint);
}
// Speculatively inline a call_refs that are likely to target the expected
// function index in this module. A fallback for if the actual callee is not
// any of the speculated expected callees is always generated. This leads to a
// control flow chain that is roughly:
//
// if (ref.func $expectedFuncIndex_1) == actualCalleeFunc:
// (call_inline $expectedFuncIndex1)
// else if (ref.func $expectedFuncIndex_2) == actualCalleeFunc:
// (call_inline $expectedFuncIndex2)
// ...
// else:
// (call_ref actualCalleeFunc)
//
bool FunctionCompiler::emitSpeculativeInlineCallRef(
uint32_t bytecodeOffset,
const FuncType& funcType,
CallRefHint expectedFuncIndices, MDefinition* actualCalleeFunc,
const DefVector& args, DefVector* results) {
// There must be at least one speculative target.
MOZ_ASSERT(!expectedFuncIndices.empty());
// Perform an up front null check on the callee function reference.
actualCalleeFunc = refAsNonNull(actualCalleeFunc);
if (!actualCalleeFunc) {
return false;
}
constexpr size_t numElseBlocks = CallRefHint::NUM_ENTRIES +
1;
Vector<MBasicBlock*, numElseBlocks, SystemAllocPolicy> elseBlocks;
if (!elseBlocks.reserve(numElseBlocks)) {
return false;
}
for (uint32_t i =
0; i < expectedFuncIndices.length(); i++) {
uint32_t funcIndex = expectedFuncIndices.get(i);
// Load the cached value of `ref.func $expectedFuncIndex` for comparing
// against `actualCalleeFunc`. This cached value may be null if the
// `ref.func` for the expected function has not been executed in this
// runtime session.
//
// This is okay because we have done a null check on the `actualCalleeFunc`
// already and so comparing it against a null expected callee func will
// return false and fall back to the general case. This can only happen if
// we've deserialized a cached module in a different session, and then run
// the code without ever acquiring a reference to the expected function. In
// that case, the expected callee could never be the target of this
// call_ref, so performing the fallback path is the right thing to do
// anyways.
MDefinition* expectedCalleeFunc = loadCachedRefFunc(funcIndex);
if (!expectedCalleeFunc) {
return false;
}
// Check if the callee funcref we have is equals to the expected callee
// funcref we're inlining.
MDefinition* isExpectedCallee =
compare(actualCalleeFunc, expectedCalleeFunc, JSOp::Eq,
MCompare::Compare_WasmAnyRef);
if (!isExpectedCallee) {
return false;
}
// Start a 'then' block, which will have the inlined code
MBasicBlock* elseBlock;
if (!branchAndStartThen(isExpectedCallee, &elseBlock)) {
return false;
}
// Inline the expected callee as we do with direct calls
DefVector inlineResults;
if (!emitInlineCall(funcType, funcIndex,
InliningHeuristics::CallKind::CallRef, args,
&inlineResults)) {
return false;
}
// Push the results for joining with the 'else' block
if (!pushDefs(inlineResults)) {
return false;
}
// Switch to the 'else' block which will have, either the check for the
// next target, or the fallback `call_ref` if we're out of targets.
if (!switchToElse(elseBlock, &elseBlock)) {
return false;
}
elseBlocks.infallibleAppend(elseBlock);
}
// The block that performs the fallback call should be cold.
curBlock_->setFrequency(Frequency::Unlikely);
DefVector callResults;
if (!callRef(funcType, actualCalleeFunc, bytecodeOffset, args,
&callResults)) {
return false;
}
// Push the results for joining with the 'then' block
if (!pushDefs(callResults)) {
return false;
}
// Join the various branches together
for (uint32_t i = elseBlocks.length() -
1; i !=
0; i--) {
DefVector results;
if (!joinIfElse(elseBlocks[i], &results) || !pushDefs(results)) {
return false;
}
}
return joinIfElse(elseBlocks[
0], results);
}
bool FunctionCompiler::emitCallRef() {
uint32_t bytecodeOffset = readBytecodeOffset();
uint32_t funcTypeIndex;
MDefinition* callee;
DefVector args;
if (!iter().readCallRef(&funcTypeIndex, &callee, &args)) {
return false;
}
// We must unconditionally read a call_ref hint so that we stay in sync with
// how baseline generates them.
CallRefHint hint = readCallRefHint();
if (inDeadCode()) {
return true;
}
const FuncType& funcType = codeMeta().types->type(funcTypeIndex).funcType();
// Ask the inlining heuristics which entries in `hint` we are allowed to
// inline.
CallRefHint approved =
auditInlineableCallees(InliningHeuristics::CallKind::CallRef, hint);
if (!approved.empty()) {
DefVector results;
if (!emitSpeculativeInlineCallRef(bytecodeOffset, funcType, approved,
callee, args, &results)) {
return false;
}
iter().setResults(results.length(), results);
return true;
}
DefVector results;
if (!callRef(funcType, callee, bytecodeOffset, args, &results)) {
return false;
}
iter().setResults(results.length(), results);
return true;
}
bool FunctionCompiler::emitStructNew() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t typeIndex;
DefVector args;
if (!iter().readStructNew(&typeIndex, &args)) {
return false;
}
uint32_t allocSiteIndex = readAllocSiteIndex(typeIndex);
if (inDeadCode()) {
return true;
}
const TypeDef&
typeDef = (*codeMeta().types)[typeIndex];
const StructType& structType =
typeDef.structType();
MOZ_ASSERT(args.length() == structType.fields_.length());
MDefinition* structObject =
createStructObject(typeIndex, allocSiteIndex,
false);
if (!structObject) {
return false;
}
// And fill in the fields.
for (uint32_t fieldIndex =
0; fieldIndex < structType.fields_.length();
fieldIndex++) {
if (!mirGen().ensureBallast()) {
return false;
}
if (!writeValueToStructField(lineOrBytecode, structType, fieldIndex,
structObject, args[fieldIndex],
WasmPreBarrierKind::None)) {
return false;
}
}
iter().setResult(structObject);
return true;
}
bool FunctionCompiler::emitStructNewDefault() {
uint32_t typeIndex;
if (!iter().readStructNewDefault(&typeIndex)) {
return false;
}
uint32_t allocSiteIndex = readAllocSiteIndex(typeIndex);
if (inDeadCode()) {
return true;
}
MDefinition* structObject =
createStructObject(typeIndex, allocSiteIndex,
true);
if (!structObject) {
return false;
}
iter().setResult(structObject);
return true;
}
bool FunctionCompiler::emitStructSet() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t typeIndex;
uint32_t fieldIndex;
MDefinition* structObject;
MDefinition* value;
if (!iter().readStructSet(&typeIndex, &fieldIndex, &structObject, &value)) {
return false;
}
if (inDeadCode()) {
return true;
}
// Check for null is done at writeValueToStructField.
// And fill in the field.
const StructType& structType = (*codeMeta().types)[typeIndex].structType();
return writeValueToStructField(lineOrBytecode, structType, fieldIndex,
structObject, value,
WasmPreBarrierKind::Normal);
}
bool FunctionCompiler::emitStructGet(FieldWideningOp wideningOp) {
uint32_t typeIndex;
uint32_t fieldIndex;
MDefinition* structObject;
if (!iter().readStructGet(&typeIndex, &fieldIndex, wideningOp,
&structObject)) {
return false;
}
if (inDeadCode()) {
return true;
}
// Check for null is done at readValueFromStructField.
// And fetch the data.
const StructType& structType = (*codeMeta().types)[typeIndex].structType();
MDefinition* load = readValueFromStructField(structType, fieldIndex,
wideningOp, structObject);
if (!load) {
return false;
}
iter().setResult(load);
return true;
}
bool FunctionCompiler::emitArrayNew() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t typeIndex;
MDefinition* numElements;
MDefinition* fillValue;
if (!iter().readArrayNew(&typeIndex, &numElements, &fillValue)) {
return false;
}
uint32_t allocSiteIndex = readAllocSiteIndex(typeIndex);
if (inDeadCode()) {
return true;
}
// Create the array object, uninitialized.
MDefinition* arrayObject =
createArrayObject(typeIndex, allocSiteIndex, numElements,
/*zeroFields=*/false);
if (!arrayObject) {
return false;
}
const ArrayType& arrayType = (*codeMeta().types)[typeIndex].arrayType();
if (!fillArray(lineOrBytecode, arrayType, arrayObject, constantI32(
0),
numElements, fillValue, WasmPreBarrierKind::None,
WasmPostBarrierKind::None)) {
return false;
}
if (arrayType.elementType().isRefRepr()) {
// Emit one whole-cell post barrier for the whole array, since there is just
// one object and one value.
if (!postBarrierWholeCell(lineOrBytecode, arrayObject, fillValue)) {
return false;
}
}
iter().setResult(arrayObject);
return true;
}
bool FunctionCompiler::emitArrayNewDefault() {
// This is almost identical to EmitArrayNew, except we skip the
// initialisation loop.
uint32_t typeIndex;
MDefinition* numElements;
if (!iter().readArrayNewDefault(&typeIndex, &numElements)) {
return false;
}
uint32_t allocSiteIndex = readAllocSiteIndex(typeIndex);
if (inDeadCode()) {
return true;
}
// Create the array object, default-initialized.
MDefinition* arrayObject =
createArrayObject(typeIndex, allocSiteIndex, numElements,
/*zeroFields=*/true);
if (!arrayObject) {
return false;
}
iter().setResult(arrayObject);
return true;
}
bool FunctionCompiler::emitArrayNewFixed() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t typeIndex, numElements;
DefVector values;
if (!iter().readArrayNewFixed(&typeIndex, &numElements, &values)) {
return false;
}
MOZ_ASSERT(values.length() == numElements);
uint32_t allocSiteIndex = readAllocSiteIndex(typeIndex);
if (inDeadCode()) {
return true;
}
MDefinition* numElementsDef = constantI32(int32_t(numElements));
if (!numElementsDef) {
return false;
}
// Create the array object, uninitialized.
const ArrayType& arrayType = (*codeMeta().types)[typeIndex].arrayType();
StorageType elemType = arrayType.elementType();
uint32_t elemSize = elemType.size();
MDefinition* arrayObject =
createArrayObject(typeIndex, allocSiteIndex, numElementsDef,
/*zeroFields=*/false);
if (!arrayObject) {
return false;
}
// Make `base` point at the first byte of the (OOL) data area.
MDefinition* base = getWasmArrayObjectData(arrayObject);
if (!base) {
return false;
}
// Write each element in turn.
// How do we know that the offset expression `i * elemSize` below remains
// within 2^31 (signed-i32) range? In the worst case we will have 16-byte
// values, and there can be at most MaxFunctionBytes expressions, if it were
// theoretically possible to generate one expression per instruction byte.
// Hence the max offset we can be expected to generate is
// `16 * MaxFunctionBytes`.
static_assert(
16 /* sizeof v128 */ * MaxFunctionBytes <=
MaxArrayPayloadBytes);
MOZ_RELEASE_ASSERT(numElements <= MaxFunctionBytes);
for (uint32_t i =
0; i < numElements; i++) {
if (!mirGen().ensureBallast()) {
return false;
}
// `i * elemSize` is made safe by the assertions above.
if (!writeGcValueAtBasePlusOffset(
lineOrBytecode, elemType, arrayObject, AliasSet::WasmArrayDataArea,
values[numElements -
1 - i], base, i * elemSize, i,
false,
WasmPreBarrierKind::None, WasmPostBarrierKind::WholeCell)) {
return false;
}
}
iter().setResult(arrayObject);
return true;
}
bool FunctionCompiler::emitArrayNewData() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t typeIndex, segIndex;
MDefinition* segByteOffset;
MDefinition* numElements;
if (!iter().readArrayNewData(&typeIndex, &segIndex, &segByteOffset,
&numElements)) {
return false;
}
uint32_t allocSiteIndex = readAllocSiteIndex(typeIndex);
if (inDeadCode()) {
return true;
}
MDefinition* typeIndexValue = constantI32(int32_t(typeIndex));
if (!typeIndexValue) {
return false;
}
MDefinition* allocSite = loadAllocSiteInstanceData(allocSiteIndex);
if (!allocSite) {
return false;
}
// Other values we need to pass to the instance call:
MDefinition* segIndexM = constantI32(int32_t(segIndex));
if (!segIndexM) {
return false;
}
// Create call:
// arrayObject = Instance::arrayNewData(segByteOffset:u32, numElements:u32,
// typeDefData:word, segIndex:u32)
// If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by
// this call will trap.
MDefinition* arrayObject;
if (!emitInstanceCall5(lineOrBytecode, SASigArrayNewData, segByteOffset,
numElements, typeIndexValue, allocSite, segIndexM,
&arrayObject)) {
return false;
}
iter().setResult(arrayObject);
return true;
}
bool FunctionCompiler::emitArrayNewElem() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t typeIndex, segIndex;
MDefinition* segElemIndex;
MDefinition* numElements;
if (!iter().readArrayNewElem(&typeIndex, &segIndex, &segElemIndex,
&numElements)) {
return false;
}
uint32_t allocSiteIndex = readAllocSiteIndex(typeIndex);
if (inDeadCode()) {
return true;
}
MDefinition* typeIndexValue = constantI32(int32_t(typeIndex));
if (!typeIndexValue) {
return false;
}
MDefinition* allocSite = loadAllocSiteInstanceData(allocSiteIndex);
if (!allocSite) {
return false;
}
// Other values we need to pass to the instance call:
MDefinition* segIndexM = constantI32(int32_t(segIndex));
if (!segIndexM) {
return false;
}
// Create call:
// arrayObject = Instance::arrayNewElem(segElemIndex:u32, numElements:u32,
// typeDefData:word, segIndex:u32)
// If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by
// this call will trap.
MDefinition* arrayObject;
if (!emitInstanceCall5(lineOrBytecode, SASigArrayNewElem, segElemIndex,
numElements, typeIndexValue, allocSite, segIndexM,
&arrayObject)) {
return false;
}
iter().setResult(arrayObject);
return true;
}
bool FunctionCompiler::emitArrayInitData() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t unusedTypeIndex, segIndex;
MDefinition* array;
MDefinition* arrayIndex;
MDefinition* segOffset;
MDefinition* length;
if (!iter().readArrayInitData(&unusedTypeIndex, &segIndex, &array,
&arrayIndex, &segOffset, &length)) {
return false;
}
if (inDeadCode()) {
return true;
}
// Other values we need to pass to the instance call:
MDefinition* segIndexM = constantI32(int32_t(segIndex));
if (!segIndexM) {
return false;
}
// Create call:
// Instance::arrayInitData(array:word, index:u32, segByteOffset:u32,
// numElements:u32, segIndex:u32) If the requested size exceeds
// MaxArrayPayloadBytes, the MIR generated by this call will trap.
return emitInstanceCall5(lineOrBytecode, SASigArrayInitData, array,
arrayIndex, segOffset, length, segIndexM);
}
bool FunctionCompiler::emitArrayInitElem() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t typeIndex, segIndex;
MDefinition* array;
MDefinition* arrayIndex;
MDefinition* segOffset;
MDefinition* length;
if (!iter().readArrayInitElem(&typeIndex, &segIndex, &array, &arrayIndex,
&segOffset, &length)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* typeIndexValue = constantI32(int32_t(typeIndex));
if (!typeIndexValue) {
return false;
}
// Other values we need to pass to the instance call:
MDefinition* segIndexM = constantI32(int32_t(segIndex));
if (!segIndexM) {
return false;
}
// Create call:
// Instance::arrayInitElem(array:word, index:u32, segByteOffset:u32,
// numElements:u32, typeDefData:word, segIndex:u32) If the requested size
// exceeds MaxArrayPayloadBytes, the MIR generated by this call will trap.
return emitInstanceCall6(lineOrBytecode, SASigArrayInitElem, array,
arrayIndex, segOffset, length, typeIndexValue,
segIndexM);
}
bool FunctionCompiler::emitArraySet() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t typeIndex;
MDefinition* value;
MDefinition* index;
MDefinition* arrayObject;
if (!iter().readArraySet(&typeIndex, &value, &index, &arrayObject)) {
return false;
}
if (inDeadCode()) {
return true;
}
// Check for null is done at setupForArrayAccess.
// Create the object null check and the array bounds check and get the OOL
// data pointer.
MDefinition* base = setupForArrayAccess(arrayObject, index);
if (!base) {
return false;
}
// And do the store.
const ArrayType& arrayType = (*codeMeta().types)[typeIndex].arrayType();
StorageType elemType = arrayType.elementType();
uint32_t elemSize = elemType.size();
MOZ_ASSERT(elemSize >=
1 && elemSize <=
16);
return writeGcValueAtBasePlusScaledIndex(
lineOrBytecode, elemType, arrayObject, AliasSet::WasmArrayDataArea, value,
base, elemSize, index, WasmPreBarrierKind::Normal,
WasmPostBarrierKind::Edge);
}
bool FunctionCompiler::emitArrayGet(FieldWideningOp wideningOp) {
uint32_t typeIndex;
MDefinition* index;
MDefinition* arrayObject;
if (!iter().readArrayGet(&typeIndex, wideningOp, &index, &arrayObject)) {
return false;
}
if (inDeadCode()) {
return true;
}
// Check for null is done at setupForArrayAccess.
// Create the object null check and the array bounds check and get the data
// pointer.
MDefinition* base = setupForArrayAccess(arrayObject, index);
if (!base) {
return false;
}
// And do the load.
const ArrayType& arrayType = (*codeMeta().types)[typeIndex].arrayType();
StorageType elemType = arrayType.elementType();
MDefinition* load =
readGcArrayValueAtIndex(elemType, wideningOp, arrayObject,
AliasSet::WasmArrayDataArea, base, index);
if (!load) {
return false;
}
iter().setResult(load);
return true;
}
bool FunctionCompiler::emitArrayLen() {
MDefinition* arrayObject;
if (!iter().readArrayLen(&arrayObject)) {
return false;
}
if (inDeadCode()) {
return true;
}
// Check for null is done at getWasmArrayObjectNumElements.
// Get the size value for the array
MDefinition* numElements = getWasmArrayObjectNumElements(arrayObject);
if (!numElements) {
return false;
}
iter().setResult(numElements);
return true;
}
bool FunctionCompiler::emitArrayCopy() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t dstArrayTypeIndex;
uint32_t srcArrayTypeIndex;
MDefinition* dstArrayObject;
MDefinition* dstArrayIndex;
MDefinition* srcArrayObject;
MDefinition* srcArrayIndex;
MDefinition* numElements;
if (!iter().readArrayCopy(&dstArrayTypeIndex, &srcArrayTypeIndex,
&dstArrayObject, &dstArrayIndex, &srcArrayObject,
&srcArrayIndex, &numElements)) {
return false;
}
if (inDeadCode()) {
return true;
}
const ArrayType& dstArrayType =
codeMeta().types->type(dstArrayTypeIndex).arrayType();
StorageType dstElemType = dstArrayType.elementType();
int32_t elemSize = int32_t(dstElemType.size());
bool elemsAreRefTyped = dstElemType.isRefType();
return createArrayCopy(lineOrBytecode, dstArrayObject, dstArrayIndex,
srcArrayObject, srcArrayIndex, numElements, elemSize,
elemsAreRefTyped);
}
bool FunctionCompiler::emitArrayFill() {
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t typeIndex;
MDefinition* array;
MDefinition* index;
MDefinition* val;
MDefinition* numElements;
if (!iter().readArrayFill(&typeIndex, &array, &index, &val, &numElements)) {
return false;
}
if (inDeadCode()) {
return true;
}
return createArrayFill(lineOrBytecode, typeIndex, array, index, val,
numElements);
}
bool FunctionCompiler::emitRefI31() {
MDefinition* input;
if (!iter().readConversion(ValType::I32,
ValType(RefType::i31().asNonNullable()), &input)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* output = refI31(input);
if (!output) {
return false;
}
iter().setResult(output);
return true;
}
bool FunctionCompiler::emitI31Get(FieldWideningOp wideningOp) {
MOZ_ASSERT(wideningOp != FieldWideningOp::None);
MDefinition* input;
if (!iter().readConversion(ValType(RefType::i31()), ValType::I32, &input)) {
return false;
}
if (inDeadCode()) {
return true;
}
input = refAsNonNull(input);
if (!input) {
return false;
}
MDefinition* output = i31Get(input, wideningOp);
if (!output) {
return false;
}
iter().setResult(output);
return true;
}
bool FunctionCompiler::emitRefTest(
bool nullable) {
MDefinition* ref;
RefType sourceType;
RefType destType;
if (!iter().readRefTest(nullable, &sourceType, &destType, &ref)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* success = refTest(ref, destType);
if (!success) {
return false;
}
iter().setResult(success);
return true;
}
bool FunctionCompiler::emitRefCast(
bool nullable) {
MDefinition* ref;
RefType sourceType;
RefType destType;
if (!iter().readRefCast(nullable, &sourceType, &destType, &ref)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* castedRef = refCast(ref, destType);
if (!castedRef) {
return false;
}
iter().setResult(castedRef);
return true;
}
bool FunctionCompiler::emitBrOnCast(
bool onSuccess) {
uint32_t labelRelativeDepth;
RefType sourceType;
RefType destType;
ResultType labelType;
DefVector values;
BranchHint branchHint =
iter().getBranchHint(funcIndex(), relativeBytecodeOffset());
if (branchHint == BranchHint::Invalid) {
branchHint = onSuccess ? BranchHint::Likely : BranchHint::Unlikely;
}
if (!iter().readBrOnCast(onSuccess, &labelRelativeDepth, &sourceType,
&destType, &labelType, &values)) {
return false;
}
return brOnCastCommon(onSuccess, labelRelativeDepth, sourceType, destType,
labelType, values, branchHint);
}
bool FunctionCompiler::emitAnyConvertExtern() {
MDefinition* ref;
if (!iter().readRefConversion(RefType::extern_(), RefType::any(), &ref)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* conversion = convertAnyExtern(ref, wasm::RefType::Kind::Any);
if (!conversion) {
return false;
}
iter().setResult(conversion);
return true;
}
bool FunctionCompiler::emitExternConvertAny() {
MDefinition* ref;
if (!iter().readRefConversion(RefType::any(), RefType::extern_(), &ref)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* conversion = convertAnyExtern(ref, wasm::RefType::Kind::
Extern);
if (!conversion) {
return false;
}
iter().setResult(conversion);
return true;
}
bool FunctionCompiler::emitCallBuiltinModuleFunc() {
const BuiltinModuleFunc* builtinModuleFunc;
DefVector params;
if (!iter().readCallBuiltinModuleFunc(&builtinModuleFunc, ¶ms)) {
return false;
}
return callBuiltinModuleFunc(*builtinModuleFunc, params);
}
#ifdef ENABLE_WASM_JSPI
bool FunctionCompiler::emitContNew() {
uint32_t typeIndex;
MDefinition* func;
if (!iter().readContNew(&typeIndex, &func)) {
return false;
}
const TypeDef&
typeDef = codeMeta().types->type(typeIndex);
const ContType& contType =
typeDef.contType();
// TODO: Temporary restriction that cont type cannot have params or results.
if (!contType.funcType().args().empty() ||
!contType.funcType().results().empty()) {
unimplementedTrap();
return true;
}
if (inDeadCode()) {
return true;
}
MDefinition* result = nullptr;
if (!emitInstanceCall1(readBytecodeOffset(), SASigContNew, func, &result)) {
return false;
}
iter().setResult(result);
return true;
}
bool FunctionCompiler::emitContBind() {
uint32_t inputContTypeIndex;
uint32_t outputContTypeIndex;
DefVector boundArgs;
MDefinition* cont;
if (!iter().readContBind(&inputContTypeIndex, &outputContTypeIndex,
&boundArgs, &cont)) {
return false;
}
if (inDeadCode()) {
return true;
}
// TODO: Not yet implemented
unimplementedTrap();
return true;
}
bool FunctionCompiler::emitStoreSuspendParams(
MDefinition* paramsArea,
const ValTypeVector& suspendTagParams,
const DefVector& suspendParams, MDefinition* suspendedCont) {
// Last param is the cont, it goes first in stack results.
size_t paramsAreaOffset =
0;
MWasmStackResultArea::StackResult loc(paramsAreaOffset,
js::jit::MIRType::WasmAnyRef);
MWasmStoreStackResult* storeCont = MWasmStoreStackResult::
New(
alloc(), paramsArea, paramsAreaOffset, suspendedCont);
if (!storeCont) {
return false;
}
curBlock_->add(storeCont);
paramsAreaOffset = loc.endOffset();
// The rest are suspendTagParams, again in reverse order.
for (uint32_t i =
0; i < suspendTagParams.length(); i++) {
if (!mirGen().ensureBallast()) {
return false;
}
size_t reverseIndex = suspendTagParams.length() - i -
1;
ValType handlerParam = suspendTagParams[reverseIndex];
MWasmStackResultArea::StackResult loc(paramsAreaOffset,
handlerParam.toMIRType());
MDefinition* param = suspendParams[reverseIndex];
MWasmStoreStackResult* store = MWasmStoreStackResult::
New(
alloc(), paramsArea, paramsAreaOffset, param);
if (!store) {
return false;
}
curBlock_->add(store);
paramsAreaOffset = loc.endOffset();
}
return true;
}
bool FunctionCompiler::emitSuspend() {
uint32_t tagIndex;
DefVector suspendParams;
if (!iter().readSuspend(&tagIndex, &suspendParams)) {
return false;
}
if (inDeadCode()) {
return true;
}
const TagDesc& tagDesc = codeMeta().tags[tagIndex];
const TagType& tagType = *tagDesc.type;
// TODO: Temporary restriction that suspend tags cannot have results.
if (!tagType.resultTypes().empty()) {
unimplementedTrap();
return true;
}
// TODO: Temporary restriction that we can't be in a try block yet. A
// resume_throw will be able to trigger an exception that we need to handle.
if (inTryCode()) {
unimplementedTrap();
return true;
}
// Load the tag we're going to search for.
MDefinition* tag = loadTag(tagIndex);
if (!tag) {
return false;
}
// Search for the handler for this tag. When this suspend is used for JS-PI
// this find handler will be infallible because we emit a different find
// handler in emitGuardSuspending (which throws an exception instead of a
// trap).
MWasmFindHandler* handler =
MWasmFindHandler::
New(alloc(), instancePointer_, tag,
Trap::NullPointerDereference, trapSiteDesc());
if (!handler) {
return false;
}
curBlock_->add(handler);
// Load the paramsArea from the handler.
MWasmLoadInstance* paramsArea =
MWasmLoadInstance::
New(alloc(), handler,
offsetof(wasm::Handler, target) +
offsetof(wasm::SwitchTarget, paramsArea),
MIRType::Pointer, AliasSet::None());
if (!paramsArea) {
return false;
}
curBlock_->add(paramsArea);
// Allocate a new continuation that will hold our stack when we suspend.
MDefinition* suspendedCont = nullptr;
if (!emitInstanceCall0(readBytecodeOffset(), SASigContNewEmpty,
&suspendedCont)) {
return false;
}
// Store all the params into the handler params area.
if (!emitStoreSuspendParams(paramsArea, tagType.argTypes(), suspendParams,
suspendedCont)) {
return false;
}
// Emit the suspend instruction.
MWasmSuspend* suspend =
MWasmSuspend::
New(alloc(), instancePointer_, suspendedCont, handler,
callSiteDesc(CallSiteKind::StackSwitch));
if (!suspend) {
return false;
}
curBlock_->add(suspend);
return true;
}
bool FunctionCompiler::emitResume() {
uint32_t typeIndex;
MDefinition* cont;
DefVector args;
HandlerExprVector handlers;
if (!iter().readResume(&typeIndex, &handlers, &args, &cont)) {
return false;
}
if (inDeadCode()) {
return true;
}
// TODO: Temporary restriction that cont type cannot have params or results.
const TypeDef&
typeDef = codeMeta().types->type(typeIndex);
const ContType& contType =
typeDef.contType();
if (!contType.funcType().args().empty() ||
!contType.funcType().results().empty()) {
unimplementedTrap();
return true;
}
// TODO: Temporary restriction that suspend tags cannot have params or
// results.
for (
const HandlerExpr& handler : handlers) {
const TagDesc& tagDesc = codeMeta().tags[handler.tagIndex()];
const TagType& tagType = *tagDesc.type;
if (handler.isSwitch() || !tagType.resultTypes().empty()) {
unimplementedTrap();
return true;
}
}
// Start with a resume barrier which will mark the stack if it hasn't yet and
// we're in an incremental GC.
MWasmResumeBarrier* barrier =
MWasmResumeBarrier::
New(alloc(), instancePointer_, cont);
if (!barrier) {
return false;
}
curBlock_->add(barrier);
MBasicBlock* fallthroughBlock = nullptr;
MBasicBlock* prePadBlock = nullptr;
ControlInstructionVector* tryLandingPadPatches = nullptr;
mozilla::Maybe<uint32_t> tryNote;
if (inTryBlock(&tryLandingPadPatches)) {
tryNote.emplace(
0);
if (!rootCompiler_.addTryNote(tryNote.ptr())) {
return false;
}
if (!newBlock(curBlock_, &prePadBlock)) {
return false;
}
}
if (!newBlock(curBlock_, &fallthroughBlock)) {
return false;
}
size_t numResultsAreaItems =
0;
for (size_t i =
0; i < handlers.length(); i++) {
const HandlerExpr& handler = handlers[i];
MOZ_ASSERT(!handler.isSwitch());
numResultsAreaItems +=
1 + codeMeta().getTagType(handler.tagIndex()).argTypes().length();
}
MWasmStackResultArea* handlersResultArea = nullptr;
if (numResultsAreaItems) {
handlersResultArea = MWasmStackResultArea::
New(alloc());
if (!handlersResultArea ||
!handlersResultArea->init(alloc(), numResultsAreaItems)) {
return false;
}
curBlock_->add(handlersResultArea);
}
MWasmResume* resume =
MWasmResume::
New(alloc(), callSiteDesc(CallSiteKind::StackSwitch),
tryNote, instancePointer_, cont, handlersResultArea);
if (!resume ||
!resume->init(fallthroughBlock, prePadBlock, handlers.length())) {
return false;
}
curBlock_->end(resume);
MBasicBlock* resumeBlock = curBlock_;
curBlock_ = nullptr;
// emitHandlerLandingPads
size_t currentResultsAreaIndex =
0;
size_t currentResultsAreaOffset =
0;
for (size_t i =
0; i < handlers.length(); i++) {
MOZ_ASSERT(!curBlock_);
const HandlerExpr& handler = handlers[i];
MOZ_ASSERT(!handler.isSwitch());
uint32_t baseResultsAreaByteOffset = currentResultsAreaOffset;
const TagDesc& tagDesc = codeMeta().tags[handler.tagIndex()];
const TagType& tagType = *tagDesc.type;
ResultType suspendTagParams = tagType.argResultType();
// Last param is the cont, it goes first in stack results.
MWasmStackResultArea::StackResult loc(currentResultsAreaOffset,
js::jit::MIRType::WasmAnyRef);
handlersResultArea->initResult(currentResultsAreaIndex, loc);
currentResultsAreaIndex++;
currentResultsAreaOffset = loc.endOffset();
// The rest are suspendTagParams, again in reverse order.
for (uint32_t i =
0; i < suspendTagParams.length(); i++) {
size_t reverseIndex = suspendTagParams.length() - i -
1;
ValType handlerParam = suspendTagParams[reverseIndex];
MWasmStackResultArea::StackResult loc(currentResultsAreaOffset,
handlerParam.toMIRType());
handlersResultArea->initResult(currentResultsAreaIndex, loc);
currentResultsAreaIndex++;
currentResultsAreaOffset = loc.endOffset();
}
MBasicBlock* target;
if (!newBlock(resumeBlock, &target)) {
return false;
}
curBlock_ = target;
if (!resume->initHandler(
i, codeMeta().offsetOfTagInstanceData(handler.tagIndex()),
baseResultsAreaByteOffset, target)) {
return false;
}
DefVector resultDefs;
size_t suspendLabelParams = suspendTagParams.length() +
1;
for (uint32_t i =
0; i < suspendLabelParams; i++) {
if (!mirGen().ensureBallast()) {
return false;
}
size_t stackResultIndex = currentResultsAreaIndex - i -
1;
MWasmStackResult* stackResult =
MWasmStackResult::
New(alloc(), handlersResultArea, stackResultIndex);
if (!stackResult || !resultDefs.append(stackResult)) {
return false;
}
curBlock_->add(stackResult);
}
if (!br(handler.labelDepth(), resultDefs)) {
return false;
}
curBlock_ = nullptr;
}
MOZ_ASSERT(!curBlock_);
if (tryNote.isSome()) {
// Switch to the prePadBlock
curBlock_ = prePadBlock;
// Mark this as the landing pad for the call
curBlock_->add(MWasmCallLandingPrePad::
New(alloc(), resumeBlock, *tryNote));
// End with a pending jump to the landing pad
if (!endWithPadPatch(tryLandingPadPatches)) {
return false;
}
}
// Compilation continues in the fallthroughBlock.
curBlock_ = fallthroughBlock;
return true;
}
bool FunctionCompiler::emitResumeThrow() {
uint32_t typeIndex;
uint32_t tagIndex;
MDefinition* cont;
DefVector args;
HandlerExprVector handlers;
if (!iter().readResumeThrow(&typeIndex, &tagIndex, &handlers, &args, &cont)) {
return false;
}
if (inDeadCode()) {
return true;
}
// TODO: Not yet implemented.
unimplementedTrap();
return true;
}
bool FunctionCompiler::emitResumeThrowRef() {
uint32_t contTypeIndex;
MDefinition* cont;
MDefinition* exception;
HandlerExprVector handlers;
if (!iter().readResumeThrowRef(&contTypeIndex, &handlers, &exception,
&cont)) {
return false;
}
if (inDeadCode()) {
return true;
}
// TODO: Not yet implemented.
unimplementedTrap();
return true;
}
bool FunctionCompiler::emitSwitch() {
uint32_t contTypeIndex;
uint32_t tagIndex;
MDefinition* cont;
DefVector args;
HandlerExprVector handlers;
if (!iter().readSwitch(&contTypeIndex, &tagIndex, &args, &cont)) {
return false;
}
if (inDeadCode()) {
return true;
}
// TODO: Not yet implemented.
unimplementedTrap();
return true;
}
bool FunctionCompiler::emitGuardSuspending() {
uint32_t tagIndex;
if (!iter().readGuardSuspending(&tagIndex)) {
return false;
}
if (inDeadCode()) {
return true;
}
MDefinition* tag = loadTag(tagIndex);
if (!tag) {
return false;
}
// This will throw an exception (not a trap) which would require us to add
// branches to catch blocks. However this is only used in self-hosted wasm
// code for JS-PI where we ensure there is no try blocks.
MOZ_ASSERT(!inTryCode());
MWasmFindHandler* handler = MWasmFindHandler::
New(
alloc(), instancePointer_, tag, Trap::ThrowSuspendError, trapSiteDesc());
if (!handler) {
return false;
}
curBlock_->add(handler);
return true;
}
#endif // ENABLE_WASM_JSPI
bool FunctionCompiler::emitBodyExprs() {
if (!iter().startFunction(funcIndex())) {
return false;
}
#define CHECK(c) \
if (!(c))
return false; \
break
while (
true) {
if (!mirGen().ensureBallast()) {
return false;
}
OpBytes op;
if (!iter().readOp(&op)) {
return false;
}
switch (op.b0) {
case uint16_t(Op::End):
if (!emitEnd()) {
return false;
}
if (iter().controlStackEmpty()) {
return true;
}
break;
// Control opcodes
case uint16_t(Op::Unreachable):
CHECK(emitUnreachable());
case uint16_t(Op::Nop):
CHECK(iter().readNop());
case uint16_t(Op::Block):
CHECK(emitBlock());
case uint16_t(Op::Loop):
CHECK(emitLoop());
case uint16_t(Op::
If):
CHECK(emitIf());
case uint16_t(Op::
Else):
CHECK(emitElse());
case uint16_t(Op::
Try):
CHECK(emitTry());
case uint16_t(Op::
Catch):
CHECK(emitCatch());
case uint16_t(Op::CatchAll):
CHECK(emitCatchAll());
case uint16_t(Op::Delegate):
CHECK(emitDelegate());
case uint16_t(Op::
Throw):
CHECK(emitThrow());
case uint16_t(Op::Rethrow):
CHECK(emitRethrow());
case uint16_t(Op::ThrowRef):
CHECK(emitThrowRef());
case uint16_t(Op::TryTable):
CHECK(emitTryTable());
case uint16_t(Op::Br):
CHECK(emitBr());
case uint16_t(Op::BrIf):
CHECK(emitBrIf());
case uint16_t(Op::BrTable):
CHECK(emitBrTable());
case uint16_t(Op::
Return):
CHECK(emitReturn());
// Calls
case uint16_t(Op::Call):
CHECK(emitCall(
/* asmJSFuncDef = */ false));
case uint16_t(Op::CallIndirect):
CHECK(emitCallIndirect(
/* oldStyle = */ false));
// Parametric operators
case uint16_t(Op::Drop):
CHECK(iter().readDrop());
case uint16_t(Op::SelectNumeric):
CHECK(emitSelect(
/*typed*/ false));
case uint16_t(Op::SelectTyped):
CHECK(emitSelect(
/*typed*/ true));
// Locals and globals
case uint16_t(Op::LocalGet):
CHECK(emitGetLocal());
case uint16_t(Op::LocalSet):
CHECK(emitSetLocal());
case uint16_t(Op::LocalTee):
CHECK(emitTeeLocal());
case uint16_t(Op::GlobalGet):
CHECK(emitGetGlobal());
case uint16_t(Op::GlobalSet):
CHECK(emitSetGlobal());
case uint16_t(Op::TableGet):
CHECK(emitTableGet());
case uint16_t(Op::TableSet):
CHECK(emitTableSet());
// Memory-related operators
case uint16_t(Op::I32Load):
CHECK(emitLoad(ValType::I32, Scalar::Int32));
case uint16_t(Op::I64Load):
CHECK(emitLoad(ValType::I64, Scalar::Int64));
case uint16_t(Op::F32Load):
CHECK(emitLoad(ValType::F32, Scalar::Float32));
case uint16_t(Op::F64Load):
CHECK(emitLoad(ValType::F64, Scalar::Float64));
case uint16_t(Op::I32Load8S):
CHECK(emitLoad(ValType::I32, Scalar::Int8));
case uint16_t(Op::I32Load8U):
CHECK(emitLoad(ValType::I32, Scalar::Uint8));
case uint16_t(Op::I32Load16S):
CHECK(emitLoad(ValType::I32, Scalar::Int16));
case uint16_t(Op::I32Load16U):
CHECK(emitLoad(ValType::I32, Scalar::Uint16));
case uint16_t(Op::I64Load8S):
CHECK(emitLoad(ValType::I64, Scalar::Int8));
case uint16_t(Op::I64Load8U):
CHECK(emitLoad(ValType::I64, Scalar::Uint8));
case uint16_t(Op::I64Load16S):
CHECK(emitLoad(ValType::I64, Scalar::Int16));
case uint16_t(Op::I64Load16U):
CHECK(emitLoad(ValType::I64, Scalar::Uint16));
case uint16_t(Op::I64Load32S):
CHECK(emitLoad(ValType::I64, Scalar::Int32));
case uint16_t(Op::I64Load32U):
CHECK(emitLoad(ValType::I64, Scalar::Uint32));
case uint16_t(Op::I32Store):
CHECK(emitStore(ValType::I32, Scalar::Int32));
case uint16_t(Op::I64Store):
CHECK(emitStore(ValType::I64, Scalar::Int64));
case uint16_t(Op::F32Store):
CHECK(emitStore(ValType::F32, Scalar::Float32));
case uint16_t(Op::F64Store):
CHECK(emitStore(ValType::F64, Scalar::Float64));
case uint16_t(Op::I32Store8):
CHECK(emitStore(ValType::I32, Scalar::Int8));
case uint16_t(Op::I32Store16):
CHECK(emitStore(ValType::I32, Scalar::Int16));
case uint16_t(Op::I64Store8):
CHECK(emitStore(ValType::I64, Scalar::Int8));
case uint16_t(Op::I64Store16):
CHECK(emitStore(ValType::I64, Scalar::Int16));
case uint16_t(Op::I64Store32):
CHECK(emitStore(ValType::I64, Scalar::Int32));
case uint16_t(Op::MemorySize):
CHECK(emitMemorySize());
case uint16_t(Op::MemoryGrow):
CHECK(emitMemoryGrow());
// Constants
case uint16_t(Op::I32Const):
CHECK(emitI32Const());
case uint16_t(Op::I64Const):
CHECK(emitI64Const());
case uint16_t(Op::F32Const):
CHECK(emitF32Const());
case uint16_t(Op::F64Const):
CHECK(emitF64Const());
// Comparison operators
case uint16_t(Op::I32Eqz):
CHECK(emitConversion<MNot>(ValType::I32, ValType::I32));
case uint16_t(Op::I32Eq):
CHECK(emitComparison(ValType::I32, JSOp::Eq, MCompare::Compare_Int32));
case uint16_t(Op::I32Ne):
CHECK(emitComparison(ValType::I32, JSOp::Ne, MCompare::Compare_Int32));
case uint16_t(Op::I32LtS):
CHECK(emitComparison(ValType::I32, JSOp::Lt, MCompare::Compare_Int32));
case uint16_t(Op::I32LtU):
CHECK(emitComparison(ValType::I32, JSOp::Lt, MCompare::Compare_UInt32));
case uint16_t(Op::I32GtS):
CHECK(emitComparison(ValType::I32, JSOp::Gt, MCompare::Compare_Int32));
case uint16_t(Op::I32GtU):
CHECK(emitComparison(ValType::I32, JSOp::Gt, MCompare::Compare_UInt32));
case uint16_t(Op::I32LeS):
CHECK(emitComparison(ValType::I32, JSOp::Le, MCompare::Compare_Int32));
case uint16_t(Op::I32LeU):
CHECK(emitComparison(ValType::I32, JSOp::Le, MCompare::Compare_UInt32));
case uint16_t(Op::I32GeS):
CHECK(emitComparison(ValType::I32, JSOp::Ge, MCompare::Compare_Int32));
case uint16_t(Op::I32GeU):
CHECK(emitComparison(ValType::I32, JSOp::Ge, MCompare::Compare_UInt32));
case uint16_t(Op::I64Eqz):
CHECK(emitConversion<MNot>(ValType::I64, ValType::I32));
case uint16_t(Op::I64Eq):
CHECK(emitComparison(ValType::I64, JSOp::Eq, MCompare::Compare_Int64));
case uint16_t(Op::I64Ne):
CHECK(emitComparison(ValType::I64, JSOp::Ne, MCompare::Compare_Int64));
case uint16_t(Op::I64LtS):
CHECK(emitComparison(ValType::I64, JSOp::Lt, MCompare::Compare_Int64));
case uint16_t(Op::I64LtU):
CHECK(emitComparison(ValType::I64, JSOp::Lt, MCompare::Compare_UInt64));
case uint16_t(Op::I64GtS):
CHECK(emitComparison(ValType::I64, JSOp::Gt, MCompare::Compare_Int64));
case uint16_t(Op::I64GtU):
CHECK(emitComparison(ValType::I64, JSOp::Gt, MCompare::Compare_UInt64));
case uint16_t(Op::I64LeS):
CHECK(emitComparison(ValType::I64, JSOp::Le, MCompare::Compare_Int64));
case uint16_t(Op::I64LeU):
CHECK(emitComparison(ValType::I64, JSOp::Le, MCompare::Compare_UInt64));
case uint16_t(Op::I64GeS):
CHECK(emitComparison(ValType::I64, JSOp::Ge, MCompare::Compare_Int64));
case uint16_t(Op::I64GeU):
CHECK(emitComparison(ValType::I64, JSOp::Ge, MCompare::Compare_UInt64));
case uint16_t(Op::F32Eq):
CHECK(
emitComparison(ValType::F32, JSOp::Eq, MCompare::Compare_Float32));
case uint16_t(Op::F32Ne):
CHECK(
emitComparison(ValType::F32, JSOp::Ne, MCompare::Compare_Float32));
case uint16_t(Op::F32Lt):
CHECK(
emitComparison(ValType::F32, JSOp::Lt, MCompare::Compare_Float32));
case uint16_t(Op::F32Gt):
CHECK(
emitComparison(ValType::F32, JSOp::Gt, MCompare::Compare_Float32));
case uint16_t(Op::F32Le):
CHECK(
emitComparison(ValType::F32, JSOp::Le, MCompare::Compare_Float32));
case uint16_t(Op::F32Ge):
CHECK(
emitComparison(ValType::F32, JSOp::Ge, MCompare::Compare_Float32));
case uint16_t(Op::F64Eq):
CHECK(emitComparison(ValType::F64, JSOp::Eq, MCompare::Compare_Double));
case uint16_t(Op::F64Ne):
CHECK(emitComparison(ValType::F64, JSOp::Ne, MCompare::Compare_Double));
case uint16_t(Op::F64Lt):
CHECK(emitComparison(ValType::F64, JSOp::Lt, MCompare::Compare_Double));
case uint16_t(Op::F64Gt):
CHECK(emitComparison(ValType::F64, JSOp::Gt, MCompare::Compare_Double));
case uint16_t(Op::F64Le):
CHECK(emitComparison(ValType::F64, JSOp::Le, MCompare::Compare_Double));
case uint16_t(Op::F64Ge):
CHECK(emitComparison(ValType::F64, JSOp::Ge, MCompare::Compare_Double));
// Numeric operators
case uint16_t(Op::I32Clz):
CHECK(emitUnaryWithType<MClz>(ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Ctz):
CHECK(emitUnaryWithType<MCtz>(ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Popcnt):
CHECK(emitUnaryWithType<MPopcnt>(ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Add):
CHECK(emitAdd(ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Sub):
CHECK(emitSub(ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Mul):
CHECK(emitMul(ValType::I32, MIRType::Int32));
case uint16_t(Op::I32DivS):
case uint16_t(Op::I32DivU):
CHECK(emitDiv(ValType::I32, MIRType::Int32, Op(op.b0) == Op::I32DivU));
case uint16_t(Op::I32RemS):
case uint16_t(Op::I32RemU):
CHECK(emitRem(ValType::I32, MIRType::Int32, Op(op.b0) == Op::I32RemU));
case uint16_t(Op::I32And):
CHECK(emitBitwiseAndOrXor(ValType::I32, MIRType::Int32,
MWasmBinaryBitwise::SubOpcode::
And));
case uint16_t(Op::I32Or):
CHECK(emitBitwiseAndOrXor(ValType::I32, MIRType::Int32,
MWasmBinaryBitwise::SubOpcode::
Or));
case uint16_t(Op::I32Xor):
CHECK(emitBitwiseAndOrXor(ValType::I32, MIRType::Int32,
MWasmBinaryBitwise::SubOpcode::
Xor));
case uint16_t(Op::I32Shl):
CHECK(emitShift<MLsh>(ValType::I32, MIRType::Int32));
case uint16_t(Op::I32ShrS):
CHECK(emitShift<MRsh>(ValType::I32, MIRType::Int32));
case uint16_t(Op::I32ShrU):
CHECK(emitUrsh(ValType::I32, MIRType::Int32));
case uint16_t(Op::I32Rotl):
case uint16_t(Op::I32Rotr):
CHECK(emitRotate(ValType::I32, Op(op.b0) == Op::I32Rotl));
case uint16_t(Op::I64Clz):
CHECK(emitUnaryWithType<MClz>(ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Ctz):
CHECK(emitUnaryWithType<MCtz>(ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Popcnt):
CHECK(emitUnaryWithType<MPopcnt>(ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Add):
CHECK(emitAdd(ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Sub):
CHECK(emitSub(ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Mul):
CHECK(emitMul(ValType::I64, MIRType::Int64));
case uint16_t(Op::I64DivS):
case uint16_t(Op::I64DivU):
CHECK(emitDiv(ValType::I64, MIRType::Int64, Op(op.b0) == Op::I64DivU));
case uint16_t(Op::I64RemS):
case uint16_t(Op::I64RemU):
CHECK(emitRem(ValType::I64, MIRType::Int64, Op(op.b0) == Op::I64RemU));
case uint16_t(Op::I64And):
CHECK(emitBitwiseAndOrXor(ValType::I64, MIRType::Int64,
MWasmBinaryBitwise::SubOpcode::
And));
case uint16_t(Op::I64Or):
CHECK(emitBitwiseAndOrXor(ValType::I64, MIRType::Int64,
MWasmBinaryBitwise::SubOpcode::
Or));
case uint16_t(Op::I64Xor):
CHECK(emitBitwiseAndOrXor(ValType::I64, MIRType::Int64,
MWasmBinaryBitwise::SubOpcode::
Xor));
case uint16_t(Op::I64Shl):
CHECK(emitShift<MLsh>(ValType::I64, MIRType::Int64));
case uint16_t(Op::I64ShrS):
CHECK(emitShift<MRsh>(ValType::I64, MIRType::Int64));
case uint16_t(Op::I64ShrU):
CHECK(emitUrsh(ValType::I64, MIRType::Int64));
case uint16_t(Op::I64Rotl):
case uint16_t(Op::I64Rotr):
CHECK(emitRotate(ValType::I64, Op(op.b0) == Op::I64Rotl));
case uint16_t(Op::F32Abs):
CHECK(emitUnaryWithType<MAbs>(ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Neg):
CHECK(emitUnaryWithType<MWasmNeg>(ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Ceil):
CHECK(emitUnaryMathBuiltinCall(SASigCeilF));
case uint16_t(Op::F32Floor):
CHECK(emitUnaryMathBuiltinCall(SASigFloorF));
case uint16_t(Op::F32Trunc):
CHECK(emitUnaryMathBuiltinCall(SASigTruncF));
case uint16_t(Op::F32Nearest):
CHECK(emitUnaryMathBuiltinCall(SASigNearbyIntF));
case uint16_t(Op::F32Sqrt):
CHECK(emitUnaryWithType<MSqrt>(ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Add):
CHECK(emitAdd(ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Sub):
CHECK(emitSub(ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Mul):
CHECK(emitMul(ValType::F32, MIRType::Float32));
case uint16_t(Op::F32Div):
CHECK(emitDiv(ValType::F32, MIRType::Float32,
/* isUnsigned = */ false));
case uint16_t(Op::F32Min):
case uint16_t(Op::F32Max):
CHECK(emitMinMax(ValType::F32, MIRType::Float32,
Op(op.b0) == Op::F32Max));
case uint16_t(Op::F32CopySign):
CHECK(emitCopySign(ValType::F32));
case uint16_t(Op::F64Abs):
CHECK(emitUnaryWithType<MAbs>(ValType::F64, MIRType::
Double));
case uint16_t(Op::F64Neg):
CHECK(emitUnaryWithType<MWasmNeg>(ValType::F64, MIRType::
Double));
case uint16_t(Op::F64Ceil):
CHECK(emitUnaryMathBuiltinCall(SASigCeilD));
case uint16_t(Op::F64Floor):
CHECK(emitUnaryMathBuiltinCall(SASigFloorD));
case uint16_t(Op::F64Trunc):
CHECK(emitUnaryMathBuiltinCall(SASigTruncD));
case uint16_t(Op::F64Nearest):
CHECK(emitUnaryMathBuiltinCall(SASigNearbyIntD));
case uint16_t(Op::F64Sqrt):
CHECK(emitUnaryWithType<MSqrt>(ValType::F64, MIRType::
Double));
case uint16_t(Op::F64Add):
CHECK(emitAdd(ValType::F64, MIRType::
Double));
case uint16_t(Op::F64Sub):
CHECK(emitSub(ValType::F64, MIRType::
Double));
case uint16_t(Op::F64Mul):
CHECK(emitMul(ValType::F64, MIRType::
Double));
case uint16_t(Op::F64Div):
CHECK(emitDiv(ValType::F64, MIRType::
Double,
/* isUnsigned = */ false));
case uint16_t(Op::F64Min):
case uint16_t(Op::F64Max):
CHECK(
emitMinMax(ValType::F64, MIRType::
Double, Op(op.b0) == Op::F64Max));
case uint16_t(Op::F64CopySign):
CHECK(emitCopySign(ValType::F64));
// Conversions
case uint16_t(Op::I32WrapI64):
CHECK(emitWrapI32());
case uint16_t(Op::I32TruncF32S):
case uint16_t(Op::I32TruncF32U):
CHECK(emitTruncate(ValType::F32, ValType::I32,
Op(op.b0) == Op::I32TruncF32U,
false));
case uint16_t(Op::I32TruncF64S):
case uint16_t(Op::I32TruncF64U):
CHECK(emitTruncate(ValType::F64, ValType::I32,
Op(op.b0) == Op::I32TruncF64U,
false));
case uint16_t(Op::I64ExtendI32S):
case uint16_t(Op::I64ExtendI32U):
CHECK(emitExtendI32(Op(op.b0) == Op::I64ExtendI32U));
case uint16_t(Op::I64TruncF32S):
case uint16_t(Op::I64TruncF32U):
CHECK(emitTruncate(ValType::F32, ValType::I64,
Op(op.b0) == Op::I64TruncF32U,
false));
case uint16_t(Op::I64TruncF64S):
case uint16_t(Op::I64TruncF64U):
CHECK(emitTruncate(ValType::F64, ValType::I64,
Op(op.b0) == Op::I64TruncF64U,
false));
case uint16_t(Op::F32ConvertI32S):
CHECK(emitConversion<MToFloat32>(ValType::I32, ValType::F32));
case uint16_t(Op::F32ConvertI32U):
CHECK(
emitConversion<MWasmUnsignedToFloat32>(ValType::I32, ValType::F32));
case uint16_t(Op::F32ConvertI64S):
case uint16_t(Op::F32ConvertI64U):
CHECK(emitConvertI64ToFloatingPoint(ValType::F32, MIRType::Float32,
Op(op.b0) == Op::F32ConvertI64U));
case uint16_t(Op::F32DemoteF64):
CHECK(emitConversion<MToFloat32>(ValType::F64, ValType::F32));
case uint16_t(Op::F64ConvertI32S):
CHECK(emitConversion<MToDouble>(ValType::I32, ValType::F64));
case uint16_t(Op::F64ConvertI32U):
CHECK(
emitConversion<MWasmUnsignedToDouble>(ValType::I32, ValType::F64));
case uint16_t(Op::F64ConvertI64S):
case uint16_t(Op::F64ConvertI64U):
CHECK(emitConvertI64ToFloatingPoint(ValType::F64, MIRType::
Double,
Op(op.b0) == Op::F64ConvertI64U));
case uint16_t(Op::F64PromoteF32):
CHECK(emitConversion<MToDouble>(ValType::F32, ValType::F64));
// Reinterpretations
case uint16_t(Op::I32ReinterpretF32):
CHECK(emitReinterpret(ValType::I32, ValType::F32, MIRType::Int32));
case uint16_t(Op::I64ReinterpretF64):
CHECK(emitReinterpret(ValType::I64, ValType::F64, MIRType::Int64));
case uint16_t(Op::F32ReinterpretI32):
CHECK(emitReinterpret(ValType::F32, ValType::I32, MIRType::Float32));
case uint16_t(Op::F64ReinterpretI64):
CHECK(emitReinterpret(ValType::F64, ValType::I64, MIRType::
Double));
case uint16_t(Op::RefEq):
CHECK(emitComparison(RefType::eq(), JSOp::Eq,
MCompare::Compare_WasmAnyRef));
case uint16_t(Op::RefFunc):
CHECK(emitRefFunc());
case uint16_t(Op::RefNull):
CHECK(emitRefNull());
case uint16_t(Op::RefIsNull):
CHECK(emitRefIsNull());
// Sign extensions
case uint16_t(Op::I32Extend8S):
CHECK(emitSignExtend(
1,
4));
case uint16_t(Op::I32Extend16S):
CHECK(emitSignExtend(
2,
4));
case uint16_t(Op::I64Extend8S):
CHECK(emitSignExtend(
1,
8));
case uint16_t(Op::I64Extend16S):
CHECK(emitSignExtend(
2,
8));
case uint16_t(Op::I64Extend32S):
CHECK(emitSignExtend(
4,
8));
case uint16_t(Op::ReturnCall): {
CHECK(emitReturnCall());
}
case uint16_t(Op::ReturnCallIndirect): {
CHECK(emitReturnCallIndirect());
}
case uint16_t(Op::RefAsNonNull):
CHECK(emitRefAsNonNull());
case uint16_t(Op::BrOnNull): {
CHECK(emitBrOnNull());
}
case uint16_t(Op::BrOnNonNull): {
CHECK(emitBrOnNonNull());
}
case uint16_t(Op::CallRef): {
CHECK(emitCallRef());
}
case uint16_t(Op::ReturnCallRef): {
CHECK(emitReturnCallRef());
}
#ifdef ENABLE_WASM_JSPI
case uint16_t(Op::ContNew): {
if (!codeMeta().stackSwitchingEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitContNew());
}
case uint16_t(Op::ContBind): {
if (!codeMeta().stackSwitchingEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitContBind());
}
case uint16_t(Op::Suspend): {
if (!codeMeta().stackSwitchingEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitSuspend());
}
case uint16_t(Op::Resume): {
if (!codeMeta().stackSwitchingEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitResume());
}
case uint16_t(Op::ResumeThrow): {
if (!codeMeta().stackSwitchingEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitResumeThrow());
}
case uint16_t(Op::ResumeThrowRef): {
if (!codeMeta().stackSwitchingEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitResumeThrowRef());
}
case uint16_t(Op::
Switch): {
if (!codeMeta().stackSwitchingEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitSwitch());
}
#endif // ENABLE_WASM_JSPI
// Gc operations
case uint16_t(Op::GcPrefix): {
switch (op.b1) {
case uint32_t(GcOp::StructNew):
CHECK(emitStructNew());
case uint32_t(GcOp::StructNewDefault):
CHECK(emitStructNewDefault());
case uint32_t(GcOp::StructSet):
CHECK(emitStructSet());
case uint32_t(GcOp::StructGet):
CHECK(emitStructGet(FieldWideningOp::None));
case uint32_t(GcOp::StructGetS):
CHECK(emitStructGet(FieldWideningOp::
Signed));
case uint32_t(GcOp::StructGetU):
CHECK(emitStructGet(FieldWideningOp::
Unsigned));
case uint32_t(GcOp::ArrayNew):
CHECK(emitArrayNew());
case uint32_t(GcOp::ArrayNewDefault):
CHECK(emitArrayNewDefault());
case uint32_t(GcOp::ArrayNewFixed):
CHECK(emitArrayNewFixed());
case uint32_t(GcOp::ArrayNewData):
CHECK(emitArrayNewData());
case uint32_t(GcOp::ArrayNewElem):
CHECK(emitArrayNewElem());
case uint32_t(GcOp::ArrayInitData):
CHECK(emitArrayInitData());
case uint32_t(GcOp::ArrayInitElem):
CHECK(emitArrayInitElem());
case uint32_t(GcOp::ArraySet):
CHECK(emitArraySet());
case uint32_t(GcOp::ArrayGet):
CHECK(emitArrayGet(FieldWideningOp::None));
case uint32_t(GcOp::ArrayGetS):
CHECK(emitArrayGet(FieldWideningOp::
Signed));
case uint32_t(GcOp::ArrayGetU):
CHECK(emitArrayGet(FieldWideningOp::
Unsigned));
case uint32_t(GcOp::ArrayLen):
CHECK(emitArrayLen());
case uint32_t(GcOp::ArrayCopy):
CHECK(emitArrayCopy());
case uint32_t(GcOp::ArrayFill):
CHECK(emitArrayFill());
case uint32_t(GcOp::RefI31):
CHECK(emitRefI31());
case uint32_t(GcOp::I31GetS):
CHECK(emitI31Get(FieldWideningOp::
Signed));
case uint32_t(GcOp::I31GetU):
CHECK(emitI31Get(FieldWideningOp::
Unsigned));
case uint32_t(GcOp::BrOnCast):
CHECK(emitBrOnCast(
/*onSuccess=*/true));
case uint32_t(GcOp::BrOnCastFail):
CHECK(emitBrOnCast(
/*onSuccess=*/false));
case uint32_t(GcOp::RefTest):
CHECK(emitRefTest(
/*nullable=*/false));
case uint32_t(GcOp::RefTestNull):
CHECK(emitRefTest(
/*nullable=*/true));
case uint32_t(GcOp::RefCast):
CHECK(emitRefCast(
/*nullable=*/false));
case uint32_t(GcOp::RefCastNull):
CHECK(emitRefCast(
/*nullable=*/true));
case uint16_t(GcOp::AnyConvertExtern):
CHECK(emitAnyConvertExtern());
case uint16_t(GcOp::ExternConvertAny):
CHECK(emitExternConvertAny());
default:
return iter().unrecognizedOpcode(&op);
}
// switch (op.b1)
break;
}
// SIMD operations
#ifdef ENABLE_WASM_SIMD
case uint16_t(Op::SimdPrefix): {
if (!codeMeta().simdAvailable()) {
return iter().unrecognizedOpcode(&op);
}
switch (op.b1) {
case uint32_t(SimdOp::V128Const):
CHECK(emitConstSimd128());
case uint32_t(SimdOp::V128Load):
CHECK(emitLoad(ValType::V128, Scalar::Simd128));
case uint32_t(SimdOp::V128Store):
CHECK(emitStore(ValType::V128, Scalar::Simd128));
case uint32_t(SimdOp::V128And):
case uint32_t(SimdOp::V128Or):
case uint32_t(SimdOp::V128Xor):
case uint32_t(SimdOp::I8x16AvgrU):
case uint32_t(SimdOp::I16x8AvgrU):
case uint32_t(SimdOp::I8x16Add):
case uint32_t(SimdOp::I8x16AddSatS):
case uint32_t(SimdOp::I8x16AddSatU):
case uint32_t(SimdOp::I8x16MinS):
case uint32_t(SimdOp::I8x16MinU):
case uint32_t(SimdOp::I8x16MaxS):
case uint32_t(SimdOp::I8x16MaxU):
case uint32_t(SimdOp::I16x8Add):
case uint32_t(SimdOp::I16x8AddSatS):
case uint32_t(SimdOp::I16x8AddSatU):
case uint32_t(SimdOp::I16x8Mul):
case uint32_t(SimdOp::I16x8MinS):
case uint32_t(SimdOp::I16x8MinU):
case uint32_t(SimdOp::I16x8MaxS):
case uint32_t(SimdOp::I16x8MaxU):
case uint32_t(SimdOp::I32x4Add):
case uint32_t(SimdOp::I32x4Mul):
case uint32_t(SimdOp::I32x4MinS):
case uint32_t(SimdOp::I32x4MinU):
case uint32_t(SimdOp::I32x4MaxS):
case uint32_t(SimdOp::I32x4MaxU):
case uint32_t(SimdOp::I64x2Add):
case uint32_t(SimdOp::I64x2Mul):
case uint32_t(SimdOp::F32x4Add):
case uint32_t(SimdOp::F32x4Mul):
case uint32_t(SimdOp::F32x4Min):
case uint32_t(SimdOp::F32x4Max):
case uint32_t(SimdOp::F64x2Add):
case uint32_t(SimdOp::F64x2Mul):
case uint32_t(SimdOp::F64x2Min):
case uint32_t(SimdOp::F64x2Max):
case uint32_t(SimdOp::I8x16Eq):
case uint32_t(SimdOp::I8x16Ne):
case uint32_t(SimdOp::I16x8Eq):
case uint32_t(SimdOp::I16x8Ne):
case uint32_t(SimdOp::I32x4Eq):
case uint32_t(SimdOp::I32x4Ne):
case uint32_t(SimdOp::I64x2Eq):
case uint32_t(SimdOp::I64x2Ne):
case uint32_t(SimdOp::F32x4Eq):
case uint32_t(SimdOp::F32x4Ne):
case uint32_t(SimdOp::F64x2Eq):
case uint32_t(SimdOp::F64x2Ne):
case uint32_t(SimdOp::I32x4DotI16x8S):
case uint32_t(SimdOp::I16x8ExtmulLowI8x16S):
case uint32_t(SimdOp::I16x8ExtmulHighI8x16S):
case uint32_t(SimdOp::I16x8ExtmulLowI8x16U):
case uint32_t(SimdOp::I16x8ExtmulHighI8x16U):
case uint32_t(SimdOp::I32x4ExtmulLowI16x8S):
case uint32_t(SimdOp::I32x4ExtmulHighI16x8S):
case uint32_t(SimdOp::I32x4ExtmulLowI16x8U):
case uint32_t(SimdOp::I32x4ExtmulHighI16x8U):
case uint32_t(SimdOp::I64x2ExtmulLowI32x4S):
case uint32_t(SimdOp::I64x2ExtmulHighI32x4S):
case uint32_t(SimdOp::I64x2ExtmulLowI32x4U):
case uint32_t(SimdOp::I64x2ExtmulHighI32x4U):
case uint32_t(SimdOp::I16x8Q15MulrSatS):
CHECK(emitBinarySimd128(
/* commutative= */ true, SimdOp(op.b1)));
case uint32_t(SimdOp::V128AndNot):
case uint32_t(SimdOp::I8x16Sub):
case uint32_t(SimdOp::I8x16SubSatS):
case uint32_t(SimdOp::I8x16SubSatU):
case uint32_t(SimdOp::I16x8Sub):
case uint32_t(SimdOp::I16x8SubSatS):
case uint32_t(SimdOp::I16x8SubSatU):
case uint32_t(SimdOp::I32x4Sub):
case uint32_t(SimdOp::I64x2Sub):
case uint32_t(SimdOp::F32x4Sub):
case uint32_t(SimdOp::F32x4Div):
case uint32_t(SimdOp::F64x2Sub):
case uint32_t(SimdOp::F64x2Div):
case uint32_t(SimdOp::I8x16NarrowI16x8S):
case uint32_t(SimdOp::I8x16NarrowI16x8U):
case uint32_t(SimdOp::I16x8NarrowI32x4S):
case uint32_t(SimdOp::I16x8NarrowI32x4U):
case uint32_t(SimdOp::I8x16LtS):
case uint32_t(SimdOp::I8x16LtU):
case uint32_t(SimdOp::I8x16GtS):
case uint32_t(SimdOp::I8x16GtU):
case uint32_t(SimdOp::I8x16LeS):
case uint32_t(SimdOp::I8x16LeU):
case uint32_t(SimdOp::I8x16GeS):
case uint32_t(SimdOp::I8x16GeU):
case uint32_t(SimdOp::I16x8LtS):
case uint32_t(SimdOp::I16x8LtU):
case uint32_t(SimdOp::I16x8GtS):
case uint32_t(SimdOp::I16x8GtU):
case uint32_t(SimdOp::I16x8LeS):
case uint32_t(SimdOp::I16x8LeU):
case uint32_t(SimdOp::I16x8GeS):
case uint32_t(SimdOp::I16x8GeU):
case uint32_t(SimdOp::I32x4LtS):
case uint32_t(SimdOp::I32x4LtU):
case uint32_t(SimdOp::I32x4GtS):
case uint32_t(SimdOp::I32x4GtU):
case uint32_t(SimdOp::I32x4LeS):
case uint32_t(SimdOp::I32x4LeU):
case uint32_t(SimdOp::I32x4GeS):
case uint32_t(SimdOp::I32x4GeU):
case uint32_t(SimdOp::I64x2LtS):
case uint32_t(SimdOp::I64x2GtS):
case uint32_t(SimdOp::I64x2LeS):
case uint32_t(SimdOp::I64x2GeS):
case uint32_t(SimdOp::F32x4Lt):
case uint32_t(SimdOp::F32x4Gt):
case uint32_t(SimdOp::F32x4Le):
case uint32_t(SimdOp::F32x4Ge):
case uint32_t(SimdOp::F64x2Lt):
case uint32_t(SimdOp::F64x2Gt):
case uint32_t(SimdOp::F64x2Le):
case uint32_t(SimdOp::F64x2Ge):
case uint32_t(SimdOp::I8x16Swizzle):
case uint32_t(SimdOp::F32x4PMax):
case uint32_t(SimdOp::F32x4PMin):
case uint32_t(SimdOp::F64x2PMax):
case uint32_t(SimdOp::F64x2PMin):
CHECK(emitBinarySimd128(
/* commutative= */ false, SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16Splat):
case uint32_t(SimdOp::I16x8Splat):
case uint32_t(SimdOp::I32x4Splat):
CHECK(emitSplatSimd128(ValType::I32, SimdOp(op.b1)));
case uint32_t(SimdOp::I64x2Splat):
CHECK(emitSplatSimd128(ValType::I64, SimdOp(op.b1)));
case uint32_t(SimdOp::F32x4Splat):
CHECK(emitSplatSimd128(ValType::F32, SimdOp(op.b1)));
case uint32_t(SimdOp::F64x2Splat):
CHECK(emitSplatSimd128(ValType::F64, SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16Neg):
case uint32_t(SimdOp::I16x8Neg):
case uint32_t(SimdOp::I16x8ExtendLowI8x16S):
case uint32_t(SimdOp::I16x8ExtendHighI8x16S):
case uint32_t(SimdOp::I16x8ExtendLowI8x16U):
case uint32_t(SimdOp::I16x8ExtendHighI8x16U):
case uint32_t(SimdOp::I32x4Neg):
case uint32_t(SimdOp::I32x4ExtendLowI16x8S):
case uint32_t(SimdOp::I32x4ExtendHighI16x8S):
case uint32_t(SimdOp::I32x4ExtendLowI16x8U):
case uint32_t(SimdOp::I32x4ExtendHighI16x8U):
case uint32_t(SimdOp::I32x4TruncSatF32x4S):
case uint32_t(SimdOp::I32x4TruncSatF32x4U):
case uint32_t(SimdOp::I64x2Neg):
case uint32_t(SimdOp::I64x2ExtendLowI32x4S):
case uint32_t(SimdOp::I64x2ExtendHighI32x4S):
case uint32_t(SimdOp::I64x2ExtendLowI32x4U):
case uint32_t(SimdOp::I64x2ExtendHighI32x4U):
case uint32_t(SimdOp::F32x4Abs):
case uint32_t(SimdOp::F32x4Neg):
case uint32_t(SimdOp::F32x4Sqrt):
case uint32_t(SimdOp::F32x4ConvertI32x4S):
case uint32_t(SimdOp::F32x4ConvertI32x4U):
case uint32_t(SimdOp::F64x2Abs):
case uint32_t(SimdOp::F64x2Neg):
case uint32_t(SimdOp::F64x2Sqrt):
case uint32_t(SimdOp::V128Not):
case uint32_t(SimdOp::I8x16Popcnt):
case uint32_t(SimdOp::I8x16Abs):
case uint32_t(SimdOp::I16x8Abs):
case uint32_t(SimdOp::I32x4Abs):
case uint32_t(SimdOp::I64x2Abs):
case uint32_t(SimdOp::F32x4Ceil):
case uint32_t(SimdOp::F32x4Floor):
case uint32_t(SimdOp::F32x4Trunc):
case uint32_t(SimdOp::F32x4Nearest):
case uint32_t(SimdOp::F64x2Ceil):
case uint32_t(SimdOp::F64x2Floor):
case uint32_t(SimdOp::F64x2Trunc):
case uint32_t(SimdOp::F64x2Nearest):
case uint32_t(SimdOp::F32x4DemoteF64x2Zero):
case uint32_t(SimdOp::F64x2PromoteLowF32x4):
case uint32_t(SimdOp::F64x2ConvertLowI32x4S):
case uint32_t(SimdOp::F64x2ConvertLowI32x4U):
case uint32_t(SimdOp::I32x4TruncSatF64x2SZero):
case uint32_t(SimdOp::I32x4TruncSatF64x2UZero):
case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16S):
case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16U):
case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8S):
case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8U):
CHECK(emitUnarySimd128(SimdOp(op.b1)));
case uint32_t(SimdOp::V128AnyTrue):
case uint32_t(SimdOp::I8x16AllTrue):
case uint32_t(SimdOp::I16x8AllTrue):
case uint32_t(SimdOp::I32x4AllTrue):
case uint32_t(SimdOp::I64x2AllTrue):
case uint32_t(SimdOp::I8x16Bitmask):
case uint32_t(SimdOp::I16x8Bitmask):
case uint32_t(SimdOp::I32x4Bitmask):
case uint32_t(SimdOp::I64x2Bitmask):
CHECK(emitReduceSimd128(SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16Shl):
case uint32_t(SimdOp::I8x16ShrS):
case uint32_t(SimdOp::I8x16ShrU):
case uint32_t(SimdOp::I16x8Shl):
case uint32_t(SimdOp::I16x8ShrS):
case uint32_t(SimdOp::I16x8ShrU):
case uint32_t(SimdOp::I32x4Shl):
case uint32_t(SimdOp::I32x4ShrS):
case uint32_t(SimdOp::I32x4ShrU):
case uint32_t(SimdOp::I64x2Shl):
case uint32_t(SimdOp::I64x2ShrS):
case uint32_t(SimdOp::I64x2ShrU):
CHECK(emitShiftSimd128(SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16ExtractLaneS):
case uint32_t(SimdOp::I8x16ExtractLaneU):
CHECK(emitExtractLaneSimd128(ValType::I32,
16, SimdOp(op.b1)));
case uint32_t(SimdOp::I16x8ExtractLaneS):
case uint32_t(SimdOp::I16x8ExtractLaneU):
CHECK(emitExtractLaneSimd128(ValType::I32,
8, SimdOp(op.b1)));
case uint32_t(SimdOp::I32x4ExtractLane):
CHECK(emitExtractLaneSimd128(ValType::I32,
4, SimdOp(op.b1)));
case uint32_t(SimdOp::I64x2ExtractLane):
CHECK(emitExtractLaneSimd128(ValType::I64,
2, SimdOp(op.b1)));
case uint32_t(SimdOp::F32x4ExtractLane):
CHECK(emitExtractLaneSimd128(ValType::F32,
4, SimdOp(op.b1)));
case uint32_t(SimdOp::F64x2ExtractLane):
CHECK(emitExtractLaneSimd128(ValType::F64,
2, SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16ReplaceLane):
CHECK(emitReplaceLaneSimd128(ValType::I32,
16, SimdOp(op.b1)));
case uint32_t(SimdOp::I16x8ReplaceLane):
CHECK(emitReplaceLaneSimd128(ValType::I32,
8, SimdOp(op.b1)));
case uint32_t(SimdOp::I32x4ReplaceLane):
CHECK(emitReplaceLaneSimd128(ValType::I32,
4, SimdOp(op.b1)));
case uint32_t(SimdOp::I64x2ReplaceLane):
CHECK(emitReplaceLaneSimd128(ValType::I64,
2, SimdOp(op.b1)));
case uint32_t(SimdOp::F32x4ReplaceLane):
CHECK(emitReplaceLaneSimd128(ValType::F32,
4, SimdOp(op.b1)));
case uint32_t(SimdOp::F64x2ReplaceLane):
CHECK(emitReplaceLaneSimd128(ValType::F64,
2, SimdOp(op.b1)));
case uint32_t(SimdOp::V128Bitselect):
CHECK(emitTernarySimd128(SimdOp(op.b1)));
case uint32_t(SimdOp::I8x16Shuffle):
CHECK(emitShuffleSimd128());
case uint32_t(SimdOp::V128Load8Splat):
CHECK(emitLoadSplatSimd128(Scalar::Uint8, SimdOp::I8x16Splat));
case uint32_t(SimdOp::V128Load16Splat):
CHECK(emitLoadSplatSimd128(Scalar::Uint16, SimdOp::I16x8Splat));
case uint32_t(SimdOp::V128Load32Splat):
CHECK(emitLoadSplatSimd128(Scalar::Float32, SimdOp::I32x4Splat));
case uint32_t(SimdOp::V128Load64Splat):
CHECK(emitLoadSplatSimd128(Scalar::Float64, SimdOp::I64x2Splat));
case uint32_t(SimdOp::V128Load8x8S):
case uint32_t(SimdOp::V128Load8x8U):
case uint32_t(SimdOp::V128Load16x4S):
case uint32_t(SimdOp::V128Load16x4U):
case uint32_t(SimdOp::V128Load32x2S):
case uint32_t(SimdOp::V128Load32x2U):
CHECK(emitLoadExtendSimd128(SimdOp(op.b1)));
case uint32_t(SimdOp::V128Load32Zero):
CHECK(emitLoadZeroSimd128(Scalar::Float32,
4));
case uint32_t(SimdOp::V128Load64Zero):
CHECK(emitLoadZeroSimd128(Scalar::Float64,
8));
case uint32_t(SimdOp::V128Load8Lane):
CHECK(emitLoadLaneSimd128(
1));
case uint32_t(SimdOp::V128Load16Lane):
CHECK(emitLoadLaneSimd128(
2));
case uint32_t(SimdOp::V128Load32Lane):
CHECK(emitLoadLaneSimd128(
4));
case uint32_t(SimdOp::V128Load64Lane):
CHECK(emitLoadLaneSimd128(
8));
case uint32_t(SimdOp::V128Store8Lane):
CHECK(emitStoreLaneSimd128(
1));
case uint32_t(SimdOp::V128Store16Lane):
CHECK(emitStoreLaneSimd128(
2));
case uint32_t(SimdOp::V128Store32Lane):
CHECK(emitStoreLaneSimd128(
4));
case uint32_t(SimdOp::V128Store64Lane):
CHECK(emitStoreLaneSimd128(
8));
# ifdef ENABLE_WASM_RELAXED_SIMD
case uint32_t(SimdOp::F32x4RelaxedMadd):
case uint32_t(SimdOp::F32x4RelaxedNmadd):
case uint32_t(SimdOp::F64x2RelaxedMadd):
case uint32_t(SimdOp::F64x2RelaxedNmadd):
case uint32_t(SimdOp::I8x16RelaxedLaneSelect):
case uint32_t(SimdOp::I16x8RelaxedLaneSelect):
case uint32_t(SimdOp::I32x4RelaxedLaneSelect):
case uint32_t(SimdOp::I64x2RelaxedLaneSelect):
case uint32_t(SimdOp::I32x4RelaxedDotI8x16I7x16AddS): {
if (!codeMeta().v128RelaxedEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitTernarySimd128(SimdOp(op.b1)));
}
case uint32_t(SimdOp::F32x4RelaxedMin):
case uint32_t(SimdOp::F32x4RelaxedMax):
case uint32_t(SimdOp::F64x2RelaxedMin):
case uint32_t(SimdOp::F64x2RelaxedMax): {
if (!codeMeta().v128RelaxedEnabled()) {
return iter().unrecognizedOpcode(&op);
}
// These aren't really commutative, because at least on Intel, the
// behaviour in the presence of NaNs depends on the order of the
// operands. And we need to have that ordering fixed, so that we
// can produce the same results as baseline. See bug 1946618.
CHECK(emitBinarySimd128(
/* commutative= */ false, SimdOp(op.b1)));
}
case uint32_t(SimdOp::I16x8RelaxedQ15MulrS): {
if (!codeMeta().v128RelaxedEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitBinarySimd128(
/* commutative= */ true, SimdOp(op.b1)));
}
case uint32_t(SimdOp::I32x4RelaxedTruncF32x4S):
case uint32_t(SimdOp::I32x4RelaxedTruncF32x4U):
case uint32_t(SimdOp::I32x4RelaxedTruncF64x2SZero):
case uint32_t(SimdOp::I32x4RelaxedTruncF64x2UZero): {
if (!codeMeta().v128RelaxedEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitUnarySimd128(SimdOp(op.b1)));
}
case uint32_t(SimdOp::I8x16RelaxedSwizzle):
case uint32_t(SimdOp::I16x8RelaxedDotI8x16I7x16S): {
if (!codeMeta().v128RelaxedEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitBinarySimd128(
/* commutative= */ false, SimdOp(op.b1)));
}
# endif
default:
return iter().unrecognizedOpcode(&op);
}
// switch (op.b1)
break;
}
#endif
// Miscellaneous operations
case uint16_t(Op::MiscPrefix): {
switch (op.b1) {
case uint32_t(MiscOp::I32TruncSatF32S):
case uint32_t(MiscOp::I32TruncSatF32U):
CHECK(emitTruncate(ValType::F32, ValType::I32,
MiscOp(op.b1) == MiscOp::I32TruncSatF32U,
true));
case uint32_t(MiscOp::I32TruncSatF64S):
case uint32_t(MiscOp::I32TruncSatF64U):
CHECK(emitTruncate(ValType::F64, ValType::I32,
MiscOp(op.b1) == MiscOp::I32TruncSatF64U,
true));
case uint32_t(MiscOp::I64TruncSatF32S):
case uint32_t(MiscOp::I64TruncSatF32U):
CHECK(emitTruncate(ValType::F32, ValType::I64,
MiscOp(op.b1) == MiscOp::I64TruncSatF32U,
true));
case uint32_t(MiscOp::I64TruncSatF64S):
case uint32_t(MiscOp::I64TruncSatF64U):
CHECK(emitTruncate(ValType::F64, ValType::I64,
MiscOp(op.b1) == MiscOp::I64TruncSatF64U,
true));
case uint32_t(MiscOp::MemoryCopy):
CHECK(emitMemCopy());
case uint32_t(MiscOp::DataDrop):
CHECK(emitDataOrElemDrop(
/*isData=*/true));
case uint32_t(MiscOp::MemoryFill):
CHECK(emitMemFill());
case uint32_t(MiscOp::MemoryInit):
CHECK(emitMemInit());
case uint32_t(MiscOp::TableCopy):
CHECK(emitTableCopy());
case uint32_t(MiscOp::ElemDrop):
CHECK(emitDataOrElemDrop(
/*isData=*/false));
case uint32_t(MiscOp::TableInit):
CHECK(emitTableInit());
case uint32_t(MiscOp::TableFill):
CHECK(emitTableFill());
#if ENABLE_WASM_MEMORY_CONTROL
case uint32_t(MiscOp::MemoryDiscard): {
if (!codeMeta().memoryControlEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitMemDiscard());
}
#endif
case uint32_t(MiscOp::TableGrow):
CHECK(emitTableGrow());
case uint32_t(MiscOp::TableSize):
CHECK(emitTableSize());
case uint32_t(MiscOp::I64Add128):
if (!codeMeta().wideArithmeticEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitI64AddSub128(
/*isAdd=*/true));
case uint32_t(MiscOp::I64Sub128):
if (!codeMeta().wideArithmeticEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitI64AddSub128(
/*isAdd=*/false));
case uint32_t(MiscOp::I64MulWideS):
if (!codeMeta().wideArithmeticEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitI64MulWide(
/*isSigned=*/true));
case uint32_t(MiscOp::I64MulWideU):
if (!codeMeta().wideArithmeticEnabled()) {
return iter().unrecognizedOpcode(&op);
}
CHECK(emitI64MulWide(
/*isSigned=*/false));
default:
return iter().unrecognizedOpcode(&op);
}
break;
}
// Thread operations
case uint16_t(Op::ThreadPrefix): {
// Though thread ops can be used on nonshared memories, we make them
// unavailable if shared memory has been disabled in the prefs, for
// maximum predictability and safety and consistency with JS.
if (codeMeta().sharedMemoryEnabled() == Shareable::
False) {
return iter().unrecognizedOpcode(&op);
}
switch (op.b1) {
case uint32_t(ThreadOp::Notify):
CHECK(emitNotify());
case uint32_t(ThreadOp::I32Wait):
CHECK(emitWait(ValType::I32,
4));
case uint32_t(ThreadOp::I64Wait):
CHECK(emitWait(ValType::I64,
8));
case uint32_t(ThreadOp::Fence):
CHECK(emitFence());
case uint32_t(ThreadOp::I32AtomicLoad):
CHECK(emitAtomicLoad(ValType::I32, Scalar::Int32));
case uint32_t(ThreadOp::I64AtomicLoad):
CHECK(emitAtomicLoad(ValType::I64, Scalar::Int64));
case uint32_t(ThreadOp::I32AtomicLoad8U):
CHECK(emitAtomicLoad(ValType::I32, Scalar::Uint8));
case uint32_t(ThreadOp::I32AtomicLoad16U):
CHECK(emitAtomicLoad(ValType::I32, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicLoad8U):
CHECK(emitAtomicLoad(ValType::I64, Scalar::Uint8));
case uint32_t(ThreadOp::I64AtomicLoad16U):
CHECK(emitAtomicLoad(ValType::I64, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicLoad32U):
CHECK(emitAtomicLoad(ValType::I64, Scalar::Uint32));
case uint32_t(ThreadOp::I32AtomicStore):
CHECK(emitAtomicStore(ValType::I32, Scalar::Int32));
case uint32_t(ThreadOp::I64AtomicStore):
CHECK(emitAtomicStore(ValType::I64, Scalar::Int64));
case uint32_t(ThreadOp::I32AtomicStore8U):
CHECK(emitAtomicStore(ValType::I32, Scalar::Uint8));
case uint32_t(ThreadOp::I32AtomicStore16U):
CHECK(emitAtomicStore(ValType::I32, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicStore8U):
CHECK(emitAtomicStore(ValType::I64, Scalar::Uint8));
case uint32_t(ThreadOp::I64AtomicStore16U):
CHECK(emitAtomicStore(ValType::I64, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicStore32U):
CHECK(emitAtomicStore(ValType::I64, Scalar::Uint32));
case uint32_t(ThreadOp::I32AtomicAdd):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicOp::Add));
case uint32_t(ThreadOp::I64AtomicAdd):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicOp::Add));
case uint32_t(ThreadOp::I32AtomicAdd8U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicOp::Add));
case uint32_t(ThreadOp::I32AtomicAdd16U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicOp::Add));
case uint32_t(ThreadOp::I64AtomicAdd8U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicOp::Add));
case uint32_t(ThreadOp::I64AtomicAdd16U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicOp::Add));
case uint32_t(ThreadOp::I64AtomicAdd32U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicOp::Add));
case uint32_t(ThreadOp::I32AtomicSub):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicOp::Sub));
case uint32_t(ThreadOp::I64AtomicSub):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicOp::Sub));
case uint32_t(ThreadOp::I32AtomicSub8U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicOp::Sub));
case uint32_t(ThreadOp::I32AtomicSub16U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicOp::Sub));
case uint32_t(ThreadOp::I64AtomicSub8U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicOp::Sub));
case uint32_t(ThreadOp::I64AtomicSub16U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicOp::Sub));
case uint32_t(ThreadOp::I64AtomicSub32U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicOp::Sub));
case uint32_t(ThreadOp::I32AtomicAnd):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicOp::
And));
case uint32_t(ThreadOp::I64AtomicAnd):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicOp::
And));
case uint32_t(ThreadOp::I32AtomicAnd8U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicOp::
And));
case uint32_t(ThreadOp::I32AtomicAnd16U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicOp::
And));
case uint32_t(ThreadOp::I64AtomicAnd8U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicOp::
And));
case uint32_t(ThreadOp::I64AtomicAnd16U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicOp::
And));
case uint32_t(ThreadOp::I64AtomicAnd32U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicOp::
And));
case uint32_t(ThreadOp::I32AtomicOr):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicOp::
Or));
case uint32_t(ThreadOp::I64AtomicOr):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicOp::
Or));
case uint32_t(ThreadOp::I32AtomicOr8U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicOp::
Or));
case uint32_t(ThreadOp::I32AtomicOr16U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicOp::
Or));
case uint32_t(ThreadOp::I64AtomicOr8U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicOp::
Or));
case uint32_t(ThreadOp::I64AtomicOr16U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicOp::
Or));
case uint32_t(ThreadOp::I64AtomicOr32U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicOp::
Or));
case uint32_t(ThreadOp::I32AtomicXor):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicOp::
Xor));
case uint32_t(ThreadOp::I64AtomicXor):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicOp::
Xor));
case uint32_t(ThreadOp::I32AtomicXor8U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicOp::
Xor));
case uint32_t(ThreadOp::I32AtomicXor16U):
CHECK(emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicOp::
Xor));
case uint32_t(ThreadOp::I64AtomicXor8U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicOp::
Xor));
case uint32_t(ThreadOp::I64AtomicXor16U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicOp::
Xor));
case uint32_t(ThreadOp::I64AtomicXor32U):
CHECK(emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicOp::
Xor));
case uint32_t(ThreadOp::I32AtomicXchg):
CHECK(emitAtomicXchg(ValType::I32, Scalar::Int32));
case uint32_t(ThreadOp::I64AtomicXchg):
CHECK(emitAtomicXchg(ValType::I64, Scalar::Int64));
case uint32_t(ThreadOp::I32AtomicXchg8U):
CHECK(emitAtomicXchg(ValType::I32, Scalar::Uint8));
case uint32_t(ThreadOp::I32AtomicXchg16U):
CHECK(emitAtomicXchg(ValType::I32, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicXchg8U):
CHECK(emitAtomicXchg(ValType::I64, Scalar::Uint8));
case uint32_t(ThreadOp::I64AtomicXchg16U):
CHECK(emitAtomicXchg(ValType::I64, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicXchg32U):
CHECK(emitAtomicXchg(ValType::I64, Scalar::Uint32));
case uint32_t(ThreadOp::I32AtomicCmpXchg):
CHECK(emitAtomicCmpXchg(ValType::I32, Scalar::Int32));
case uint32_t(ThreadOp::I64AtomicCmpXchg):
CHECK(emitAtomicCmpXchg(ValType::I64, Scalar::Int64));
case uint32_t(ThreadOp::I32AtomicCmpXchg8U):
CHECK(emitAtomicCmpXchg(ValType::I32, Scalar::Uint8));
case uint32_t(ThreadOp::I32AtomicCmpXchg16U):
CHECK(emitAtomicCmpXchg(ValType::I32, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicCmpXchg8U):
CHECK(emitAtomicCmpXchg(ValType::I64, Scalar::Uint8));
case uint32_t(ThreadOp::I64AtomicCmpXchg16U):
CHECK(emitAtomicCmpXchg(ValType::I64, Scalar::Uint16));
case uint32_t(ThreadOp::I64AtomicCmpXchg32U):
CHECK(emitAtomicCmpXchg(ValType::I64, Scalar::Uint32));
default:
return iter().unrecognizedOpcode(&op);
}
break;
}
// asm.js-specific operators
case uint16_t(Op::MozPrefix): {
// Perform a single feature check based on whether the opcode is asm.js
// or for builtin modules.
if (op.b1 <= uint32_t(MozOp::LastAsmJSOp)) {
if (!codeMeta().isAsmJS()) {
return iter().unrecognizedOpcode(&op);
}
}
else {
if (!codeMeta().isBuiltinModule()) {
return iter().unrecognizedOpcode(&op);
}
}
switch (op.b1) {
case uint32_t(MozOp::TeeGlobal):
CHECK(emitTeeGlobal());
case uint32_t(MozOp::I32Min):
case uint32_t(MozOp::I32Max):
CHECK(emitMinMax(ValType::I32, MIRType::Int32,
MozOp(op.b1) == MozOp::I32Max));
case uint32_t(MozOp::I32Neg):
CHECK(emitUnaryWithType<MWasmNeg>(ValType::I32, MIRType::Int32));
case uint32_t(MozOp::I32BitNot):
CHECK(emitBitNot(ValType::I32, MIRType::Int32));
case uint32_t(MozOp::I32Abs):
CHECK(emitUnaryWithType<MAbs>(ValType::I32, MIRType::Int32));
case uint32_t(MozOp::F32TeeStoreF64):
CHECK(emitTeeStoreWithCoercion(ValType::F32, Scalar::Float64));
case uint32_t(MozOp::F64TeeStoreF32):
CHECK(emitTeeStoreWithCoercion(ValType::F64, Scalar::Float32));
case uint32_t(MozOp::I32TeeStore8):
CHECK(emitTeeStore(ValType::I32, Scalar::Int8));
case uint32_t(MozOp::I32TeeStore16):
CHECK(emitTeeStore(ValType::I32, Scalar::Int16));
case uint32_t(MozOp::I64TeeStore8):
CHECK(emitTeeStore(ValType::I64, Scalar::Int8));
case uint32_t(MozOp::I64TeeStore16):
CHECK(emitTeeStore(ValType::I64, Scalar::Int16));
case uint32_t(MozOp::I64TeeStore32):
CHECK(emitTeeStore(ValType::I64, Scalar::Int32));
case uint32_t(MozOp::I32TeeStore):
CHECK(emitTeeStore(ValType::I32, Scalar::Int32));
case uint32_t(MozOp::I64TeeStore):
CHECK(emitTeeStore(ValType::I64, Scalar::Int64));
case uint32_t(MozOp::F32TeeStore):
CHECK(emitTeeStore(ValType::F32, Scalar::Float32));
case uint32_t(MozOp::F64TeeStore):
CHECK(emitTeeStore(ValType::F64, Scalar::Float64));
case uint32_t(MozOp::F64Mod):
CHECK(emitRem(ValType::F64, MIRType::
Double,
/* isUnsigned = */ false));
case uint32_t(MozOp::F64SinNative):
CHECK(emitUnaryMathBuiltinCall(SASigSinNativeD));
case uint32_t(MozOp::F64SinFdlibm):
CHECK(emitUnaryMathBuiltinCall(SASigSinFdlibmD));
case uint32_t(MozOp::F64CosNative):
CHECK(emitUnaryMathBuiltinCall(SASigCosNativeD));
case uint32_t(MozOp::F64CosFdlibm):
CHECK(emitUnaryMathBuiltinCall(SASigCosFdlibmD));
case uint32_t(MozOp::F64TanNative):
CHECK(emitUnaryMathBuiltinCall(SASigTanNativeD));
case uint32_t(MozOp::F64TanFdlibm):
CHECK(emitUnaryMathBuiltinCall(SASigTanFdlibmD));
case uint32_t(MozOp::F64Asin):
CHECK(emitUnaryMathBuiltinCall(SASigASinD));
case uint32_t(MozOp::F64Acos):
CHECK(emitUnaryMathBuiltinCall(SASigACosD));
case uint32_t(MozOp::F64Atan):
CHECK(emitUnaryMathBuiltinCall(SASigATanD));
case uint32_t(MozOp::F64Exp):
CHECK(emitUnaryMathBuiltinCall(SASigExpD));
case uint32_t(MozOp::F64Log):
CHECK(emitUnaryMathBuiltinCall(SASigLogD));
case uint32_t(MozOp::F64Pow):
CHECK(emitBinaryMathBuiltinCall(SASigPowD));
case uint32_t(MozOp::F64Atan2):
CHECK(emitBinaryMathBuiltinCall(SASigATan2D));
case uint32_t(MozOp::OldCallDirect):
CHECK(emitCall(
/* asmJSFuncDef = */ true));
case uint32_t(MozOp::OldCallIndirect):
CHECK(emitCallIndirect(
/* oldStyle = */ true));
case uint32_t(MozOp::CallBuiltinModuleFunc):
CHECK(emitCallBuiltinModuleFunc());
#ifdef ENABLE_WASM_JSPI
case uint32_t(MozOp::GuardSuspending):
CHECK(emitGuardSuspending());
#endif // ENABLE_WASM_JSPI
default:
return iter().unrecognizedOpcode(&op);
}
break;
}
default:
return iter().unrecognizedOpcode(&op);
}
}
MOZ_CRASH(
"unreachable");
#undef CHECK
}
}
// end anonymous namespace
bool RootCompiler::generate() {
// Only activate branch hinting if the option is enabled and some hints were
// parsed.
if (codeMeta_.branchHintingEnabled() && !codeMeta_.branchHints.isEmpty()) {
compileInfo_.setBranchHinting(
true);
}
// Figure out what the inlining budget for this function is. If we've
// already exceeded the module-level limit, the budget is zero. See
// "[SMDOC] Per-function and per-module inlining limits" (WasmHeuristics.h)
if (codeTailMeta_) {
auto guard = codeTailMeta_->inliningBudget.lock();
if (guard.get() >
0) {
localInliningBudget_ =
int64_t(codeMeta_.codeSectionSize()) * PerFunctionMaxInliningRatio;
localInliningBudget_ =
std::min<int64_t>(localInliningBudget_, guard.get());
}
else {
localInliningBudget_ =
0;
}
MOZ_ASSERT(localInliningBudget_ >=
0);
}
else {
localInliningBudget_ =
0;
}
// Build the MIR graph
FunctionCompiler funcCompiler(*
this, decoder_, func_, locals_, compileInfo_);
if (!funcCompiler.initRoot() || !funcCompiler.startBlock() ||
!funcCompiler.emitBodyExprs()) {
return false;
}
funcCompiler.finish();
observedFeatures_ = funcCompiler.featureUsage();
MOZ_ASSERT(loopDepth_ ==
0);
funcStats_.numFuncs +=
1;
funcStats_.bytecodeSize += func_.bytecodeSize();
funcStats_.inlinedDirectCallCount += inliningStats_.inlinedDirectFunctions;
funcStats_.inlinedCallRefCount += inliningStats_.inlinedCallRefFunctions;
funcStats_.inlinedDirectCallBytecodeSize +=
inliningStats_.inlinedDirectBytecodeSize;
funcStats_.inlinedCallRefBytecodeSize +=
inliningStats_.inlinedCallRefBytecodeSize;
funcStats_.numLargeFunctionBackoffs +=
inliningStats_.largeFunctionBackoff ?
1 :
0;
if (codeTailMeta_) {
auto guard = codeTailMeta_->inliningBudget.lock();
// Update the module's inlining budget accordingly. If it is already
// negative, no more inlining for the module can happen, so there's no
// point in updating it further.
if (guard.get() >=
0) {
guard.get() -= int64_t(inliningStats_.inlinedDirectBytecodeSize);
guard.get() -= int64_t(inliningStats_.inlinedCallRefBytecodeSize);
if (guard.get() <
0) {
JS_LOG(wasmPerf, Info,
"CM=..%06lx RC::generate "
"Inlining budget for entire module exceeded",
0xFFFFFF & (
unsigned long)uintptr_t(&codeMeta_));
}
}
// If this particular root function overran the function-level
// limit, note that in the module too.
if (localInliningBudget_ <
0) {
funcStats_.numInliningBudgetOverruns +=
1;
}
}
return true;
}
CompileInfo* RootCompiler::startInlineCall(
uint32_t callerFuncIndex, BytecodeOffset callerOffset,
uint32_t calleeFuncIndex, uint32_t numLocals, size_t inlineeBytecodeSize,
InliningHeuristics::CallKind callKind) {
if (callKind == InliningHeuristics::CallKind::Direct) {
inliningStats_.inlinedDirectBytecodeSize += inlineeBytecodeSize;
inliningStats_.inlinedDirectFunctions +=
1;
}
else {
MOZ_ASSERT(callKind == InliningHeuristics::CallKind::CallRef);
inliningStats_.inlinedCallRefBytecodeSize += inlineeBytecodeSize;
inliningStats_.inlinedCallRefFunctions +=
1;
}
// Update the inlining budget accordingly. If it is already negative, no
// more inlining within this root function can happen, so there's no
// point in updating it further.
if (localInliningBudget_ >=
0) {
localInliningBudget_ -= int64_t(inlineeBytecodeSize);
#ifdef JS_JITSPEW
if (localInliningBudget_ <=
0) {
JS_LOG(wasmPerf, Info,
"CM=..%06lx RC::startInlineCall "
"Inlining budget for fI=%u exceeded",
0xFFFFFF & (
unsigned long)uintptr_t(&codeMeta_), callerFuncIndex);
}
#endif
}
// Add the callers offset to the stack of inlined caller offsets
if (!inlinedCallerOffsets_.append(callerOffset)) {
return nullptr;
}
// Cache a copy of the current stack of inlined caller offsets that can be
// shared across all call sites
InlinedCallerOffsets inlinedCallerOffsets;
if (!inlinedCallerOffsets.appendAll(inlinedCallerOffsets_)) {
return nullptr;
}
if (!inliningContext_.append(std::move(inlinedCallerOffsets),
&inlinedCallerOffsetsIndex_)) {
return nullptr;
}
UniqueCompileInfo compileInfo = MakeUnique<CompileInfo>(numLocals);
if (!compileInfo || !compileInfos_.append(std::move(compileInfo))) {
return nullptr;
}
return compileInfos_[compileInfos_.length() -
1].get();
}
void RootCompiler::finishInlineCall() { inlinedCallerOffsets_.popBack(); }
bool wasm::IonCompileFunctions(
const CodeMetadata& codeMeta,
const CodeTailMetadata* codeTailMeta,
const CompilerEnvironment& compilerEnv,
LifoAlloc& lifo,
const FuncCompileInputVector& inputs,
CompiledCode* code, UniqueChars* error) {
MOZ_ASSERT(compilerEnv.tier() == Tier::Optimized);
MOZ_ASSERT(compilerEnv.debug() == DebugEnabled::
False);
MOZ_ASSERT_IF(compilerEnv.mode() == CompileMode::LazyTiering, !!codeTailMeta);
// We should not interact with the GC heap, nor allocate from it when we are
// compiling wasm code. Ion data structures have some fields for GC objects
// that we do not use, yet can confuse the static analysis here. Disable it
// for this function.
JS::AutoSuppressGCAnalysis nogc;
TempAllocator alloc(&lifo);
JitContext jitContext;
MOZ_ASSERT(IsCompilingWasm());
WasmMacroAssembler masm(alloc);
#if defined(JS_CODEGEN_ARM64)
masm.SetStackPointer64(PseudoStackPointer64);
#endif
// Swap in already-allocated empty vectors to avoid malloc/free.
MOZ_ASSERT(code->empty());
if (!code->swap(masm)) {
return false;
}
// Create a description of the stack layout created by GenerateTrapExit().
RegisterOffsets trapExitLayout;
size_t trapExitLayoutNumWords;
GenerateTrapExitRegisterOffsets(&trapExitLayout, &trapExitLayoutNumWords);
for (
const FuncCompileInput& func : inputs) {
JitSpew(JitSpew_Codegen,
"\n");
JitSpew(JitSpew_Codegen,
"# ================================"
"==================================");
JitSpew(JitSpew_Codegen,
"# ==");
JitSpew(JitSpew_Codegen,
"# wasm::IonCompileFunctions: starting on function index %d",
(
int)func.index);
Decoder d(func.begin, func.end, func.lineOrBytecode, error);
// Build the local types vector.
ValTypeVector locals;
if (!DecodeLocalEntriesWithParams(d, codeMeta, func.index, &locals)) {
return false;
}
// Set up for Ion compilation.
RootCompiler rootCompiler(compilerEnv, codeMeta, codeTailMeta, alloc,
locals, func, d, masm.tryNotes(),
masm.inliningContext());
if (!rootCompiler.generate()) {
return false;
}
// Record observed feature usage
FeatureUsage observedFeatures = rootCompiler.observedFeatures();
code->featureUsage |= observedFeatures;
// Compile MIR graph
{
rootCompiler.mirGen().spewBeginWasmFunction(func.index);
jit::AutoSpewEndFunction spewEndFunction(&rootCompiler.mirGen());
if (!OptimizeMIR(&rootCompiler.mirGen())) {
return false;
}
LIRGraph* lir = GenerateLIR(&rootCompiler.mirGen());
if (!lir) {
return false;
}
size_t unwindInfoBefore = masm.codeRangeUnwindInfos().length();
CodeGenerator codegen(&rootCompiler.mirGen(), lir, &masm, &codeMeta);
TrapSiteDesc prologueTrapSiteDesc(
wasm::BytecodeOffset(func.lineOrBytecode));
FuncOffsets offsets;
ArgTypeVector args(codeMeta.getFuncType(func.index));
if (!codegen.generateWasm(CallIndirectId::forFunc(codeMeta, func.index),
prologueTrapSiteDesc, args, trapExitLayout,
trapExitLayoutNumWords, &offsets,
&code->stackMaps, &d)) {
return false;
}
bool hasUnwindInfo =
unwindInfoBefore != masm.codeRangeUnwindInfos().length();
// Record this function's code range
if (!code->codeRanges.emplaceBack(func.index, offsets, hasUnwindInfo)) {
return false;
}
}
if (PerfEnabled()) {
IonPerfSpewer spewer = std::move(rootCompiler.mirGen().perfSpewer());
if (!code->funcIonSpewers.emplaceBack(func.index, std::move(spewer))) {
return false;
}
}
// Record this function's compilation stats
code->compileStats.merge(rootCompiler.funcStats());
// Record this function's specific feature usage
if (!code->funcs.emplaceBack(func.index, observedFeatures)) {
return false;
}
JitSpew(JitSpew_Codegen,
"# wasm::IonCompileFunctions: completed function index %d",
(
int)func.index);
JitSpew(JitSpew_Codegen,
"# ==");
JitSpew(JitSpew_Codegen,
"# ================================"
"==================================");
JitSpew(JitSpew_Codegen,
"\n");
}
masm.finish();
if (masm.oom()) {
return false;
}
return code->swap(masm);
}
bool wasm::IonDumpFunction(
const CompilerEnvironment& compilerEnv,
const CodeMetadata& codeMeta,
const FuncCompileInput& func, GenericPrinter& out,
UniqueChars* error) {
#ifdef JS_JITSPEW
LifoAlloc lifo(TempAllocator::PreferredLifoChunkSize,
js::BackgroundMallocArena);
TempAllocator alloc(&lifo);
JitContext jitContext;
Decoder d(func.begin, func.end, func.lineOrBytecode, error);
// Decode the locals.
ValTypeVector locals;
if (!DecodeLocalEntriesWithParams(d, codeMeta, func.index, &locals)) {
return false;
}
TryNoteVector tryNotes;
InliningContext inliningContext;
RootCompiler rootCompiler(compilerEnv, codeMeta, nullptr, alloc, locals, func,
d, tryNotes, inliningContext);
MIRGenerator& mirGen = rootCompiler.mirGen();
GraphSpewer graphSpewer(out, &codeMeta);
graphSpewer.begin();
mirGen.setGraphSpewer(&graphSpewer);
mirGen.spewBeginWasmFunction(func.index);
if (!rootCompiler.generate() || !OptimizeMIR(&mirGen) ||
!GenerateLIR(&mirGen)) {
return false;
}
mirGen.spewEndFunction();
graphSpewer.end();
return true;
#else
UniqueChars errStr =
DuplicateString(
"cannot dump Ion without --enable-jitspew");
if (!errStr) {
return false;
}
*error = std::move(errStr);
return false;
#endif
}
bool js::wasm::IonPlatformSupport() {
#if defined(JS_CODEGEN_X64) ||
defined(JS_CODEGEN_X86) || \
defined(JS_CODEGEN_ARM) ||
defined(JS_CODEGEN_MIPS64) || \
defined(JS_CODEGEN_ARM64) ||
defined(JS_CODEGEN_LOONG64) || \
defined(JS_CODEGEN_RISCV64)
return true;
#else
return false;
#endif
}
