// Copyright 2015 the V8 project authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "src/compiler/wasm-compiler.h" #include <memory> #include "src/assembler-inl.h" #include "src/base/optional.h" #include "src/base/platform/elapsed-timer.h" #include "src/base/platform/platform.h" #include "src/builtins/builtins.h" #include "src/code-factory.h" #include "src/compiler/access-builder.h" #include "src/compiler/code-generator.h" #include "src/compiler/common-operator.h" #include "src/compiler/compiler-source-position-table.h" #include "src/compiler/diamond.h" #include "src/compiler/graph-visualizer.h" #include "src/compiler/graph.h" #include "src/compiler/instruction-selector.h" #include "src/compiler/int64-lowering.h" #include "src/compiler/js-graph.h" #include "src/compiler/js-operator.h" #include "src/compiler/linkage.h" #include "src/compiler/machine-operator.h" #include "src/compiler/node-matchers.h" #include "src/compiler/pipeline.h" #include "src/compiler/simd-scalar-lowering.h" #include "src/compiler/zone-stats.h" #include "src/factory.h" #include "src/isolate-inl.h" #include "src/log-inl.h" #include "src/trap-handler/trap-handler.h" #include "src/wasm/function-body-decoder.h" #include "src/wasm/memory-tracing.h" #include "src/wasm/wasm-code-manager.h" #include "src/wasm/wasm-limits.h" #include "src/wasm/wasm-module.h" #include "src/wasm/wasm-objects-inl.h" #include "src/wasm/wasm-opcodes.h" #include "src/wasm/wasm-text.h" namespace v8 { namespace internal { namespace compiler { // TODO(titzer): pull WASM_64 up to a common header. #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64 #define WASM_64 1 #else #define WASM_64 0 #endif #define FATAL_UNSUPPORTED_OPCODE(opcode) \ V8_Fatal(__FILE__, __LINE__, "Unsupported opcode #%d:%s", (opcode), \ wasm::WasmOpcodes::OpcodeName(opcode)); namespace { constexpr uint32_t kBytesPerExceptionValuesArrayElement = 2; void MergeControlToEnd(JSGraph* jsgraph, Node* node) { Graph* g = jsgraph->graph(); if (g->end()) { NodeProperties::MergeControlToEnd(g, jsgraph->common(), node); } else { g->SetEnd(g->NewNode(jsgraph->common()->End(1), node)); } } bool ContainsSimd(wasm::FunctionSig* sig) { for (wasm::ValueType t : sig->all()) { if (t == wasm::kWasmS128) return true; } return false; } } // namespace WasmGraphBuilder::WasmGraphBuilder( ModuleEnv* env, Zone* zone, JSGraph* jsgraph, Handle<Code> centry_stub, wasm::FunctionSig* sig, compiler::SourcePositionTable* source_position_table, RuntimeExceptionSupport exception_support) : zone_(zone), jsgraph_(jsgraph), centry_stub_node_(jsgraph_->HeapConstant(centry_stub)), env_(env), function_tables_(zone), cur_buffer_(def_buffer_), cur_bufsize_(kDefaultBufferSize), has_simd_(ContainsSimd(sig)), untrusted_code_mitigations_(FLAG_untrusted_code_mitigations), runtime_exception_support_(exception_support), sig_(sig), source_position_table_(source_position_table) { DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled()); DCHECK_NOT_NULL(jsgraph_); } Node* WasmGraphBuilder::Error() { return jsgraph()->Dead(); } Node* WasmGraphBuilder::Start(unsigned params) { Node* start = graph()->NewNode(jsgraph()->common()->Start(params)); graph()->SetStart(start); return start; } Node* WasmGraphBuilder::Param(unsigned index) { return graph()->NewNode(jsgraph()->common()->Parameter(index), graph()->start()); } Node* WasmGraphBuilder::Loop(Node* entry) { return graph()->NewNode(jsgraph()->common()->Loop(1), entry); } Node* WasmGraphBuilder::Terminate(Node* effect, Node* control) { Node* terminate = graph()->NewNode(jsgraph()->common()->Terminate(), effect, control); MergeControlToEnd(jsgraph(), terminate); return terminate; } bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) { return phi && IrOpcode::IsPhiOpcode(phi->opcode()) && NodeProperties::GetControlInput(phi) == merge; } bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success, Node** if_exception) { if (node->op()->HasProperty(compiler::Operator::kNoThrow)) { return false; } *if_success = graph()->NewNode(jsgraph()->common()->IfSuccess(), node); *if_exception = graph()->NewNode(jsgraph()->common()->IfException(), node, node); return true; } void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) { DCHECK(IrOpcode::IsMergeOpcode(merge->opcode())); merge->AppendInput(jsgraph()->zone(), from); int new_size = merge->InputCount(); NodeProperties::ChangeOp( merge, jsgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size)); } void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) { DCHECK(IrOpcode::IsPhiOpcode(phi->opcode())); int new_size = phi->InputCount(); phi->InsertInput(jsgraph()->zone(), phi->InputCount() - 1, from); NodeProperties::ChangeOp( phi, jsgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size)); } Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) { return graph()->NewNode(jsgraph()->common()->Merge(count), count, controls); } Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals, Node* control) { DCHECK(IrOpcode::IsMergeOpcode(control->opcode())); Node** buf = Realloc(vals, count, count + 1); buf[count] = control; return graph()->NewNode(jsgraph()->common()->Phi(type, count), count + 1, buf); } Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects, Node* control) { DCHECK(IrOpcode::IsMergeOpcode(control->opcode())); Node** buf = Realloc(effects, count, count + 1); buf[count] = control; return graph()->NewNode(jsgraph()->common()->EffectPhi(count), count + 1, buf); } Node* WasmGraphBuilder::NumberConstant(int32_t value) { return jsgraph()->Constant(value); } Node* WasmGraphBuilder::Uint32Constant(uint32_t value) { return jsgraph()->Uint32Constant(value); } Node* WasmGraphBuilder::Int32Constant(int32_t value) { return jsgraph()->Int32Constant(value); } Node* WasmGraphBuilder::Int64Constant(int64_t value) { return jsgraph()->Int64Constant(value); } Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) { return jsgraph()->IntPtrConstant(value); } void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position, Node** effect, Node** control) { // TODO(mtrofin): "!env_" happens when we generate a wrapper. // We should factor wrappers separately from wasm codegen. if (FLAG_wasm_no_stack_checks || !env_ || !runtime_exception_support_) { return; } if (effect == nullptr) effect = effect_; if (control == nullptr) control = control_; Node* limit = graph()->NewNode( jsgraph()->machine()->Load(MachineType::Pointer()), jsgraph()->ExternalConstant( ExternalReference::address_of_stack_limit(jsgraph()->isolate())), jsgraph()->IntPtrConstant(0), *effect, *control); *effect = limit; Node* pointer = graph()->NewNode(jsgraph()->machine()->LoadStackPointer()); Node* check = graph()->NewNode(jsgraph()->machine()->UintLessThan(), limit, pointer); Diamond stack_check(graph(), jsgraph()->common(), check, BranchHint::kTrue); stack_check.Chain(*control); Handle<Code> code = BUILTIN_CODE(jsgraph()->isolate(), WasmStackGuard); CallInterfaceDescriptor idesc = WasmRuntimeCallDescriptor(jsgraph()->isolate()); CallDescriptor* desc = Linkage::GetStubCallDescriptor( jsgraph()->isolate(), jsgraph()->zone(), idesc, 0, CallDescriptor::kNoFlags, Operator::kNoProperties, MachineType::AnyTagged(), 1, Linkage::kNoContext); Node* stub_code = jsgraph()->HeapConstant(code); Node* call = graph()->NewNode(jsgraph()->common()->Call(desc), stub_code, *effect, stack_check.if_false); SetSourcePosition(call, position); Node* ephi = graph()->NewNode(jsgraph()->common()->EffectPhi(2), *effect, call, stack_check.merge); *control = stack_check.merge; *effect = ephi; } void WasmGraphBuilder::PatchInStackCheckIfNeeded() { if (!needs_stack_check_) return; Node* start = graph()->start(); // Place a stack check which uses a dummy node as control and effect. Node* dummy = graph()->NewNode(jsgraph()->common()->Dead()); Node* control = dummy; Node* effect = dummy; // The function-prologue stack check is associated with position 0, which // is never a position of any instruction in the function. StackCheck(0, &effect, &control); // In testing, no steck checks were emitted. Nothing to rewire then. if (effect == dummy) return; // Now patch all control uses of {start} to use {control} and all effect uses // to use {effect} instead. Then rewire the dummy node to use start instead. NodeProperties::ReplaceUses(start, start, effect, control); NodeProperties::ReplaceUses(dummy, nullptr, start, start); } Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right, wasm::WasmCodePosition position) { const Operator* op; MachineOperatorBuilder* m = jsgraph()->machine(); switch (opcode) { case wasm::kExprI32Add: op = m->Int32Add(); break; case wasm::kExprI32Sub: op = m->Int32Sub(); break; case wasm::kExprI32Mul: op = m->Int32Mul(); break; case wasm::kExprI32DivS: return BuildI32DivS(left, right, position); case wasm::kExprI32DivU: return BuildI32DivU(left, right, position); case wasm::kExprI32RemS: return BuildI32RemS(left, right, position); case wasm::kExprI32RemU: return BuildI32RemU(left, right, position); case wasm::kExprI32And: op = m->Word32And(); break; case wasm::kExprI32Ior: op = m->Word32Or(); break; case wasm::kExprI32Xor: op = m->Word32Xor(); break; case wasm::kExprI32Shl: op = m->Word32Shl(); right = MaskShiftCount32(right); break; case wasm::kExprI32ShrU: op = m->Word32Shr(); right = MaskShiftCount32(right); break; case wasm::kExprI32ShrS: op = m->Word32Sar(); right = MaskShiftCount32(right); break; case wasm::kExprI32Ror: op = m->Word32Ror(); right = MaskShiftCount32(right); break; case wasm::kExprI32Rol: right = MaskShiftCount32(right); return BuildI32Rol(left, right); case wasm::kExprI32Eq: op = m->Word32Equal(); break; case wasm::kExprI32Ne: return Invert(Binop(wasm::kExprI32Eq, left, right)); case wasm::kExprI32LtS: op = m->Int32LessThan(); break; case wasm::kExprI32LeS: op = m->Int32LessThanOrEqual(); break; case wasm::kExprI32LtU: op = m->Uint32LessThan(); break; case wasm::kExprI32LeU: op = m->Uint32LessThanOrEqual(); break; case wasm::kExprI32GtS: op = m->Int32LessThan(); std::swap(left, right); break; case wasm::kExprI32GeS: op = m->Int32LessThanOrEqual(); std::swap(left, right); break; case wasm::kExprI32GtU: op = m->Uint32LessThan(); std::swap(left, right); break; case wasm::kExprI32GeU: op = m->Uint32LessThanOrEqual(); std::swap(left, right); break; case wasm::kExprI64And: op = m->Word64And(); break; case wasm::kExprI64Add: op = m->Int64Add(); break; case wasm::kExprI64Sub: op = m->Int64Sub(); break; case wasm::kExprI64Mul: op = m->Int64Mul(); break; case wasm::kExprI64DivS: return BuildI64DivS(left, right, position); case wasm::kExprI64DivU: return BuildI64DivU(left, right, position); case wasm::kExprI64RemS: return BuildI64RemS(left, right, position); case wasm::kExprI64RemU: return BuildI64RemU(left, right, position); case wasm::kExprI64Ior: op = m->Word64Or(); break; case wasm::kExprI64Xor: op = m->Word64Xor(); break; case wasm::kExprI64Shl: op = m->Word64Shl(); right = MaskShiftCount64(right); break; case wasm::kExprI64ShrU: op = m->Word64Shr(); right = MaskShiftCount64(right); break; case wasm::kExprI64ShrS: op = m->Word64Sar(); right = MaskShiftCount64(right); break; case wasm::kExprI64Eq: op = m->Word64Equal(); break; case wasm::kExprI64Ne: return Invert(Binop(wasm::kExprI64Eq, left, right)); case wasm::kExprI64LtS: op = m->Int64LessThan(); break; case wasm::kExprI64LeS: op = m->Int64LessThanOrEqual(); break; case wasm::kExprI64LtU: op = m->Uint64LessThan(); break; case wasm::kExprI64LeU: op = m->Uint64LessThanOrEqual(); break; case wasm::kExprI64GtS: op = m->Int64LessThan(); std::swap(left, right); break; case wasm::kExprI64GeS: op = m->Int64LessThanOrEqual(); std::swap(left, right); break; case wasm::kExprI64GtU: op = m->Uint64LessThan(); std::swap(left, right); break; case wasm::kExprI64GeU: op = m->Uint64LessThanOrEqual(); std::swap(left, right); break; case wasm::kExprI64Ror: op = m->Word64Ror(); right = MaskShiftCount64(right); break; case wasm::kExprI64Rol: return BuildI64Rol(left, right); case wasm::kExprF32CopySign: return BuildF32CopySign(left, right); case wasm::kExprF64CopySign: return BuildF64CopySign(left, right); case wasm::kExprF32Add: op = m->Float32Add(); break; case wasm::kExprF32Sub: op = m->Float32Sub(); break; case wasm::kExprF32Mul: op = m->Float32Mul(); break; case wasm::kExprF32Div: op = m->Float32Div(); break; case wasm::kExprF32Eq: op = m->Float32Equal(); break; case wasm::kExprF32Ne: return Invert(Binop(wasm::kExprF32Eq, left, right)); case wasm::kExprF32Lt: op = m->Float32LessThan(); break; case wasm::kExprF32Ge: op = m->Float32LessThanOrEqual(); std::swap(left, right); break; case wasm::kExprF32Gt: op = m->Float32LessThan(); std::swap(left, right); break; case wasm::kExprF32Le: op = m->Float32LessThanOrEqual(); break; case wasm::kExprF64Add: op = m->Float64Add(); break; case wasm::kExprF64Sub: op = m->Float64Sub(); break; case wasm::kExprF64Mul: op = m->Float64Mul(); break; case wasm::kExprF64Div: op = m->Float64Div(); break; case wasm::kExprF64Eq: op = m->Float64Equal(); break; case wasm::kExprF64Ne: return Invert(Binop(wasm::kExprF64Eq, left, right)); case wasm::kExprF64Lt: op = m->Float64LessThan(); break; case wasm::kExprF64Le: op = m->Float64LessThanOrEqual(); break; case wasm::kExprF64Gt: op = m->Float64LessThan(); std::swap(left, right); break; case wasm::kExprF64Ge: op = m->Float64LessThanOrEqual(); std::swap(left, right); break; case wasm::kExprF32Min: op = m->Float32Min(); break; case wasm::kExprF64Min: op = m->Float64Min(); break; case wasm::kExprF32Max: op = m->Float32Max(); break; case wasm::kExprF64Max: op = m->Float64Max(); break; case wasm::kExprF64Pow: return BuildF64Pow(left, right); case wasm::kExprF64Atan2: op = m->Float64Atan2(); break; case wasm::kExprF64Mod: return BuildF64Mod(left, right); case wasm::kExprI32AsmjsDivS: return BuildI32AsmjsDivS(left, right); case wasm::kExprI32AsmjsDivU: return BuildI32AsmjsDivU(left, right); case wasm::kExprI32AsmjsRemS: return BuildI32AsmjsRemS(left, right); case wasm::kExprI32AsmjsRemU: return BuildI32AsmjsRemU(left, right); case wasm::kExprI32AsmjsStoreMem8: return BuildAsmjsStoreMem(MachineType::Int8(), left, right); case wasm::kExprI32AsmjsStoreMem16: return BuildAsmjsStoreMem(MachineType::Int16(), left, right); case wasm::kExprI32AsmjsStoreMem: return BuildAsmjsStoreMem(MachineType::Int32(), left, right); case wasm::kExprF32AsmjsStoreMem: return BuildAsmjsStoreMem(MachineType::Float32(), left, right); case wasm::kExprF64AsmjsStoreMem: return BuildAsmjsStoreMem(MachineType::Float64(), left, right); default: FATAL_UNSUPPORTED_OPCODE(opcode); } return graph()->NewNode(op, left, right); } Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input, wasm::WasmCodePosition position) { const Operator* op; MachineOperatorBuilder* m = jsgraph()->machine(); switch (opcode) { case wasm::kExprI32Eqz: op = m->Word32Equal(); return graph()->NewNode(op, input, jsgraph()->Int32Constant(0)); case wasm::kExprF32Abs: op = m->Float32Abs(); break; case wasm::kExprF32Neg: { op = m->Float32Neg(); break; } case wasm::kExprF32Sqrt: op = m->Float32Sqrt(); break; case wasm::kExprF64Abs: op = m->Float64Abs(); break; case wasm::kExprF64Neg: { op = m->Float64Neg(); break; } case wasm::kExprF64Sqrt: op = m->Float64Sqrt(); break; case wasm::kExprI32SConvertF64: return BuildI32SConvertF64(input, position, NumericImplementation::kTrap); case wasm::kExprI32SConvertSatF64: return BuildI32SConvertF64(input, position, NumericImplementation::kSaturate); case wasm::kExprI32UConvertF64: return BuildI32UConvertF64(input, position, NumericImplementation::kTrap); case wasm::kExprI32UConvertSatF64: return BuildI32UConvertF64(input, position, NumericImplementation::kSaturate); case wasm::kExprI32AsmjsSConvertF64: return BuildI32AsmjsSConvertF64(input); case wasm::kExprI32AsmjsUConvertF64: return BuildI32AsmjsUConvertF64(input); case wasm::kExprF32ConvertF64: op = m->TruncateFloat64ToFloat32(); break; case wasm::kExprF64SConvertI32: op = m->ChangeInt32ToFloat64(); break; case wasm::kExprF64UConvertI32: op = m->ChangeUint32ToFloat64(); break; case wasm::kExprF32SConvertI32: op = m->RoundInt32ToFloat32(); break; case wasm::kExprF32UConvertI32: op = m->RoundUint32ToFloat32(); break; case wasm::kExprI32SConvertF32: return BuildI32SConvertF32(input, position, NumericImplementation::kTrap); case wasm::kExprI32SConvertSatF32: return BuildI32SConvertF32(input, position, NumericImplementation::kSaturate); case wasm::kExprI32UConvertF32: return BuildI32UConvertF32(input, position, NumericImplementation::kTrap); case wasm::kExprI32UConvertSatF32: return BuildI32UConvertF32(input, position, NumericImplementation::kSaturate); case wasm::kExprI32AsmjsSConvertF32: return BuildI32AsmjsSConvertF32(input); case wasm::kExprI32AsmjsUConvertF32: return BuildI32AsmjsUConvertF32(input); case wasm::kExprF64ConvertF32: op = m->ChangeFloat32ToFloat64(); break; case wasm::kExprF32ReinterpretI32: op = m->BitcastInt32ToFloat32(); break; case wasm::kExprI32ReinterpretF32: op = m->BitcastFloat32ToInt32(); break; case wasm::kExprI32Clz: op = m->Word32Clz(); break; case wasm::kExprI32Ctz: { if (m->Word32Ctz().IsSupported()) { op = m->Word32Ctz().op(); break; } else if (m->Word32ReverseBits().IsSupported()) { Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input); Node* result = graph()->NewNode(m->Word32Clz(), reversed); return result; } else { return BuildI32Ctz(input); } } case wasm::kExprI32Popcnt: { if (m->Word32Popcnt().IsSupported()) { op = m->Word32Popcnt().op(); break; } else { return BuildI32Popcnt(input); } } case wasm::kExprF32Floor: { if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input); op = m->Float32RoundDown().op(); break; } case wasm::kExprF32Ceil: { if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input); op = m->Float32RoundUp().op(); break; } case wasm::kExprF32Trunc: { if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input); op = m->Float32RoundTruncate().op(); break; } case wasm::kExprF32NearestInt: { if (!m->Float32RoundTiesEven().IsSupported()) return BuildF32NearestInt(input); op = m->Float32RoundTiesEven().op(); break; } case wasm::kExprF64Floor: { if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input); op = m->Float64RoundDown().op(); break; } case wasm::kExprF64Ceil: { if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input); op = m->Float64RoundUp().op(); break; } case wasm::kExprF64Trunc: { if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input); op = m->Float64RoundTruncate().op(); break; } case wasm::kExprF64NearestInt: { if (!m->Float64RoundTiesEven().IsSupported()) return BuildF64NearestInt(input); op = m->Float64RoundTiesEven().op(); break; } case wasm::kExprF64Acos: { return BuildF64Acos(input); } case wasm::kExprF64Asin: { return BuildF64Asin(input); } case wasm::kExprF64Atan: op = m->Float64Atan(); break; case wasm::kExprF64Cos: { op = m->Float64Cos(); break; } case wasm::kExprF64Sin: { op = m->Float64Sin(); break; } case wasm::kExprF64Tan: { op = m->Float64Tan(); break; } case wasm::kExprF64Exp: { op = m->Float64Exp(); break; } case wasm::kExprF64Log: op = m->Float64Log(); break; case wasm::kExprI32ConvertI64: op = m->TruncateInt64ToInt32(); break; case wasm::kExprI64SConvertI32: op = m->ChangeInt32ToInt64(); break; case wasm::kExprI64UConvertI32: op = m->ChangeUint32ToUint64(); break; case wasm::kExprF64ReinterpretI64: op = m->BitcastInt64ToFloat64(); break; case wasm::kExprI64ReinterpretF64: op = m->BitcastFloat64ToInt64(); break; case wasm::kExprI64Clz: op = m->Word64Clz(); break; case wasm::kExprI64Ctz: { OptionalOperator ctz64 = m->Word64Ctz(); if (ctz64.IsSupported()) { op = ctz64.op(); break; } else if (m->Is32() && m->Word32Ctz().IsSupported()) { op = ctz64.placeholder(); break; } else if (m->Word64ReverseBits().IsSupported()) { Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input); Node* result = graph()->NewNode(m->Word64Clz(), reversed); return result; } else { return BuildI64Ctz(input); } } case wasm::kExprI64Popcnt: { OptionalOperator popcnt64 = m->Word64Popcnt(); if (popcnt64.IsSupported()) { op = popcnt64.op(); } else if (m->Is32() && m->Word32Popcnt().IsSupported()) { op = popcnt64.placeholder(); } else { return BuildI64Popcnt(input); } break; } case wasm::kExprI64Eqz: op = m->Word64Equal(); return graph()->NewNode(op, input, jsgraph()->Int64Constant(0)); case wasm::kExprF32SConvertI64: if (m->Is32()) { return BuildF32SConvertI64(input); } op = m->RoundInt64ToFloat32(); break; case wasm::kExprF32UConvertI64: if (m->Is32()) { return BuildF32UConvertI64(input); } op = m->RoundUint64ToFloat32(); break; case wasm::kExprF64SConvertI64: if (m->Is32()) { return BuildF64SConvertI64(input); } op = m->RoundInt64ToFloat64(); break; case wasm::kExprF64UConvertI64: if (m->Is32()) { return BuildF64UConvertI64(input); } op = m->RoundUint64ToFloat64(); break; case wasm::kExprI32SExtendI8: op = m->SignExtendWord8ToInt32(); break; case wasm::kExprI32SExtendI16: op = m->SignExtendWord16ToInt32(); break; case wasm::kExprI64SExtendI8: op = m->SignExtendWord8ToInt64(); break; case wasm::kExprI64SExtendI16: op = m->SignExtendWord16ToInt64(); break; case wasm::kExprI64SExtendI32: op = m->SignExtendWord32ToInt64(); break; case wasm::kExprI64SConvertF32: return BuildI64SConvertF32(input, position); case wasm::kExprI64SConvertF64: return BuildI64SConvertF64(input, position); case wasm::kExprI64UConvertF32: return BuildI64UConvertF32(input, position); case wasm::kExprI64UConvertF64: return BuildI64UConvertF64(input, position); case wasm::kExprI32AsmjsLoadMem8S: return BuildAsmjsLoadMem(MachineType::Int8(), input); case wasm::kExprI32AsmjsLoadMem8U: return BuildAsmjsLoadMem(MachineType::Uint8(), input); case wasm::kExprI32AsmjsLoadMem16S: return BuildAsmjsLoadMem(MachineType::Int16(), input); case wasm::kExprI32AsmjsLoadMem16U: return BuildAsmjsLoadMem(MachineType::Uint16(), input); case wasm::kExprI32AsmjsLoadMem: return BuildAsmjsLoadMem(MachineType::Int32(), input); case wasm::kExprF32AsmjsLoadMem: return BuildAsmjsLoadMem(MachineType::Float32(), input); case wasm::kExprF64AsmjsLoadMem: return BuildAsmjsLoadMem(MachineType::Float64(), input); default: FATAL_UNSUPPORTED_OPCODE(opcode); } return graph()->NewNode(op, input); } Node* WasmGraphBuilder::Float32Constant(float value) { return jsgraph()->Float32Constant(value); } Node* WasmGraphBuilder::Float64Constant(double value) { return jsgraph()->Float64Constant(value); } Node* WasmGraphBuilder::HeapConstant(Handle<HeapObject> value) { return jsgraph()->HeapConstant(value); } namespace { Node* Branch(JSGraph* jsgraph, Node* cond, Node** true_node, Node** false_node, Node* control, BranchHint hint) { DCHECK_NOT_NULL(cond); DCHECK_NOT_NULL(control); Node* branch = jsgraph->graph()->NewNode(jsgraph->common()->Branch(hint), cond, control); *true_node = jsgraph->graph()->NewNode(jsgraph->common()->IfTrue(), branch); *false_node = jsgraph->graph()->NewNode(jsgraph->common()->IfFalse(), branch); return branch; } } // namespace Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node, Node** false_node) { return Branch(jsgraph(), cond, true_node, false_node, *control_, BranchHint::kNone); } Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node, Node** false_node) { return Branch(jsgraph(), cond, true_node, false_node, *control_, BranchHint::kTrue); } Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node, Node** false_node) { return Branch(jsgraph(), cond, true_node, false_node, *control_, BranchHint::kFalse); } Builtins::Name WasmGraphBuilder::GetBuiltinIdForTrap(wasm::TrapReason reason) { if (runtime_exception_support_ == kNoRuntimeExceptionSupport) { // We use Builtins::builtin_count as a marker to tell the code generator // to generate a call to a testing c-function instead of a runtime // function. This code should only be called from a cctest. return Builtins::builtin_count; } switch (reason) { #define TRAPREASON_TO_MESSAGE(name) \ case wasm::k##name: \ return Builtins::kThrowWasm##name; FOREACH_WASM_TRAPREASON(TRAPREASON_TO_MESSAGE) #undef TRAPREASON_TO_MESSAGE default: UNREACHABLE(); } } Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond, wasm::WasmCodePosition position) { Builtins::Name trap_id = GetBuiltinIdForTrap(reason); Node* node = graph()->NewNode(jsgraph()->common()->TrapIf(trap_id), cond, Effect(), Control()); *control_ = node; SetSourcePosition(node, position); return node; } Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond, wasm::WasmCodePosition position) { Builtins::Name trap_id = GetBuiltinIdForTrap(reason); Node* node = graph()->NewNode(jsgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control()); *control_ = node; SetSourcePosition(node, position); return node; } // Add a check that traps if {node} is equal to {val}. Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node, int32_t val, wasm::WasmCodePosition position) { Int32Matcher m(node); if (m.HasValue() && !m.Is(val)) return graph()->start(); if (val == 0) { return TrapIfFalse(reason, node, position); } else { return TrapIfTrue(reason, graph()->NewNode(jsgraph()->machine()->Word32Equal(), node, jsgraph()->Int32Constant(val)), position); } } // Add a check that traps if {node} is zero. Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node, wasm::WasmCodePosition position) { return TrapIfEq32(reason, node, 0, position); } // Add a check that traps if {node} is equal to {val}. Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node, int64_t val, wasm::WasmCodePosition position) { Int64Matcher m(node); if (m.HasValue() && !m.Is(val)) return graph()->start(); return TrapIfTrue(reason, graph()->NewNode(jsgraph()->machine()->Word64Equal(), node, jsgraph()->Int64Constant(val)), position); } // Add a check that traps if {node} is zero. Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node, wasm::WasmCodePosition position) { return TrapIfEq64(reason, node, 0, position); } Node* WasmGraphBuilder::Switch(unsigned count, Node* key) { return graph()->NewNode(jsgraph()->common()->Switch(count), key, *control_); } Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) { DCHECK_EQ(IrOpcode::kSwitch, sw->opcode()); return graph()->NewNode(jsgraph()->common()->IfValue(value), sw); } Node* WasmGraphBuilder::IfDefault(Node* sw) { DCHECK_EQ(IrOpcode::kSwitch, sw->opcode()); return graph()->NewNode(jsgraph()->common()->IfDefault(), sw); } Node* WasmGraphBuilder::Return(unsigned count, Node** vals) { DCHECK_NOT_NULL(*control_); DCHECK_NOT_NULL(*effect_); static const int kStackAllocatedNodeBufferSize = 8; Node* stack_buffer[kStackAllocatedNodeBufferSize]; std::vector<Node*> heap_buffer; Node** buf = stack_buffer; if (count + 3 > kStackAllocatedNodeBufferSize) { heap_buffer.resize(count + 3); buf = heap_buffer.data(); } buf[0] = jsgraph()->Int32Constant(0); memcpy(buf + 1, vals, sizeof(void*) * count); buf[count + 1] = *effect_; buf[count + 2] = *control_; Node* ret = graph()->NewNode(jsgraph()->common()->Return(count), count + 3, buf); MergeControlToEnd(jsgraph(), ret); return ret; } Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); } Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) { TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position); ReturnVoid(); return nullptr; } Node* WasmGraphBuilder::MaskShiftCount32(Node* node) { static const int32_t kMask32 = 0x1F; if (!jsgraph()->machine()->Word32ShiftIsSafe()) { // Shifts by constants are so common we pattern-match them here. Int32Matcher match(node); if (match.HasValue()) { int32_t masked = (match.Value() & kMask32); if (match.Value() != masked) node = jsgraph()->Int32Constant(masked); } else { node = graph()->NewNode(jsgraph()->machine()->Word32And(), node, jsgraph()->Int32Constant(kMask32)); } } return node; } Node* WasmGraphBuilder::MaskShiftCount64(Node* node) { static const int64_t kMask64 = 0x3F; if (!jsgraph()->machine()->Word32ShiftIsSafe()) { // Shifts by constants are so common we pattern-match them here. Int64Matcher match(node); if (match.HasValue()) { int64_t masked = (match.Value() & kMask64); if (match.Value() != masked) node = jsgraph()->Int64Constant(masked); } else { node = graph()->NewNode(jsgraph()->machine()->Word64And(), node, jsgraph()->Int64Constant(kMask64)); } } return node; } static bool ReverseBytesSupported(MachineOperatorBuilder* m, size_t size_in_bytes) { switch (size_in_bytes) { case 4: case 16: return m->Word32ReverseBytes().IsSupported(); case 8: return m->Word64ReverseBytes().IsSupported(); default: break; } return false; } Node* WasmGraphBuilder::BuildChangeEndiannessStore( Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) { Node* result; Node* value = node; MachineOperatorBuilder* m = jsgraph()->machine(); int valueSizeInBytes = 1 << ElementSizeLog2Of(wasmtype); int valueSizeInBits = 8 * valueSizeInBytes; bool isFloat = false; switch (wasmtype) { case wasm::kWasmF64: value = graph()->NewNode(m->BitcastFloat64ToInt64(), node); isFloat = true; case wasm::kWasmI64: result = jsgraph()->Int64Constant(0); break; case wasm::kWasmF32: value = graph()->NewNode(m->BitcastFloat32ToInt32(), node); isFloat = true; case wasm::kWasmI32: result = jsgraph()->Int32Constant(0); break; case wasm::kWasmS128: DCHECK(ReverseBytesSupported(m, valueSizeInBytes)); break; default: UNREACHABLE(); break; } if (mem_rep == MachineRepresentation::kWord8) { // No need to change endianness for byte size, return original node return node; } if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) { // In case we store lower part of WasmI64 expression, we can truncate // upper 32bits value = graph()->NewNode(m->TruncateInt64ToInt32(), value); valueSizeInBytes = 1 << ElementSizeLog2Of(wasm::kWasmI32); valueSizeInBits = 8 * valueSizeInBytes; if (mem_rep == MachineRepresentation::kWord16) { value = graph()->NewNode(m->Word32Shl(), value, jsgraph()->Int32Constant(16)); } } else if (wasmtype == wasm::kWasmI32 && mem_rep == MachineRepresentation::kWord16) { value = graph()->NewNode(m->Word32Shl(), value, jsgraph()->Int32Constant(16)); } int i; uint32_t shiftCount; if (ReverseBytesSupported(m, valueSizeInBytes)) { switch (valueSizeInBytes) { case 4: result = graph()->NewNode(m->Word32ReverseBytes().op(), value); break; case 8: result = graph()->NewNode(m->Word64ReverseBytes().op(), value); break; case 16: { Node* byte_reversed_lanes[4]; for (int lane = 0; lane < 4; lane++) { byte_reversed_lanes[lane] = graph()->NewNode( m->Word32ReverseBytes().op(), graph()->NewNode(jsgraph()->machine()->I32x4ExtractLane(lane), value)); } // This is making a copy of the value. result = graph()->NewNode(jsgraph()->machine()->S128And(), value, value); for (int lane = 0; lane < 4; lane++) { result = graph()->NewNode(jsgraph()->machine()->I32x4ReplaceLane(3 - lane), result, byte_reversed_lanes[lane]); } break; } default: UNREACHABLE(); break; } } else { for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2; i += 8, shiftCount -= 16) { Node* shiftLower; Node* shiftHigher; Node* lowerByte; Node* higherByte; DCHECK_LT(0, shiftCount); DCHECK_EQ(0, (shiftCount + 8) % 16); if (valueSizeInBits > 32) { shiftLower = graph()->NewNode(m->Word64Shl(), value, jsgraph()->Int64Constant(shiftCount)); shiftHigher = graph()->NewNode(m->Word64Shr(), value, jsgraph()->Int64Constant(shiftCount)); lowerByte = graph()->NewNode( m->Word64And(), shiftLower, jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << (valueSizeInBits - 8 - i))); higherByte = graph()->NewNode( m->Word64And(), shiftHigher, jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i)); result = graph()->NewNode(m->Word64Or(), result, lowerByte); result = graph()->NewNode(m->Word64Or(), result, higherByte); } else { shiftLower = graph()->NewNode(m->Word32Shl(), value, jsgraph()->Int32Constant(shiftCount)); shiftHigher = graph()->NewNode(m->Word32Shr(), value, jsgraph()->Int32Constant(shiftCount)); lowerByte = graph()->NewNode( m->Word32And(), shiftLower, jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << (valueSizeInBits - 8 - i))); higherByte = graph()->NewNode( m->Word32And(), shiftHigher, jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i)); result = graph()->NewNode(m->Word32Or(), result, lowerByte); result = graph()->NewNode(m->Word32Or(), result, higherByte); } } } if (isFloat) { switch (wasmtype) { case wasm::kWasmF64: result = graph()->NewNode(m->BitcastInt64ToFloat64(), result); break; case wasm::kWasmF32: result = graph()->NewNode(m->BitcastInt32ToFloat32(), result); break; default: UNREACHABLE(); break; } } return result; } Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node, MachineType memtype, wasm::ValueType wasmtype) { Node* result; Node* value = node; MachineOperatorBuilder* m = jsgraph()->machine(); int valueSizeInBytes = 1 << ElementSizeLog2Of(memtype.representation()); int valueSizeInBits = 8 * valueSizeInBytes; bool isFloat = false; switch (memtype.representation()) { case MachineRepresentation::kFloat64: value = graph()->NewNode(m->BitcastFloat64ToInt64(), node); isFloat = true; case MachineRepresentation::kWord64: result = jsgraph()->Int64Constant(0); break; case MachineRepresentation::kFloat32: value = graph()->NewNode(m->BitcastFloat32ToInt32(), node); isFloat = true; case MachineRepresentation::kWord32: case MachineRepresentation::kWord16: result = jsgraph()->Int32Constant(0); break; case MachineRepresentation::kWord8: // No need to change endianness for byte size, return original node return node; break; case MachineRepresentation::kSimd128: DCHECK(ReverseBytesSupported(m, valueSizeInBytes)); break; default: UNREACHABLE(); break; } int i; uint32_t shiftCount; if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) { switch (valueSizeInBytes) { case 2: result = graph()->NewNode(m->Word32ReverseBytes().op(), graph()->NewNode(m->Word32Shl(), value, jsgraph()->Int32Constant(16))); break; case 4: result = graph()->NewNode(m->Word32ReverseBytes().op(), value); break; case 8: result = graph()->NewNode(m->Word64ReverseBytes().op(), value); break; case 16: { Node* byte_reversed_lanes[4]; for (int lane = 0; lane < 4; lane++) { byte_reversed_lanes[lane] = graph()->NewNode( m->Word32ReverseBytes().op(), graph()->NewNode(jsgraph()->machine()->I32x4ExtractLane(lane), value)); } // This is making a copy of the value. result = graph()->NewNode(jsgraph()->machine()->S128And(), value, value); for (int lane = 0; lane < 4; lane++) { result = graph()->NewNode(jsgraph()->machine()->I32x4ReplaceLane(3 - lane), result, byte_reversed_lanes[lane]); } break; } default: UNREACHABLE(); } } else { for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2; i += 8, shiftCount -= 16) { Node* shiftLower; Node* shiftHigher; Node* lowerByte; Node* higherByte; DCHECK_LT(0, shiftCount); DCHECK_EQ(0, (shiftCount + 8) % 16); if (valueSizeInBits > 32) { shiftLower = graph()->NewNode(m->Word64Shl(), value, jsgraph()->Int64Constant(shiftCount)); shiftHigher = graph()->NewNode(m->Word64Shr(), value, jsgraph()->Int64Constant(shiftCount)); lowerByte = graph()->NewNode( m->Word64And(), shiftLower, jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << (valueSizeInBits - 8 - i))); higherByte = graph()->NewNode( m->Word64And(), shiftHigher, jsgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i)); result = graph()->NewNode(m->Word64Or(), result, lowerByte); result = graph()->NewNode(m->Word64Or(), result, higherByte); } else { shiftLower = graph()->NewNode(m->Word32Shl(), value, jsgraph()->Int32Constant(shiftCount)); shiftHigher = graph()->NewNode(m->Word32Shr(), value, jsgraph()->Int32Constant(shiftCount)); lowerByte = graph()->NewNode( m->Word32And(), shiftLower, jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << (valueSizeInBits - 8 - i))); higherByte = graph()->NewNode( m->Word32And(), shiftHigher, jsgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i)); result = graph()->NewNode(m->Word32Or(), result, lowerByte); result = graph()->NewNode(m->Word32Or(), result, higherByte); } } } if (isFloat) { switch (memtype.representation()) { case MachineRepresentation::kFloat64: result = graph()->NewNode(m->BitcastInt64ToFloat64(), result); break; case MachineRepresentation::kFloat32: result = graph()->NewNode(m->BitcastInt32ToFloat32(), result); break; default: UNREACHABLE(); break; } } // We need to sign extend the value if (memtype.IsSigned()) { DCHECK(!isFloat); if (valueSizeInBits < 32) { Node* shiftBitCount; // Perform sign extension using following trick // result = (x << machine_width - type_width) >> (machine_width - // type_width) if (wasmtype == wasm::kWasmI64) { shiftBitCount = jsgraph()->Int32Constant(64 - valueSizeInBits); result = graph()->NewNode( m->Word64Sar(), graph()->NewNode(m->Word64Shl(), graph()->NewNode(m->ChangeInt32ToInt64(), result), shiftBitCount), shiftBitCount); } else if (wasmtype == wasm::kWasmI32) { shiftBitCount = jsgraph()->Int32Constant(32 - valueSizeInBits); result = graph()->NewNode( m->Word32Sar(), graph()->NewNode(m->Word32Shl(), result, shiftBitCount), shiftBitCount); } } } return result; } Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) { Node* result = Unop( wasm::kExprF32ReinterpretI32, Binop(wasm::kExprI32Ior, Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left), jsgraph()->Int32Constant(0x7FFFFFFF)), Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right), jsgraph()->Int32Constant(0x80000000)))); return result; } Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) { #if WASM_64 Node* result = Unop( wasm::kExprF64ReinterpretI64, Binop(wasm::kExprI64Ior, Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left), jsgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)), Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right), jsgraph()->Int64Constant(0x8000000000000000)))); return result; #else MachineOperatorBuilder* m = jsgraph()->machine(); Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left); Node* high_word_right = graph()->NewNode(m->Float64ExtractHighWord32(), right); Node* new_high_word = Binop(wasm::kExprI32Ior, Binop(wasm::kExprI32And, high_word_left, jsgraph()->Int32Constant(0x7FFFFFFF)), Binop(wasm::kExprI32And, high_word_right, jsgraph()->Int32Constant(0x80000000))); return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word); #endif } // Helper classes for float to int conversions. struct WasmGraphBuilder::IntConvertOps { MachineRepresentation word_rep() const { return MachineRepresentation::kWord32; } Node* zero() const { return builder_->Int32Constant(0); } virtual Node* min() const = 0; virtual Node* max() const = 0; virtual ~IntConvertOps() = default; protected: explicit IntConvertOps(WasmGraphBuilder* builder) : builder_(builder) {} WasmGraphBuilder* builder_; DISALLOW_IMPLICIT_CONSTRUCTORS(IntConvertOps); }; struct I32SConvertOps final : public WasmGraphBuilder::IntConvertOps { explicit I32SConvertOps(WasmGraphBuilder* builder) : WasmGraphBuilder::IntConvertOps(builder) {} ~I32SConvertOps() = default; Node* min() const { return builder_->Int32Constant(std::numeric_limits<int32_t>::min()); } Node* max() const { return builder_->Int32Constant(std::numeric_limits<int32_t>::max()); } DISALLOW_IMPLICIT_CONSTRUCTORS(I32SConvertOps); }; struct I32UConvertOps final : public WasmGraphBuilder::IntConvertOps { explicit I32UConvertOps(WasmGraphBuilder* builder) : WasmGraphBuilder::IntConvertOps(builder) {} ~I32UConvertOps() = default; Node* min() const { return builder_->Int32Constant(std::numeric_limits<uint32_t>::min()); } Node* max() const { return builder_->Int32Constant(std::numeric_limits<uint32_t>::max()); } DISALLOW_IMPLICIT_CONSTRUCTORS(I32UConvertOps); }; struct WasmGraphBuilder::FloatConvertOps { virtual Node* zero() const = 0; virtual wasm::WasmOpcode trunc_op() const = 0; virtual wasm::WasmOpcode ne_op() const = 0; virtual wasm::WasmOpcode lt_op() const = 0; virtual ~FloatConvertOps() = default; protected: explicit FloatConvertOps(WasmGraphBuilder* builder) : builder_(builder) {} WasmGraphBuilder* builder_; DISALLOW_IMPLICIT_CONSTRUCTORS(FloatConvertOps); }; struct F32ConvertOps final : public WasmGraphBuilder::FloatConvertOps { explicit F32ConvertOps(WasmGraphBuilder* builder) : WasmGraphBuilder::FloatConvertOps(builder) {} ~F32ConvertOps() = default; Node* zero() const { return builder_->Float32Constant(0.0); } wasm::WasmOpcode trunc_op() const { return wasm::kExprF32Trunc; } wasm::WasmOpcode ne_op() const { return wasm::kExprF32Ne; } wasm::WasmOpcode lt_op() const { return wasm::kExprF32Lt; } DISALLOW_IMPLICIT_CONSTRUCTORS(F32ConvertOps); }; struct F64ConvertOps final : public WasmGraphBuilder::FloatConvertOps { explicit F64ConvertOps(WasmGraphBuilder* builder) : WasmGraphBuilder::FloatConvertOps(builder) {} ~F64ConvertOps() = default; Node* zero() const { return builder_->Float64Constant(0.0); } wasm::WasmOpcode trunc_op() const { return wasm::kExprF64Trunc; } wasm::WasmOpcode ne_op() const { return wasm::kExprF64Ne; } wasm::WasmOpcode lt_op() const { return wasm::kExprF64Lt; } DISALLOW_IMPLICIT_CONSTRUCTORS(F64ConvertOps); }; Node* WasmGraphBuilder::BuildConvertCheck(Node* test, Node* result, Node* input, wasm::WasmCodePosition position, NumericImplementation impl, const IntConvertOps* int_ops, const FloatConvertOps* float_ops) { switch (impl) { case NumericImplementation::kTrap: TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position); return result; case NumericImplementation::kSaturate: { Diamond tl_d(graph(), jsgraph()->common(), test, BranchHint::kFalse); tl_d.Chain(*control_); Diamond nan_d(graph(), jsgraph()->common(), Binop(float_ops->ne_op(), input, input), // Checks if NaN. BranchHint::kFalse); nan_d.Nest(tl_d, true); Diamond sat_d(graph(), jsgraph()->common(), Binop(float_ops->lt_op(), input, float_ops->zero()), BranchHint::kNone); sat_d.Nest(nan_d, false); Node* sat_val = sat_d.Phi(int_ops->word_rep(), int_ops->min(), int_ops->max()); Node* nan_val = nan_d.Phi(int_ops->word_rep(), int_ops->zero(), sat_val); return tl_d.Phi(int_ops->word_rep(), nan_val, result); } } UNREACHABLE(); } Node* WasmGraphBuilder::BuildI32ConvertOp( Node* input, wasm::WasmCodePosition position, NumericImplementation impl, const Operator* op, wasm::WasmOpcode check_op, const IntConvertOps* int_ops, const FloatConvertOps* float_ops) { // Truncation of the input value is needed for the overflow check later. Node* trunc = Unop(float_ops->trunc_op(), input); Node* result = graph()->NewNode(op, trunc); // Convert the result back to f64. If we end up at a different value than the // truncated input value, then there has been an overflow and we // trap/saturate. Node* check = Unop(check_op, result); Node* overflow = Binop(float_ops->ne_op(), trunc, check); return BuildConvertCheck(overflow, result, input, position, impl, int_ops, float_ops); } Node* WasmGraphBuilder::BuildI32SConvertF32(Node* input, wasm::WasmCodePosition position, NumericImplementation impl) { I32SConvertOps int_ops(this); F32ConvertOps float_ops(this); return BuildI32ConvertOp(input, position, impl, jsgraph()->machine()->TruncateFloat32ToInt32(), wasm::kExprF32SConvertI32, &int_ops, &float_ops); } Node* WasmGraphBuilder::BuildI32SConvertF64(Node* input, wasm::WasmCodePosition position, NumericImplementation impl) { I32SConvertOps int_ops(this); F64ConvertOps float_ops(this); return BuildI32ConvertOp(input, position, impl, jsgraph()->machine()->ChangeFloat64ToInt32(), wasm::kExprF64SConvertI32, &int_ops, &float_ops); } Node* WasmGraphBuilder::BuildI32UConvertF32(Node* input, wasm::WasmCodePosition position, NumericImplementation impl) { I32UConvertOps int_ops(this); F32ConvertOps float_ops(this); return BuildI32ConvertOp(input, position, impl, jsgraph()->machine()->TruncateFloat32ToUint32(), wasm::kExprF32UConvertI32, &int_ops, &float_ops); } Node* WasmGraphBuilder::BuildI32UConvertF64(Node* input, wasm::WasmCodePosition position, NumericImplementation impl) { I32UConvertOps int_ops(this); F64ConvertOps float_ops(this); return BuildI32ConvertOp(input, position, impl, jsgraph()->machine()->TruncateFloat64ToUint32(), wasm::kExprF64UConvertI32, &int_ops, &float_ops); } Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) { MachineOperatorBuilder* m = jsgraph()->machine(); // asm.js must use the wacky JS semantics. input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input); return graph()->NewNode(m->TruncateFloat64ToWord32(), input); } Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) { MachineOperatorBuilder* m = jsgraph()->machine(); // asm.js must use the wacky JS semantics. return graph()->NewNode(m->TruncateFloat64ToWord32(), input); } Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) { MachineOperatorBuilder* m = jsgraph()->machine(); // asm.js must use the wacky JS semantics. input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input); return graph()->NewNode(m->TruncateFloat64ToWord32(), input); } Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) { MachineOperatorBuilder* m = jsgraph()->machine(); // asm.js must use the wacky JS semantics. return graph()->NewNode(m->TruncateFloat64ToWord32(), input); } Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref, MachineRepresentation input_type) { Node* stack_slot_param = graph()->NewNode(jsgraph()->machine()->StackSlot(input_type)); const Operator* store_op = jsgraph()->machine()->Store( StoreRepresentation(input_type, kNoWriteBarrier)); *effect_ = graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0), input, *effect_, *control_); MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 1); sig_builder.AddReturn(MachineType::Int32()); sig_builder.AddParam(MachineType::Pointer()); Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); return BuildCCall(sig_builder.Build(), function, stack_slot_param); } Node* WasmGraphBuilder::BuildI32Ctz(Node* input) { return BuildBitCountingCall( input, ExternalReference::wasm_word32_ctz(jsgraph()->isolate()), MachineRepresentation::kWord32); } Node* WasmGraphBuilder::BuildI64Ctz(Node* input) { return Unop(wasm::kExprI64UConvertI32, BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz( jsgraph()->isolate()), MachineRepresentation::kWord64)); } Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) { return BuildBitCountingCall( input, ExternalReference::wasm_word32_popcnt(jsgraph()->isolate()), MachineRepresentation::kWord32); } Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) { return Unop(wasm::kExprI64UConvertI32, BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt( jsgraph()->isolate()), MachineRepresentation::kWord64)); } Node* WasmGraphBuilder::BuildF32Trunc(Node* input) { MachineType type = MachineType::Float32(); ExternalReference ref = ExternalReference::wasm_f32_trunc(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF32Floor(Node* input) { MachineType type = MachineType::Float32(); ExternalReference ref = ExternalReference::wasm_f32_floor(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF32Ceil(Node* input) { MachineType type = MachineType::Float32(); ExternalReference ref = ExternalReference::wasm_f32_ceil(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) { MachineType type = MachineType::Float32(); ExternalReference ref = ExternalReference::wasm_f32_nearest_int(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF64Trunc(Node* input) { MachineType type = MachineType::Float64(); ExternalReference ref = ExternalReference::wasm_f64_trunc(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF64Floor(Node* input) { MachineType type = MachineType::Float64(); ExternalReference ref = ExternalReference::wasm_f64_floor(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF64Ceil(Node* input) { MachineType type = MachineType::Float64(); ExternalReference ref = ExternalReference::wasm_f64_ceil(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) { MachineType type = MachineType::Float64(); ExternalReference ref = ExternalReference::wasm_f64_nearest_int(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF64Acos(Node* input) { MachineType type = MachineType::Float64(); ExternalReference ref = ExternalReference::f64_acos_wrapper_function(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF64Asin(Node* input) { MachineType type = MachineType::Float64(); ExternalReference ref = ExternalReference::f64_asin_wrapper_function(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, input); } Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) { MachineType type = MachineType::Float64(); ExternalReference ref = ExternalReference::wasm_float64_pow(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, left, right); } Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) { MachineType type = MachineType::Float64(); ExternalReference ref = ExternalReference::f64_mod_wrapper_function(jsgraph()->isolate()); return BuildCFuncInstruction(ref, type, left, right); } Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref, MachineType type, Node* input0, Node* input1) { // We do truncation by calling a C function which calculates the result. // The input is passed to the C function as a double*'s to avoid double // parameters. For this we reserve slots on the stack, store the parameters // in those slots, pass pointers to the slot to the C function, // and after calling the C function we collect the return value from // the stack slot. Node* stack_slot_param0 = graph()->NewNode(jsgraph()->machine()->StackSlot(type.representation())); const Operator* store_op0 = jsgraph()->machine()->Store( StoreRepresentation(type.representation(), kNoWriteBarrier)); *effect_ = graph()->NewNode(store_op0, stack_slot_param0, jsgraph()->Int32Constant(0), input0, *effect_, *control_); Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); if (input1 == nullptr) { const int input_count = 1; Signature<MachineType>::Builder sig_builder(jsgraph()->zone(), 0, input_count); sig_builder.AddParam(MachineType::Pointer()); BuildCCall(sig_builder.Build(), function, stack_slot_param0); } else { Node* stack_slot_param1 = graph()->NewNode( jsgraph()->machine()->StackSlot(type.representation())); const Operator* store_op1 = jsgraph()->machine()->Store( StoreRepresentation(type.representation(), kNoWriteBarrier)); *effect_ = graph()->NewNode(store_op1, stack_slot_param1, jsgraph()->Int32Constant(0), input1, *effect_, *control_); const int input_count = 2; Signature<MachineType>::Builder sig_builder(jsgraph()->zone(), 0, input_count); sig_builder.AddParam(MachineType::Pointer()); sig_builder.AddParam(MachineType::Pointer()); BuildCCall(sig_builder.Build(), function, stack_slot_param0, stack_slot_param1); } const Operator* load_op = jsgraph()->machine()->Load(type); Node* load = graph()->NewNode(load_op, stack_slot_param0, jsgraph()->Int32Constant(0), *effect_, *control_); *effect_ = load; return load; } Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) { // TODO(titzer/bradnelson): Check handlng of asm.js case. return BuildIntToFloatConversionInstruction( input, ExternalReference::wasm_int64_to_float32(jsgraph()->isolate()), MachineRepresentation::kWord64, MachineType::Float32()); } Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) { // TODO(titzer/bradnelson): Check handlng of asm.js case. return BuildIntToFloatConversionInstruction( input, ExternalReference::wasm_uint64_to_float32(jsgraph()->isolate()), MachineRepresentation::kWord64, MachineType::Float32()); } Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) { return BuildIntToFloatConversionInstruction( input, ExternalReference::wasm_int64_to_float64(jsgraph()->isolate()), MachineRepresentation::kWord64, MachineType::Float64()); } Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) { return BuildIntToFloatConversionInstruction( input, ExternalReference::wasm_uint64_to_float64(jsgraph()->isolate()), MachineRepresentation::kWord64, MachineType::Float64()); } Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction( Node* input, ExternalReference ref, MachineRepresentation parameter_representation, const MachineType result_type) { Node* stack_slot_param = graph()->NewNode( jsgraph()->machine()->StackSlot(parameter_representation)); Node* stack_slot_result = graph()->NewNode( jsgraph()->machine()->StackSlot(result_type.representation())); const Operator* store_op = jsgraph()->machine()->Store( StoreRepresentation(parameter_representation, kNoWriteBarrier)); *effect_ = graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0), input, *effect_, *control_); MachineSignature::Builder sig_builder(jsgraph()->zone(), 0, 2); sig_builder.AddParam(MachineType::Pointer()); sig_builder.AddParam(MachineType::Pointer()); Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); BuildCCall(sig_builder.Build(), function, stack_slot_param, stack_slot_result); const Operator* load_op = jsgraph()->machine()->Load(result_type); Node* load = graph()->NewNode(load_op, stack_slot_result, jsgraph()->Int32Constant(0), *effect_, *control_); *effect_ = load; return load; } Node* WasmGraphBuilder::BuildI64SConvertF32(Node* input, wasm::WasmCodePosition position) { if (jsgraph()->machine()->Is32()) { return BuildFloatToIntConversionInstruction( input, ExternalReference::wasm_float32_to_int64(jsgraph()->isolate()), MachineRepresentation::kFloat32, MachineType::Int64(), position); } else { Node* trunc = graph()->NewNode( jsgraph()->machine()->TryTruncateFloat32ToInt64(), input); Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc, graph()->start()); Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc, graph()->start()); ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position); return result; } } Node* WasmGraphBuilder::BuildI64UConvertF32(Node* input, wasm::WasmCodePosition position) { if (jsgraph()->machine()->Is32()) { return BuildFloatToIntConversionInstruction( input, ExternalReference::wasm_float32_to_uint64(jsgraph()->isolate()), MachineRepresentation::kFloat32, MachineType::Int64(), position); } else { Node* trunc = graph()->NewNode( jsgraph()->machine()->TryTruncateFloat32ToUint64(), input); Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc, graph()->start()); Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc, graph()->start()); ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position); return result; } } Node* WasmGraphBuilder::BuildI64SConvertF64(Node* input, wasm::WasmCodePosition position) { if (jsgraph()->machine()->Is32()) { return BuildFloatToIntConversionInstruction( input, ExternalReference::wasm_float64_to_int64(jsgraph()->isolate()), MachineRepresentation::kFloat64, MachineType::Int64(), position); } else { Node* trunc = graph()->NewNode( jsgraph()->machine()->TryTruncateFloat64ToInt64(), input); Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc, graph()->start()); Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc, graph()->start()); ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position); return result; } } Node* WasmGraphBuilder::BuildI64UConvertF64(Node* input, wasm::WasmCodePosition position) { if (jsgraph()->machine()->Is32()) { return BuildFloatToIntConversionInstruction( input, ExternalReference::wasm_float64_to_uint64(jsgraph()->isolate()), MachineRepresentation::kFloat64, MachineType::Int64(), position); } else { Node* trunc = graph()->NewNode( jsgraph()->machine()->TryTruncateFloat64ToUint64(), input); Node* result = graph()->NewNode(jsgraph()->common()->Projection(0), trunc, graph()->start()); Node* overflow = graph()->NewNode(jsgraph()->common()->Projection(1), trunc, graph()->start()); ZeroCheck64(wasm::kTrapFloatUnrepresentable, overflow, position); return result; } } Node* WasmGraphBuilder::BuildFloatToIntConversionInstruction( Node* input, ExternalReference ref, MachineRepresentation parameter_representation, const MachineType result_type, wasm::WasmCodePosition position) { Node* stack_slot_param = graph()->NewNode( jsgraph()->machine()->StackSlot(parameter_representation)); Node* stack_slot_result = graph()->NewNode( jsgraph()->machine()->StackSlot(result_type.representation())); const Operator* store_op = jsgraph()->machine()->Store( StoreRepresentation(parameter_representation, kNoWriteBarrier)); *effect_ = graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0), input, *effect_, *control_); MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 2); sig_builder.AddReturn(MachineType::Int32()); sig_builder.AddParam(MachineType::Pointer()); sig_builder.AddParam(MachineType::Pointer()); Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); ZeroCheck32(wasm::kTrapFloatUnrepresentable, BuildCCall(sig_builder.Build(), function, stack_slot_param, stack_slot_result), position); const Operator* load_op = jsgraph()->machine()->Load(result_type); Node* load = graph()->NewNode(load_op, stack_slot_result, jsgraph()->Int32Constant(0), *effect_, *control_); *effect_ = load; return load; } Node* WasmGraphBuilder::GrowMemory(Node* input) { SetNeedsStackCheck(); Diamond check_input_range( graph(), jsgraph()->common(), graph()->NewNode(jsgraph()->machine()->Uint32LessThanOrEqual(), input, jsgraph()->Uint32Constant(FLAG_wasm_max_mem_pages)), BranchHint::kTrue); check_input_range.Chain(*control_); Node* parameters[] = {BuildChangeUint32ToSmi(input)}; Node* old_effect = *effect_; *control_ = check_input_range.if_true; Node* call = BuildCallToRuntime(Runtime::kWasmGrowMemory, parameters, arraysize(parameters)); Node* result = BuildChangeSmiToInt32(call); result = check_input_range.Phi(MachineRepresentation::kWord32, result, jsgraph()->Int32Constant(-1)); *effect_ = graph()->NewNode(jsgraph()->common()->EffectPhi(2), *effect_, old_effect, check_input_range.merge); *control_ = check_input_range.merge; return result; } uint32_t WasmGraphBuilder::GetExceptionEncodedSize( const wasm::WasmException* exception) const { const wasm::WasmExceptionSig* sig = exception->sig; uint32_t encoded_size = 0; for (size_t i = 0; i < sig->parameter_count(); ++i) { size_t byte_size = size_t(1) << ElementSizeLog2Of(sig->GetParam(i)); DCHECK_EQ(byte_size % kBytesPerExceptionValuesArrayElement, 0); DCHECK_LE(1, byte_size / kBytesPerExceptionValuesArrayElement); encoded_size += byte_size / kBytesPerExceptionValuesArrayElement; } return encoded_size; } Node* WasmGraphBuilder::Throw(uint32_t tag, const wasm::WasmException* exception, const Vector<Node*> values) { SetNeedsStackCheck(); uint32_t encoded_size = GetExceptionEncodedSize(exception); Node* create_parameters[] = { BuildChangeUint32ToSmi(ConvertExceptionTagToRuntimeId(tag)), BuildChangeUint32ToSmi(Uint32Constant(encoded_size))}; BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters, arraysize(create_parameters)); uint32_t index = 0; const wasm::WasmExceptionSig* sig = exception->sig; MachineOperatorBuilder* m = jsgraph()->machine(); for (size_t i = 0; i < sig->parameter_count(); ++i) { Node* value = values[i]; switch (sig->GetParam(i)) { case wasm::kWasmF32: value = graph()->NewNode(m->BitcastFloat32ToInt32(), value); // Intentionally fall to next case. case wasm::kWasmI32: BuildEncodeException32BitValue(&index, value); break; case wasm::kWasmF64: value = graph()->NewNode(m->BitcastFloat64ToInt64(), value); // Intentionally fall to next case. case wasm::kWasmI64: { Node* upper32 = graph()->NewNode( m->TruncateInt64ToInt32(), Binop(wasm::kExprI64ShrU, value, Int64Constant(32))); BuildEncodeException32BitValue(&index, upper32); Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value); BuildEncodeException32BitValue(&index, lower32); break; } default: UNREACHABLE(); } } DCHECK_EQ(encoded_size, index); return BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0); } void WasmGraphBuilder::BuildEncodeException32BitValue(uint32_t* index, Node* value) { MachineOperatorBuilder* machine = jsgraph()->machine(); Node* upper_parameters[] = { BuildChangeUint32ToSmi(Int32Constant(*index)), BuildChangeUint32ToSmi( graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16))), }; BuildCallToRuntime(Runtime::kWasmExceptionSetElement, upper_parameters, arraysize(upper_parameters)); ++(*index); Node* lower_parameters[] = { BuildChangeUint32ToSmi(Int32Constant(*index)), BuildChangeUint32ToSmi(graph()->NewNode(machine->Word32And(), value, Int32Constant(0xFFFFu))), }; BuildCallToRuntime(Runtime::kWasmExceptionSetElement, lower_parameters, arraysize(lower_parameters)); ++(*index); } Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* const* values, uint32_t* index) { MachineOperatorBuilder* machine = jsgraph()->machine(); Node* upper = BuildChangeSmiToInt32(values[*index]); (*index)++; upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16)); Node* lower = BuildChangeSmiToInt32(values[*index]); (*index)++; Node* value = graph()->NewNode(machine->Word32Or(), upper, lower); return value; } Node* WasmGraphBuilder::Rethrow() { SetNeedsStackCheck(); Node* result = BuildCallToRuntime(Runtime::kWasmThrow, nullptr, 0); return result; } Node* WasmGraphBuilder::ConvertExceptionTagToRuntimeId(uint32_t tag) { // TODO(kschimpf): Handle exceptions from different modules, when they are // linked at runtime. return Uint32Constant(tag); } Node* WasmGraphBuilder::GetExceptionRuntimeId() { SetNeedsStackCheck(); return BuildChangeSmiToInt32( BuildCallToRuntime(Runtime::kWasmGetExceptionRuntimeId, nullptr, 0)); } Node** WasmGraphBuilder::GetExceptionValues( const wasm::WasmException* except_decl) { // TODO(kschimpf): We need to move this code to the function-body-decoder.cc // in order to build landing-pad (exception) edges in case the runtime // call causes an exception. // Start by getting the encoded values from the exception. uint32_t encoded_size = GetExceptionEncodedSize(except_decl); Node** values = Buffer(encoded_size); for (uint32_t i = 0; i < encoded_size; ++i) { Node* parameters[] = {BuildChangeUint32ToSmi(Uint32Constant(i))}; values[i] = BuildCallToRuntime(Runtime::kWasmExceptionGetElement, parameters, arraysize(parameters)); } // Now convert the leading entries to the corresponding parameter values. uint32_t index = 0; const wasm::WasmExceptionSig* sig = except_decl->sig; for (size_t i = 0; i < sig->parameter_count(); ++i) { Node* value = BuildDecodeException32BitValue(values, &index); switch (wasm::ValueType type = sig->GetParam(i)) { case wasm::kWasmF32: { value = Unop(wasm::kExprF32ReinterpretI32, value); break; } case wasm::kWasmI32: break; case wasm::kWasmF64: case wasm::kWasmI64: { Node* upper = Binop(wasm::kExprI64Shl, Unop(wasm::kExprI64UConvertI32, value), Int64Constant(32)); Node* lower = Unop(wasm::kExprI64UConvertI32, BuildDecodeException32BitValue(values, &index)); value = Binop(wasm::kExprI64Ior, upper, lower); if (type == wasm::kWasmF64) { value = Unop(wasm::kExprF64ReinterpretI64, value); } break; } default: UNREACHABLE(); } values[i] = value; } DCHECK_EQ(index, encoded_size); return values; } Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right, wasm::WasmCodePosition position) { MachineOperatorBuilder* m = jsgraph()->machine(); ZeroCheck32(wasm::kTrapDivByZero, right, position); Node* before = *control_; Node* denom_is_m1; Node* denom_is_not_m1; BranchExpectFalse( graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(-1)), &denom_is_m1, &denom_is_not_m1); *control_ = denom_is_m1; TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position); if (*control_ != denom_is_m1) { *control_ = graph()->NewNode(jsgraph()->common()->Merge(2), denom_is_not_m1, *control_); } else { *control_ = before; } return graph()->NewNode(m->Int32Div(), left, right, *control_); } Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right, wasm::WasmCodePosition position) { MachineOperatorBuilder* m = jsgraph()->machine(); ZeroCheck32(wasm::kTrapRemByZero, right, position); Diamond d( graph(), jsgraph()->common(), graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(-1)), BranchHint::kFalse); d.Chain(*control_); return d.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0), graph()->NewNode(m->Int32Mod(), left, right, d.if_false)); } Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right, wasm::WasmCodePosition position) { MachineOperatorBuilder* m = jsgraph()->machine(); return graph()->NewNode(m->Uint32Div(), left, right, ZeroCheck32(wasm::kTrapDivByZero, right, position)); } Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right, wasm::WasmCodePosition position) { MachineOperatorBuilder* m = jsgraph()->machine(); return graph()->NewNode(m->Uint32Mod(), left, right, ZeroCheck32(wasm::kTrapRemByZero, right, position)); } Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) { MachineOperatorBuilder* m = jsgraph()->machine(); Int32Matcher mr(right); if (mr.HasValue()) { if (mr.Value() == 0) { return jsgraph()->Int32Constant(0); } else if (mr.Value() == -1) { // The result is the negation of the left input. return graph()->NewNode(m->Int32Sub(), jsgraph()->Int32Constant(0), left); } return graph()->NewNode(m->Int32Div(), left, right, *control_); } // asm.js semantics return 0 on divide or mod by zero. if (m->Int32DivIsSafe()) { // The hardware instruction does the right thing (e.g. arm). return graph()->NewNode(m->Int32Div(), left, right, graph()->start()); } // Check denominator for zero. Diamond z( graph(), jsgraph()->common(), graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(0)), BranchHint::kFalse); // Check numerator for -1. (avoid minint / -1 case). Diamond n( graph(), jsgraph()->common(), graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(-1)), BranchHint::kFalse); Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false); Node* neg = graph()->NewNode(m->Int32Sub(), jsgraph()->Int32Constant(0), left); return n.Phi( MachineRepresentation::kWord32, neg, z.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0), div)); } Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) { CommonOperatorBuilder* c = jsgraph()->common(); MachineOperatorBuilder* m = jsgraph()->machine(); Node* const zero = jsgraph()->Int32Constant(0); Int32Matcher mr(right); if (mr.HasValue()) { if (mr.Value() == 0 || mr.Value() == -1) { return zero; } return graph()->NewNode(m->Int32Mod(), left, right, *control_); } // General case for signed integer modulus, with optimization for (unknown) // power of 2 right hand side. // // if 0 < right then // msk = right - 1 // if right & msk != 0 then // left % right // else // if left < 0 then // -(-left & msk) // else // left & msk // else // if right < -1 then // left % right // else // zero // // Note: We do not use the Diamond helper class here, because it really hurts // readability with nested diamonds. Node* const minus_one = jsgraph()->Int32Constant(-1); const Operator* const merge_op = c->Merge(2); const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2); Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right); Node* branch0 = graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start()); Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0); Node* true0; { Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one); Node* check1 = graph()->NewNode(m->Word32And(), right, msk); Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0); Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1); Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1); Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1); Node* false1; { Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero); Node* branch2 = graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1); Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2); Node* true2 = graph()->NewNode( m->Int32Sub(), zero, graph()->NewNode(m->Word32And(), graph()->NewNode(m->Int32Sub(), zero, left), msk)); Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2); Node* false2 = graph()->NewNode(m->Word32And(), left, msk); if_false1 = graph()->NewNode(merge_op, if_true2, if_false2); false1 = graph()->NewNode(phi_op, true2, false2, if_false1); } if_true0 = graph()->NewNode(merge_op, if_true1, if_false1); true0 = graph()->NewNode(phi_op, true1, false1, if_true0); } Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0); Node* false0; { Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one); Node* branch1 = graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0); Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1); Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1); Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1); Node* false1 = zero; if_false0 = graph()->NewNode(merge_op, if_true1, if_false1); false0 = graph()->NewNode(phi_op, true1, false1, if_false0); } Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0); return graph()->NewNode(phi_op, true0, false0, merge0); } Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) { MachineOperatorBuilder* m = jsgraph()->machine(); // asm.js semantics return 0 on divide or mod by zero. if (m->Uint32DivIsSafe()) { // The hardware instruction does the right thing (e.g. arm). return graph()->NewNode(m->Uint32Div(), left, right, graph()->start()); } // Explicit check for x % 0. Diamond z( graph(), jsgraph()->common(), graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(0)), BranchHint::kFalse); return z.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0), graph()->NewNode(jsgraph()->machine()->Uint32Div(), left, right, z.if_false)); } Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) { MachineOperatorBuilder* m = jsgraph()->machine(); // asm.js semantics return 0 on divide or mod by zero. // Explicit check for x % 0. Diamond z( graph(), jsgraph()->common(), graph()->NewNode(m->Word32Equal(), right, jsgraph()->Int32Constant(0)), BranchHint::kFalse); Node* rem = graph()->NewNode(jsgraph()->machine()->Uint32Mod(), left, right, z.if_false); return z.Phi(MachineRepresentation::kWord32, jsgraph()->Int32Constant(0), rem); } Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right, wasm::WasmCodePosition position) { if (jsgraph()->machine()->Is32()) { return BuildDiv64Call( left, right, ExternalReference::wasm_int64_div(jsgraph()->isolate()), MachineType::Int64(), wasm::kTrapDivByZero, position); } ZeroCheck64(wasm::kTrapDivByZero, right, position); Node* before = *control_; Node* denom_is_m1; Node* denom_is_not_m1; BranchExpectFalse(graph()->NewNode(jsgraph()->machine()->Word64Equal(), right, jsgraph()->Int64Constant(-1)), &denom_is_m1, &denom_is_not_m1); *control_ = denom_is_m1; TrapIfEq64(wasm::kTrapDivUnrepresentable, left, std::numeric_limits<int64_t>::min(), position); if (*control_ != denom_is_m1) { *control_ = graph()->NewNode(jsgraph()->common()->Merge(2), denom_is_not_m1, *control_); } else { *control_ = before; } return graph()->NewNode(jsgraph()->machine()->Int64Div(), left, right, *control_); } Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right, wasm::WasmCodePosition position) { if (jsgraph()->machine()->Is32()) { return BuildDiv64Call( left, right, ExternalReference::wasm_int64_mod(jsgraph()->isolate()), MachineType::Int64(), wasm::kTrapRemByZero, position); } ZeroCheck64(wasm::kTrapRemByZero, right, position); Diamond d(jsgraph()->graph(), jsgraph()->common(), graph()->NewNode(jsgraph()->machine()->Word64Equal(), right, jsgraph()->Int64Constant(-1))); d.Chain(*control_); Node* rem = graph()->NewNode(jsgraph()->machine()->Int64Mod(), left, right, d.if_false); return d.Phi(MachineRepresentation::kWord64, jsgraph()->Int64Constant(0), rem); } Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right, wasm::WasmCodePosition position) { if (jsgraph()->machine()->Is32()) { return BuildDiv64Call( left, right, ExternalReference::wasm_uint64_div(jsgraph()->isolate()), MachineType::Int64(), wasm::kTrapDivByZero, position); } return graph()->NewNode(jsgraph()->machine()->Uint64Div(), left, right, ZeroCheck64(wasm::kTrapDivByZero, right, position)); } Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right, wasm::WasmCodePosition position) { if (jsgraph()->machine()->Is32()) { return BuildDiv64Call( left, right, ExternalReference::wasm_uint64_mod(jsgraph()->isolate()), MachineType::Int64(), wasm::kTrapRemByZero, position); } return graph()->NewNode(jsgraph()->machine()->Uint64Mod(), left, right, ZeroCheck64(wasm::kTrapRemByZero, right, position)); } Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right, ExternalReference ref, MachineType result_type, int trap_zero, wasm::WasmCodePosition position) { Node* stack_slot_dst = graph()->NewNode( jsgraph()->machine()->StackSlot(MachineRepresentation::kWord64)); Node* stack_slot_src = graph()->NewNode( jsgraph()->machine()->StackSlot(MachineRepresentation::kWord64)); const Operator* store_op = jsgraph()->machine()->Store( StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier)); *effect_ = graph()->NewNode(store_op, stack_slot_dst, jsgraph()->Int32Constant(0), left, *effect_, *control_); *effect_ = graph()->NewNode(store_op, stack_slot_src, jsgraph()->Int32Constant(0), right, *effect_, *control_); MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 2); sig_builder.AddReturn(MachineType::Int32()); sig_builder.AddParam(MachineType::Pointer()); sig_builder.AddParam(MachineType::Pointer()); Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref)); Node* call = BuildCCall(sig_builder.Build(), function, stack_slot_dst, stack_slot_src); ZeroCheck32(static_cast<wasm::TrapReason>(trap_zero), call, position); TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position); const Operator* load_op = jsgraph()->machine()->Load(result_type); Node* load = graph()->NewNode(load_op, stack_slot_dst, jsgraph()->Int32Constant(0), *effect_, *control_); *effect_ = load; return load; } template <typename... Args> Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function, Args... args) { DCHECK_LE(sig->return_count(), 1); DCHECK_EQ(sizeof...(args), sig->parameter_count()); Node* const call_args[] = {function, args..., *effect_, *control_}; CallDescriptor* desc = Linkage::GetSimplifiedCDescriptor(jsgraph()->zone(), sig); const Operator* op = jsgraph()->common()->Call(desc); Node* call = graph()->NewNode(op, arraysize(call_args), call_args); *effect_ = call; return call; } Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args, Node*** rets, wasm::WasmCodePosition position) { DCHECK_NOT_NULL(wasm_context_); SetNeedsStackCheck(); const size_t params = sig->parameter_count(); const size_t extra = 3; // wasm_context, effect, and control. const size_t count = 1 + params + extra; // Reallocate the buffer to make space for extra inputs. args = Realloc(args, 1 + params, count); // Make room for the wasm_context parameter at index 1, just after code. memmove(&args[2], &args[1], params * sizeof(Node*)); args[1] = wasm_context_.get(); // Add effect and control inputs. args[params + 2] = *effect_; args[params + 3] = *control_; CallDescriptor* descriptor = GetWasmCallDescriptor(jsgraph()->zone(), sig); const Operator* op = jsgraph()->common()->Call(descriptor); Node* call = graph()->NewNode(op, static_cast<int>(count), args); SetSourcePosition(call, position); *effect_ = call; size_t ret_count = sig->return_count(); if (ret_count == 0) return call; // No return value. *rets = Buffer(ret_count); if (ret_count == 1) { // Only a single return value. (*rets)[0] = call; } else { // Create projections for all return values. for (size_t i = 0; i < ret_count; i++) { (*rets)[i] = graph()->NewNode(jsgraph()->common()->Projection(i), call, graph()->start()); } } return call; } Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets, wasm::WasmCodePosition position) { DCHECK_NULL(args[0]); wasm::FunctionSig* sig = env_->module->functions[index].sig; if (FLAG_wasm_jit_to_native) { // Just encode the function index. This will be patched at instantiation. Address code = reinterpret_cast<Address>(index); args[0] = jsgraph()->RelocatableIntPtrConstant( reinterpret_cast<intptr_t>(code), RelocInfo::WASM_CALL); } else { // Add code object as constant. Handle<Code> code = index < env_->function_code.size() ? env_->function_code[index] : env_->default_function_code; DCHECK(!code.is_null()); args[0] = HeapConstant(code); } return BuildWasmCall(sig, args, rets, position); } Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args, Node*** rets, wasm::WasmCodePosition position) { DCHECK_NOT_NULL(args[0]); DCHECK_NOT_NULL(env_); // Assume only one table for now. uint32_t table_index = 0; wasm::FunctionSig* sig = env_->module->signatures[sig_index]; EnsureFunctionTableNodes(); MachineOperatorBuilder* machine = jsgraph()->machine(); Node* key = args[0]; // Bounds check against the table size. Node* size = function_tables_[table_index].size; Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, size); TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position); // Mask the key to prevent SSCA. if (untrusted_code_mitigations_) { // mask = ((key - size) & ~key) >> 31 Node* neg_key = graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1)); Node* masked_diff = graph()->NewNode( machine->Word32And(), graph()->NewNode(machine->Int32Sub(), key, size), neg_key); Node* mask = graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31)); key = graph()->NewNode(machine->Word32And(), key, mask); } Node* table_address = function_tables_[table_index].table_addr; Node* table = graph()->NewNode( jsgraph()->machine()->Load(MachineType::AnyTagged()), table_address, jsgraph()->IntPtrConstant(0), *effect_, *control_); // Load signature from the table and check. // The table is a FixedArray; signatures are encoded as SMIs. // [sig1, code1, sig2, code2, sig3, code3, ...] static_assert(compiler::kFunctionTableEntrySize == 2, "consistency"); static_assert(compiler::kFunctionTableSignatureOffset == 0, "consistency"); static_assert(compiler::kFunctionTableCodeOffset == 1, "consistency"); ElementAccess access = AccessBuilder::ForFixedArrayElement(); const int fixed_offset = access.header_size - access.tag(); Node* key_offset = graph()->NewNode(machine->Word32Shl(), key, Int32Constant(kPointerSizeLog2 + 1)); Node* load_sig = graph()->NewNode(machine->Load(MachineType::AnyTagged()), table, graph()->NewNode(machine->Int32Add(), key_offset, Int32Constant(fixed_offset)), *effect_, *control_); int32_t canonical_sig_num = env_->module->signature_ids[sig_index]; CHECK_GE(canonical_sig_num, 0); Node* sig_match = graph()->NewNode(machine->WordEqual(), load_sig, jsgraph()->SmiConstant(canonical_sig_num)); TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position); // Load code object from the table. It is held by a Foreign. Node* entry = graph()->NewNode( machine->Load(MachineType::AnyTagged()), table, graph()->NewNode(machine->Int32Add(), key_offset, Uint32Constant(fixed_offset + kPointerSize)), *effect_, *control_); if (FLAG_wasm_jit_to_native) { Node* address = graph()->NewNode( machine->Load(MachineType::Pointer()), entry, Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag), *effect_, *control_); args[0] = address; } else { args[0] = entry; } return BuildWasmCall(sig, args, rets, position); } Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) { // Implement Rol by Ror since TurboFan does not have Rol opcode. // TODO(weiliang): support Word32Rol opcode in TurboFan. Int32Matcher m(right); if (m.HasValue()) { return Binop(wasm::kExprI32Ror, left, jsgraph()->Int32Constant(32 - m.Value())); } else { return Binop(wasm::kExprI32Ror, left, Binop(wasm::kExprI32Sub, jsgraph()->Int32Constant(32), right)); } } Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) { // Implement Rol by Ror since TurboFan does not have Rol opcode. // TODO(weiliang): support Word64Rol opcode in TurboFan. Int64Matcher m(right); if (m.HasValue()) { return Binop(wasm::kExprI64Ror, left, jsgraph()->Int64Constant(64 - m.Value())); } else { return Binop(wasm::kExprI64Ror, left, Binop(wasm::kExprI64Sub, jsgraph()->Int64Constant(64), right)); } } Node* WasmGraphBuilder::Invert(Node* node) { return Unop(wasm::kExprI32Eqz, node); } Node* WasmGraphBuilder::BuildChangeInt32ToTagged(Node* value) { MachineOperatorBuilder* machine = jsgraph()->machine(); CommonOperatorBuilder* common = jsgraph()->common(); if (machine->Is64()) { return BuildChangeInt32ToSmi(value); } Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value, graph()->start()); Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start()); Node* branch = graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, graph()->start()); Node* if_true = graph()->NewNode(common->IfTrue(), branch); Node* vtrue = BuildAllocateHeapNumberWithValue( graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true); Node* if_false = graph()->NewNode(common->IfFalse(), branch); Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false); Node* merge = graph()->NewNode(common->Merge(2), if_true, if_false); Node* phi = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2), vtrue, vfalse, merge); return phi; } Node* WasmGraphBuilder::BuildChangeFloat64ToTagged(Node* value) { MachineOperatorBuilder* machine = jsgraph()->machine(); CommonOperatorBuilder* common = jsgraph()->common(); Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value); Node* check_same = graph()->NewNode( machine->Float64Equal(), value, graph()->NewNode(machine->ChangeInt32ToFloat64(), value32)); Node* branch_same = graph()->NewNode(common->Branch(), check_same, graph()->start()); Node* if_smi = graph()->NewNode(common->IfTrue(), branch_same); Node* vsmi; Node* if_box = graph()->NewNode(common->IfFalse(), branch_same); Node* vbox; // We only need to check for -0 if the {value} can potentially contain -0. Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32, jsgraph()->Int32Constant(0)); Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse), check_zero, if_smi); Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero); Node* if_notzero = graph()->NewNode(common->IfFalse(), branch_zero); // In case of 0, we need to check the high bits for the IEEE -0 pattern. Node* check_negative = graph()->NewNode( machine->Int32LessThan(), graph()->NewNode(machine->Float64ExtractHighWord32(), value), jsgraph()->Int32Constant(0)); Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse), check_negative, if_zero); Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative); Node* if_notnegative = graph()->NewNode(common->IfFalse(), branch_negative); // We need to create a box for negative 0. if_smi = graph()->NewNode(common->Merge(2), if_notzero, if_notnegative); if_box = graph()->NewNode(common->Merge(2), if_box, if_negative); // On 64-bit machines we can just wrap the 32-bit integer in a smi, for 32-bit // machines we need to deal with potential overflow and fallback to boxing. if (machine->Is64()) { vsmi = BuildChangeInt32ToSmi(value32); } else { Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32, value32, if_smi); Node* check_ovf = graph()->NewNode(common->Projection(1), smi_tag, if_smi); Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse), check_ovf, if_smi); Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf); if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box); if_smi = graph()->NewNode(common->IfFalse(), branch_ovf); vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi); } // Allocate the box for the {value}. vbox = BuildAllocateHeapNumberWithValue(value, if_box); Node* control = graph()->NewNode(common->Merge(2), if_smi, if_box); value = graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2), vsmi, vbox, control); return value; } Node* WasmGraphBuilder::ToJS(Node* node, wasm::ValueType type) { switch (type) { case wasm::kWasmI32: return BuildChangeInt32ToTagged(node); case wasm::kWasmS128: case wasm::kWasmI64: UNREACHABLE(); case wasm::kWasmF32: node = graph()->NewNode(jsgraph()->machine()->ChangeFloat32ToFloat64(), node); return BuildChangeFloat64ToTagged(node); case wasm::kWasmF64: return BuildChangeFloat64ToTagged(node); case wasm::kWasmStmt: return jsgraph()->UndefinedConstant(); default: UNREACHABLE(); } } Node* WasmGraphBuilder::BuildJavaScriptToNumber(Node* node, Node* js_context) { Callable callable = Builtins::CallableFor(jsgraph()->isolate(), Builtins::kToNumber); CallDescriptor* desc = Linkage::GetStubCallDescriptor( jsgraph()->isolate(), jsgraph()->zone(), callable.descriptor(), 0, CallDescriptor::kNoFlags, Operator::kNoProperties); Node* stub_code = jsgraph()->HeapConstant(callable.code()); Node* result = graph()->NewNode(jsgraph()->common()->Call(desc), stub_code, node, js_context, *effect_, *control_); SetSourcePosition(result, 1); *effect_ = result; return result; } bool CanCover(Node* value, IrOpcode::Value opcode) { if (value->opcode() != opcode) return false; bool first = true; for (Edge const edge : value->use_edges()) { if (NodeProperties::IsControlEdge(edge)) continue; if (NodeProperties::IsEffectEdge(edge)) continue; DCHECK(NodeProperties::IsValueEdge(edge)); if (!first) return false; first = false; } return true; } Node* WasmGraphBuilder::BuildChangeTaggedToFloat64(Node* value) { MachineOperatorBuilder* machine = jsgraph()->machine(); CommonOperatorBuilder* common = jsgraph()->common(); Node* check = BuildTestNotSmi(value); Node* branch = graph()->NewNode(common->Branch(BranchHint::kFalse), check, graph()->start()); Node* if_not_smi = graph()->NewNode(common->IfTrue(), branch); Node* vnot_smi; Node* check_undefined = graph()->NewNode(machine->WordEqual(), value, jsgraph()->UndefinedConstant()); Node* branch_undefined = graph()->NewNode(common->Branch(BranchHint::kFalse), check_undefined, if_not_smi); Node* if_undefined = graph()->NewNode(common->IfTrue(), branch_undefined); Node* vundefined = jsgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN()); Node* if_not_undefined = graph()->NewNode(common->IfFalse(), branch_undefined); Node* vheap_number = BuildLoadHeapNumberValue(value, if_not_undefined); if_not_smi = graph()->NewNode(common->Merge(2), if_undefined, if_not_undefined); vnot_smi = graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2), vundefined, vheap_number, if_not_smi); Node* if_smi = graph()->NewNode(common->IfFalse(), branch); Node* vfrom_smi = BuildChangeSmiToFloat64(value); Node* merge = graph()->NewNode(common->Merge(2), if_not_smi, if_smi); Node* phi = graph()->NewNode(common->Phi(MachineRepresentation::kFloat64, 2), vnot_smi, vfrom_smi, merge); return phi; } Node* WasmGraphBuilder::FromJS(Node* node, Node* js_context, wasm::ValueType type) { DCHECK_NE(wasm::kWasmStmt, type); // Do a JavaScript ToNumber. Node* num = BuildJavaScriptToNumber(node, js_context); // Change representation. SimplifiedOperatorBuilder simplified(jsgraph()->zone()); num = BuildChangeTaggedToFloat64(num); switch (type) { case wasm::kWasmI32: { num = graph()->NewNode(jsgraph()->machine()->TruncateFloat64ToWord32(), num); break; } case wasm::kWasmS128: case wasm::kWasmI64: UNREACHABLE(); case wasm::kWasmF32: num = graph()->NewNode(jsgraph()->machine()->TruncateFloat64ToFloat32(), num); break; case wasm::kWasmF64: break; default: UNREACHABLE(); } return num; } Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) { if (jsgraph()->machine()->Is64()) { value = graph()->NewNode(jsgraph()->machine()->ChangeInt32ToInt64(), value); } return graph()->NewNode(jsgraph()->machine()->WordShl(), value, BuildSmiShiftBitsConstant()); } Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) { value = graph()->NewNode(jsgraph()->machine()->WordSar(), value, BuildSmiShiftBitsConstant()); if (jsgraph()->machine()->Is64()) { value = graph()->NewNode(jsgraph()->machine()->TruncateInt64ToInt32(), value); } return value; } Node* WasmGraphBuilder::BuildChangeUint32ToSmi(Node* value) { return graph()->NewNode(jsgraph()->machine()->WordShl(), Uint32ToUintptr(value), BuildSmiShiftBitsConstant()); } Node* WasmGraphBuilder::BuildChangeSmiToFloat64(Node* value) { return graph()->NewNode(jsgraph()->machine()->ChangeInt32ToFloat64(), BuildChangeSmiToInt32(value)); } Node* WasmGraphBuilder::BuildTestNotSmi(Node* value) { STATIC_ASSERT(kSmiTag == 0); STATIC_ASSERT(kSmiTagMask == 1); return graph()->NewNode(jsgraph()->machine()->WordAnd(), value, jsgraph()->IntPtrConstant(kSmiTagMask)); } Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() { return jsgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize); } Node* WasmGraphBuilder::BuildAllocateHeapNumberWithValue(Node* value, Node* control) { MachineOperatorBuilder* machine = jsgraph()->machine(); CommonOperatorBuilder* common = jsgraph()->common(); // The AllocateHeapNumber builtin does not use the js_context, so we can // safely pass in Smi zero here. Callable callable = Builtins::CallableFor(jsgraph()->isolate(), Builtins::kAllocateHeapNumber); Node* target = jsgraph()->HeapConstant(callable.code()); Node* js_context = jsgraph()->NoContextConstant(); Node* effect = graph()->NewNode(common->BeginRegion(RegionObservability::kNotObservable), graph()->start()); if (!allocate_heap_number_operator_.is_set()) { CallDescriptor* descriptor = Linkage::GetStubCallDescriptor( jsgraph()->isolate(), jsgraph()->zone(), callable.descriptor(), 0, CallDescriptor::kNoFlags, Operator::kNoThrow); allocate_heap_number_operator_.set(common->Call(descriptor)); } Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(), target, js_context, effect, control); Node* store = graph()->NewNode(machine->Store(StoreRepresentation( MachineRepresentation::kFloat64, kNoWriteBarrier)), heap_number, BuildHeapNumberValueIndexConstant(), value, heap_number, control); return graph()->NewNode(common->FinishRegion(), heap_number, store); } Node* WasmGraphBuilder::BuildLoadHeapNumberValue(Node* value, Node* control) { return graph()->NewNode(jsgraph()->machine()->Load(MachineType::Float64()), value, BuildHeapNumberValueIndexConstant(), graph()->start(), control); } Node* WasmGraphBuilder::BuildHeapNumberValueIndexConstant() { return jsgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag); } void WasmGraphBuilder::BuildJSToWasmWrapper(WasmCodeWrapper wasm_code, Address wasm_context_address) { const int wasm_count = static_cast<int>(sig_->parameter_count()); const int count = wasm_count + 4; // wasm_code, wasm_context, effect, and control. Node** args = Buffer(count); // Build the start and the JS parameter nodes. Node* start = Start(wasm_count + 5); *control_ = start; *effect_ = start; // Create the js_context parameter Node* js_context = graph()->NewNode( jsgraph()->common()->Parameter( Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"), graph()->start()); // Create the wasm_context node to pass as parameter. This must be a // RelocatableIntPtrConstant because JSToWasm wrappers are compiled at module // compile time and patched at instance build time. DCHECK_NULL(wasm_context_); wasm_context_ = jsgraph()->RelocatableIntPtrConstant( reinterpret_cast<uintptr_t>(wasm_context_address), RelocInfo::WASM_CONTEXT_REFERENCE); Node* wasm_code_node = nullptr; if (!wasm_code.IsCodeObject()) { const wasm::WasmCode* code = wasm_code.GetWasmCode(); Address instr_start = code == nullptr ? nullptr : code->instructions().start(); wasm_code_node = jsgraph()->RelocatableIntPtrConstant( reinterpret_cast<intptr_t>(instr_start), RelocInfo::JS_TO_WASM_CALL); } else { wasm_code_node = HeapConstant(wasm_code.GetCode()); } if (!wasm::IsJSCompatibleSignature(sig_)) { // Throw a TypeError. Use the js_context of the calling javascript function // (passed as a parameter), such that the generated code is js_context // independent. BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context, nullptr, 0); // Add a dummy call to the wasm function so that the generated wrapper // contains a reference to the wrapped wasm function. Without this reference // the wasm function could not be re-imported into another wasm module. int pos = 0; args[pos++] = wasm_code_node; args[pos++] = wasm_context_.get(); args[pos++] = *effect_; args[pos++] = *control_; // We only need a dummy call descriptor. wasm::FunctionSig::Builder dummy_sig_builder(jsgraph()->zone(), 0, 0); CallDescriptor* desc = GetWasmCallDescriptor(jsgraph()->zone(), dummy_sig_builder.Build()); *effect_ = graph()->NewNode(jsgraph()->common()->Call(desc), pos, args); Return(jsgraph()->UndefinedConstant()); return; } int pos = 0; args[pos++] = wasm_code_node; args[pos++] = wasm_context_.get(); // Convert JS parameters to wasm numbers. for (int i = 0; i < wasm_count; ++i) { Node* param = Param(i + 1); Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i)); args[pos++] = wasm_param; } // Set the ThreadInWasm flag before we do the actual call. BuildModifyThreadInWasmFlag(true); args[pos++] = *effect_; args[pos++] = *control_; // Call the wasm code. CallDescriptor* desc = GetWasmCallDescriptor(jsgraph()->zone(), sig_); Node* call = graph()->NewNode(jsgraph()->common()->Call(desc), count, args); *effect_ = call; // Clear the ThreadInWasmFlag BuildModifyThreadInWasmFlag(false); Node* retval = call; Node* jsval = ToJS( retval, sig_->return_count() == 0 ? wasm::kWasmStmt : sig_->GetReturn()); Return(jsval); } int WasmGraphBuilder::AddParameterNodes(Node** args, int pos, int param_count, wasm::FunctionSig* sig) { // Convert wasm numbers to JS values. for (int i = 0; i < param_count; ++i) { Node* param = Param(i + 1); // Start from index 1 to drop the wasm_context. args[pos++] = ToJS(param, sig->GetParam(i)); } return pos; } Node* WasmGraphBuilder::LoadImportDataAtOffset(int offset, Node* table) { offset = FixedArray::OffsetOfElementAt(offset) - kHeapObjectTag; Node* offset_node = jsgraph()->Int32Constant(offset); Node* import_data = graph()->NewNode( jsgraph()->machine()->Load(LoadRepresentation::TaggedPointer()), table, offset_node, *effect_, *control_); *effect_ = import_data; return import_data; } Node* WasmGraphBuilder::LoadNativeContext(Node* table) { // The js_imports_table is set up so that index 0 has isolate->native_context return LoadImportDataAtOffset(0, table); } int OffsetForImportData(int index, WasmGraphBuilder::ImportDataType type) { // The js_imports_table is set up so that index 0 has isolate->native_context // and for every index, 3*index+1 has the JSReceiver, 3*index+2 has function's // global proxy and 3*index+3 has function's context. return 3 * index + type; } Node* WasmGraphBuilder::LoadImportData(int index, ImportDataType type, Node* table) { return LoadImportDataAtOffset(OffsetForImportData(index, type), table); } bool WasmGraphBuilder::BuildWasmToJSWrapper( Handle<JSReceiver> target, Handle<FixedArray> global_js_imports_table, int index) { DCHECK(target->IsCallable()); int wasm_count = static_cast<int>(sig_->parameter_count()); // Build the start and the parameter nodes. Isolate* isolate = jsgraph()->isolate(); CallDescriptor* desc; Node* start = Start(wasm_count + 3); *effect_ = start; *control_ = start; // We add the target function to a table and look it up during runtime. This // ensures that if the GC kicks in, it doesn't need to patch the code for the // JS function. // js_imports_table is fixed array with global handle scope whose lifetime is // tied to the instance. // TODO(aseemgarg): explore using per-import global handle instead of a table Node* table_ptr = jsgraph()->IntPtrConstant( reinterpret_cast<intptr_t>(global_js_imports_table.location())); Node* table = graph()->NewNode( jsgraph()->machine()->Load(LoadRepresentation::TaggedPointer()), table_ptr, jsgraph()->IntPtrConstant(0), *effect_, *control_); *effect_ = table; if (!wasm::IsJSCompatibleSignature(sig_)) { // Throw a TypeError. Node* native_context = LoadNativeContext(table); BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, native_context, nullptr, 0); // We don't need to return a value here, as the runtime call will not return // anyway (the c entry stub will trigger stack unwinding). ReturnVoid(); return false; } Node** args = Buffer(wasm_count + 7); Node* call = nullptr; BuildModifyThreadInWasmFlag(false); if (target->IsJSFunction()) { Handle<JSFunction> function = Handle<JSFunction>::cast(target); if (function->shared()->internal_formal_parameter_count() == wasm_count) { int pos = 0; args[pos++] = LoadImportData(index, kFunction, table); // target callable. // Receiver. if (is_sloppy(function->shared()->language_mode()) && !function->shared()->native()) { args[pos++] = LoadImportData(index, kGlobalProxy, table); } else { args[pos++] = jsgraph()->Constant( handle(isolate->heap()->undefined_value(), isolate)); } desc = Linkage::GetJSCallDescriptor( graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags); // Convert wasm numbers to JS values. pos = AddParameterNodes(args, pos, wasm_count, sig_); args[pos++] = jsgraph()->UndefinedConstant(); // new target args[pos++] = jsgraph()->Int32Constant(wasm_count); // argument count args[pos++] = LoadImportData(index, kFunctionContext, table); args[pos++] = *effect_; args[pos++] = *control_; call = graph()->NewNode(jsgraph()->common()->Call(desc), pos, args); } } // We cannot call the target directly, we have to use the Call builtin. Node* native_context = nullptr; if (!call) { int pos = 0; Callable callable = CodeFactory::Call(isolate); args[pos++] = jsgraph()->HeapConstant(callable.code()); args[pos++] = LoadImportData(index, kFunction, table); // target callable. args[pos++] = jsgraph()->Int32Constant(wasm_count); // argument count args[pos++] = jsgraph()->Constant( handle(isolate->heap()->undefined_value(), isolate)); // receiver desc = Linkage::GetStubCallDescriptor(isolate, graph()->zone(), callable.descriptor(), wasm_count + 1, CallDescriptor::kNoFlags); // Convert wasm numbers to JS values. pos = AddParameterNodes(args, pos, wasm_count, sig_); // The native_context is sufficient here, because all kind of callables // which depend on the context provide their own context. The context here // is only needed if the target is a constructor to throw a TypeError, if // the target is a native function, or if the target is a callable JSObject, // which can only be constructed by the runtime. native_context = LoadNativeContext(table); args[pos++] = native_context; args[pos++] = *effect_; args[pos++] = *control_; call = graph()->NewNode(jsgraph()->common()->Call(desc), pos, args); } *effect_ = call; SetSourcePosition(call, 0); BuildModifyThreadInWasmFlag(true); // Convert the return value back. Node* val = sig_->return_count() == 0 ? jsgraph()->Int32Constant(0) : FromJS(call, native_context != nullptr ? native_context : LoadNativeContext(table), sig_->GetReturn()); Return(val); return true; } namespace { bool HasInt64ParamOrReturn(wasm::FunctionSig* sig) { for (auto type : sig->all()) { if (type == wasm::kWasmI64) return true; } return false; } } // namespace void WasmGraphBuilder::BuildWasmToWasmWrapper(WasmCodeWrapper wasm_code, Address new_context_address) { int wasm_count = static_cast<int>(sig_->parameter_count()); int count = wasm_count + 4; // wasm_code, wasm_context, effect, and control. Node** args = Buffer(count); // Build the start node. Node* start = Start(count + 1); *control_ = start; *effect_ = start; int pos = 0; // Add the wasm code target. if (!wasm_code.IsCodeObject()) { const wasm::WasmCode* code = wasm_code.GetWasmCode(); Address instr_start = code == nullptr ? nullptr : code->instructions().start(); args[pos++] = jsgraph()->RelocatableIntPtrConstant( reinterpret_cast<intptr_t>(instr_start), RelocInfo::JS_TO_WASM_CALL); } else { args[pos++] = jsgraph()->HeapConstant(wasm_code.GetCode()); } // Add the wasm_context of the other instance. args[pos++] = jsgraph()->IntPtrConstant( reinterpret_cast<uintptr_t>(new_context_address)); // Add the parameters starting from index 1 since the parameter with index 0 // is the old wasm_context. for (int i = 0; i < wasm_count; ++i) { args[pos++] = Param(i + 1); } args[pos++] = *effect_; args[pos++] = *control_; // Tail-call the wasm code. CallDescriptor* desc = GetWasmCallDescriptor(jsgraph()->zone(), sig_); Node* tail_call = graph()->NewNode(jsgraph()->common()->TailCall(desc), count, args); MergeControlToEnd(jsgraph(), tail_call); } void WasmGraphBuilder::BuildWasmInterpreterEntry(uint32_t func_index) { int param_count = static_cast<int>(sig_->parameter_count()); // Build the start and the parameter nodes. Node* start = Start(param_count + 3); *effect_ = start; *control_ = start; // Compute size for the argument buffer. int args_size_bytes = 0; for (wasm::ValueType type : sig_->parameters()) { args_size_bytes += 1 << ElementSizeLog2Of(type); } // The return value is also passed via this buffer: DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count()); // TODO(wasm): Handle multi-value returns. DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns); int return_size_bytes = sig_->return_count() == 0 ? 0 : 1 << ElementSizeLog2Of(sig_->GetReturn()); // Get a stack slot for the arguments. Node* arg_buffer = args_size_bytes == 0 && return_size_bytes == 0 ? jsgraph()->IntPtrConstant(0) : graph()->NewNode(jsgraph()->machine()->StackSlot( std::max(args_size_bytes, return_size_bytes), 8)); // Now store all our arguments to the buffer. int offset = 0; for (int i = 0; i < param_count; ++i) { wasm::ValueType type = sig_->GetParam(i); // Start from the parameter with index 1 to drop the wasm_context. *effect_ = graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer, Int32Constant(offset), Param(i + 1), *effect_, *control_); offset += 1 << ElementSizeLog2Of(type); } DCHECK_EQ(args_size_bytes, offset); // We are passing the raw arg_buffer here. To the GC and other parts, it looks // like a Smi (lowest bit not set). In the runtime function however, don't // call Smi::value on it, but just cast it to a byte pointer. Node* parameters[] = { jsgraph()->SmiConstant(func_index), // function index arg_buffer, // argument buffer }; BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters, arraysize(parameters)); // Read back the return value. if (sig_->return_count() == 0) { Return(Int32Constant(0)); } else { // TODO(wasm): Implement multi-return. DCHECK_EQ(1, sig_->return_count()); MachineType load_rep = wasm::WasmOpcodes::MachineTypeFor(sig_->GetReturn()); Node* val = graph()->NewNode(jsgraph()->machine()->Load(load_rep), arg_buffer, Int32Constant(0), *effect_, *control_); Return(val); } if (HasInt64ParamOrReturn(sig_)) LowerInt64(); } void WasmGraphBuilder::BuildCWasmEntry(Address wasm_context_address) { // Build the start and the JS parameter nodes. Node* start = Start(CWasmEntryParameters::kNumParameters + 5); *control_ = start; *effect_ = start; // Create the wasm_context node to pass as parameter. DCHECK_NULL(wasm_context_); wasm_context_ = jsgraph()->IntPtrConstant( reinterpret_cast<uintptr_t>(wasm_context_address)); // Create parameter nodes (offset by 1 for the receiver parameter). Node* code_obj = nullptr; if (FLAG_wasm_jit_to_native) { Node* foreign_code_obj = Param(CWasmEntryParameters::kCodeObject + 1); MachineOperatorBuilder* machine = jsgraph()->machine(); code_obj = graph()->NewNode( machine->Load(MachineType::Pointer()), foreign_code_obj, Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag), *effect_, *control_); } else { code_obj = Param(CWasmEntryParameters::kCodeObject + 1); } Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1); int wasm_arg_count = static_cast<int>(sig_->parameter_count()); int arg_count = wasm_arg_count + 4; // code, wasm_context, control, effect Node** args = Buffer(arg_count); int pos = 0; args[pos++] = code_obj; args[pos++] = wasm_context_.get(); int offset = 0; for (wasm::ValueType type : sig_->parameters()) { Node* arg_load = graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer, Int32Constant(offset), *effect_, *control_); *effect_ = arg_load; args[pos++] = arg_load; offset += 1 << ElementSizeLog2Of(type); } args[pos++] = *effect_; args[pos++] = *control_; DCHECK_EQ(arg_count, pos); // Call the wasm code. CallDescriptor* desc = GetWasmCallDescriptor(jsgraph()->zone(), sig_); Node* call = graph()->NewNode(jsgraph()->common()->Call(desc), arg_count, args); *effect_ = call; // Store the return value. DCHECK_GE(1, sig_->return_count()); if (sig_->return_count() == 1) { StoreRepresentation store_rep(sig_->GetReturn(), kNoWriteBarrier); Node* store = graph()->NewNode(jsgraph()->machine()->Store(store_rep), arg_buffer, Int32Constant(0), call, *effect_, *control_); *effect_ = store; } Return(jsgraph()->SmiConstant(0)); if (jsgraph()->machine()->Is32() && HasInt64ParamOrReturn(sig_)) { MachineRepresentation sig_reps[] = { MachineRepresentation::kWord32, // return value MachineRepresentation::kTagged, // receiver MachineRepresentation::kTagged, // arg0 (code) MachineRepresentation::kTagged // arg1 (buffer) }; wasm::FunctionSig c_entry_sig(1, 2, sig_reps); Int64Lowering r(jsgraph()->graph(), jsgraph()->machine(), jsgraph()->common(), jsgraph()->zone(), &c_entry_sig); r.LowerGraph(); } } void WasmGraphBuilder::InitContextCache(WasmContextCacheNodes* context_cache) { DCHECK_NOT_NULL(wasm_context_); DCHECK_NOT_NULL(*control_); DCHECK_NOT_NULL(*effect_); // Load the memory start. Node* mem_start = graph()->NewNode( jsgraph()->machine()->Load(MachineType::UintPtr()), wasm_context_.get(), jsgraph()->Int32Constant( static_cast<int32_t>(offsetof(WasmContext, mem_start))), *effect_, *control_); *effect_ = mem_start; context_cache->mem_start = mem_start; // Load the memory size. Node* mem_size = graph()->NewNode( jsgraph()->machine()->Load(MachineType::Uint32()), wasm_context_.get(), jsgraph()->Int32Constant( static_cast<int32_t>(offsetof(WasmContext, mem_size))), *effect_, *control_); *effect_ = mem_size; context_cache->mem_size = mem_size; if (untrusted_code_mitigations_) { // Load the memory mask. Node* mem_mask = graph()->NewNode( jsgraph()->machine()->Load(MachineType::Uint32()), wasm_context_.get(), jsgraph()->Int32Constant( static_cast<int32_t>(offsetof(WasmContext, mem_mask))), *effect_, *control_); *effect_ = mem_mask; context_cache->mem_mask = mem_mask; } else { // Explicitly set to nullptr to ensure a SEGV when we try to use it. context_cache->mem_mask = nullptr; } } void WasmGraphBuilder::PrepareContextCacheForLoop( WasmContextCacheNodes* context_cache, Node* control) { #define INTRODUCE_PHI(field, rep) \ context_cache->field = Phi(rep, 1, &context_cache->field, control); INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32); if (untrusted_code_mitigations_) { INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32); } #undef INTRODUCE_PHI } void WasmGraphBuilder::NewContextCacheMerge(WasmContextCacheNodes* to, WasmContextCacheNodes* from, Node* merge) { #define INTRODUCE_PHI(field, rep) \ if (to->field != from->field) { \ Node* vals[] = {to->field, from->field}; \ to->field = Phi(rep, 2, vals, merge); \ } INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32); if (untrusted_code_mitigations_) { INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32); } #undef INTRODUCE_PHI } void WasmGraphBuilder::MergeContextCacheInto(WasmContextCacheNodes* to, WasmContextCacheNodes* from, Node* merge) { to->mem_size = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge, to->mem_size, from->mem_size); to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(), merge, to->mem_start, from->mem_start); if (untrusted_code_mitigations_) { to->mem_mask = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge, to->mem_mask, from->mem_mask); } } Node* WasmGraphBuilder::CreateOrMergeIntoPhi(wasm::ValueType type, Node* merge, Node* tnode, Node* fnode) { if (IsPhiWithMerge(tnode, merge)) { AppendToPhi(tnode, fnode); } else if (tnode != fnode) { uint32_t count = merge->InputCount(); Node** vals = Buffer(count); for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode; vals[count - 1] = fnode; return Phi(type, count, vals, merge); } return tnode; } Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode, Node* fnode) { if (IsPhiWithMerge(tnode, merge)) { AppendToPhi(tnode, fnode); } else if (tnode != fnode) { uint32_t count = merge->InputCount(); Node** effects = Buffer(count); for (uint32_t j = 0; j < count - 1; j++) { effects[j] = tnode; } effects[count - 1] = fnode; tnode = EffectPhi(count, effects, merge); } return tnode; } void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type, uint32_t offset, Node** base_node, Node** offset_node) { DCHECK_NOT_NULL(wasm_context_); if (globals_start_ == nullptr) { // Load globals_start from the WasmContext at runtime. // TODO(wasm): we currently generate only one load of the {globals_start} // start per graph, which means it can be placed anywhere by the scheduler. // This is legal because the globals_start should never change. // However, in some cases (e.g. if the WasmContext is already in a // register), it is slightly more efficient to reload this value from the // WasmContext. Since this depends on register allocation, it is not // possible to express in the graph, and would essentially constitute a // "mem2reg" optimization in TurboFan. globals_start_ = graph()->NewNode( jsgraph()->machine()->Load(MachineType::UintPtr()), wasm_context_.get(), jsgraph()->Int32Constant( static_cast<int32_t>(offsetof(WasmContext, globals_start))), graph()->start(), graph()->start()); } *base_node = globals_start_.get(); *offset_node = jsgraph()->Int32Constant(offset); if (mem_type == MachineType::Simd128() && offset != 0) { // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken. *base_node = graph()->NewNode(kPointerSize == 4 ? jsgraph()->machine()->Int32Add() : jsgraph()->machine()->Int64Add(), *base_node, *offset_node); *offset_node = jsgraph()->Int32Constant(0); } } Node* WasmGraphBuilder::MemBuffer(uint32_t offset) { DCHECK_NOT_NULL(context_cache_); Node* mem_start = context_cache_->mem_start; DCHECK_NOT_NULL(mem_start); if (offset == 0) return mem_start; return graph()->NewNode(jsgraph()->machine()->IntAdd(), mem_start, jsgraph()->IntPtrConstant(offset)); } Node* WasmGraphBuilder::CurrentMemoryPages() { // CurrentMemoryPages can not be called from asm.js. DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin()); DCHECK_NOT_NULL(context_cache_); Node* mem_size = context_cache_->mem_size; DCHECK_NOT_NULL(mem_size); if (jsgraph()->machine()->Is64()) { mem_size = graph()->NewNode(jsgraph()->machine()->TruncateInt64ToInt32(), mem_size); } return graph()->NewNode( jsgraph()->machine()->Word32Shr(), mem_size, jsgraph()->Int32Constant(WhichPowerOf2(wasm::kWasmPageSize))); } void WasmGraphBuilder::EnsureFunctionTableNodes() { if (function_tables_.size() > 0) return; size_t tables_size = env_->function_tables.size(); for (size_t i = 0; i < tables_size; ++i) { wasm::GlobalHandleAddress function_handle_address = env_->function_tables[i]; Node* table_addr = jsgraph()->RelocatableIntPtrConstant( reinterpret_cast<intptr_t>(function_handle_address), RelocInfo::WASM_GLOBAL_HANDLE); uint32_t table_size = env_->module->function_tables[i].initial_size; Node* size = jsgraph()->RelocatableInt32Constant( static_cast<uint32_t>(table_size), RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE); function_tables_.push_back({table_addr, size}); } } Node* WasmGraphBuilder::BuildModifyThreadInWasmFlag(bool new_value) { // TODO(eholk): generate code to modify the thread-local storage directly, // rather than calling the runtime. if (!use_trap_handler()) { return *control_; } // Using two functions instead of taking the new value as a parameter saves // one instruction on each call to set up the parameter. ExternalReference ref = new_value ? ExternalReference::wasm_set_thread_in_wasm_flag( jsgraph()->isolate()) : ExternalReference::wasm_clear_thread_in_wasm_flag( jsgraph()->isolate()); MachineSignature::Builder sig_builder(jsgraph()->zone(), 0, 0); return BuildCCall( sig_builder.Build(), graph()->NewNode(jsgraph()->common()->ExternalConstant(ref))); } // Only call this function for code which is not reused across instantiations, // as we do not patch the embedded js_context. Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f, Node* js_context, Node** parameters, int parameter_count) { const Runtime::Function* fun = Runtime::FunctionForId(f); CallDescriptor* desc = Linkage::GetRuntimeCallDescriptor( jsgraph()->zone(), f, fun->nargs, Operator::kNoProperties, CallDescriptor::kNoFlags); // CEntryStubConstant nodes have to be created and cached in the main // thread. At the moment this is only done for CEntryStubConstant(1). DCHECK_EQ(1, fun->result_size); // At the moment we only allow 4 parameters. If more parameters are needed, // increase this constant accordingly. static const int kMaxParams = 4; DCHECK_GE(kMaxParams, parameter_count); Node* inputs[kMaxParams + 6]; int count = 0; inputs[count++] = centry_stub_node_; for (int i = 0; i < parameter_count; i++) { inputs[count++] = parameters[i]; } inputs[count++] = jsgraph()->ExternalConstant( ExternalReference(f, jsgraph()->isolate())); // ref inputs[count++] = jsgraph()->Int32Constant(fun->nargs); // arity inputs[count++] = js_context; // js_context inputs[count++] = *effect_; inputs[count++] = *control_; Node* node = jsgraph()->graph()->NewNode(jsgraph()->common()->Call(desc), count, inputs); *effect_ = node; return node; } Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f, Node** parameters, int parameter_count) { return BuildCallToRuntimeWithContext(f, jsgraph()->NoContextConstant(), parameters, parameter_count); } Node* WasmGraphBuilder::GetGlobal(uint32_t index) { MachineType mem_type = wasm::WasmOpcodes::MachineTypeFor(env_->module->globals[index].type); Node* base = nullptr; Node* offset = nullptr; GetGlobalBaseAndOffset(mem_type, env_->module->globals[index].offset, &base, &offset); Node* node = graph()->NewNode(jsgraph()->machine()->Load(mem_type), base, offset, *effect_, *control_); *effect_ = node; return node; } Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) { MachineType mem_type = wasm::WasmOpcodes::MachineTypeFor(env_->module->globals[index].type); Node* base = nullptr; Node* offset = nullptr; GetGlobalBaseAndOffset(mem_type, env_->module->globals[index].offset, &base, &offset); const Operator* op = jsgraph()->machine()->Store( StoreRepresentation(mem_type.representation(), kNoWriteBarrier)); Node* node = graph()->NewNode(op, base, offset, val, *effect_, *control_); *effect_ = node; return node; } Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index, uint32_t offset, wasm::WasmCodePosition position, EnforceBoundsCheck enforce_check) { if (FLAG_wasm_no_bounds_checks) return Uint32ToUintptr(index); DCHECK_NOT_NULL(context_cache_); Node* mem_size = context_cache_->mem_size; DCHECK_NOT_NULL(mem_size); auto m = jsgraph()->machine(); if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) { // Simply zero out the 32-bits on 64-bit targets and let the trap handler // do its job. return Uint32ToUintptr(index); } uint32_t min_size = env_->module->initial_pages * wasm::kWasmPageSize; uint32_t max_size = (env_->module->has_maximum_pages ? env_->module->maximum_pages : wasm::kV8MaxWasmMemoryPages) * wasm::kWasmPageSize; if (access_size > max_size || offset > max_size - access_size) { // The access will be out of bounds, even for the largest memory. TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position); return jsgraph()->IntPtrConstant(0); } DCHECK_LE(1, access_size); // This computation cannot overflow, since // {offset <= max_size - access_size <= kMaxUint32 - access_size}. // It also cannot underflow, since {access_size >= 1}. uint32_t end_offset = offset + access_size - 1; Node* end_offset_node = Int32Constant(end_offset); // The accessed memory is [index + offset, index + end_offset]. // Check that the last read byte (at {index + end_offset}) is in bounds. // 1) Check that {end_offset < mem_size}. This also ensures that we can safely // compute {effective_size} as {mem_size - end_offset)}. // {effective_size} is >= 1 if condition 1) holds. // 2) Check that {index + end_offset < mem_size} by // - computing {effective_size} as {mem_size - end_offset} and // - checking that {index < effective_size}. if (end_offset >= min_size) { // The end offset is larger than the smallest memory. // Dynamically check the end offset against the actual memory size, which // is not known at compile time. Node* cond = graph()->NewNode(jsgraph()->machine()->Uint32LessThan(), end_offset_node, mem_size); TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position); } else { // The end offset is within the bounds of the smallest memory, so only // one check is required. Check to see if the index is also a constant. Uint32Matcher match(index); if (match.HasValue()) { uint32_t index_val = match.Value(); if (index_val < min_size - end_offset) { // The input index is a constant and everything is statically within // bounds of the smallest possible memory. return Uint32ToUintptr(index); } } } // This produces a positive number, since {end_offset < min_size <= mem_size}. Node* effective_size = graph()->NewNode(jsgraph()->machine()->Int32Sub(), mem_size, end_offset_node); // Introduce the actual bounds check. Node* cond = graph()->NewNode(m->Uint32LessThan(), index, effective_size); TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position); if (untrusted_code_mitigations_) { // In the fallthrough case, condition the index with the memory mask. Node* mem_mask = context_cache_->mem_mask; DCHECK_NOT_NULL(mem_mask); index = graph()->NewNode(m->Word32And(), index, mem_mask); } return Uint32ToUintptr(index); } const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset, wasm::ValueType type) { int alignment = offset % (1 << ElementSizeLog2Of(type)); MachineType mach_type = wasm::WasmOpcodes::MachineTypeFor(type); if (alignment == 0 || jsgraph()->machine()->UnalignedLoadSupported(type)) { return jsgraph()->machine()->Load(mach_type); } return jsgraph()->machine()->UnalignedLoad(mach_type); } const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset, wasm::ValueType type) { int alignment = offset % (1 << ElementSizeLog2Of(type)); if (alignment == 0 || jsgraph()->machine()->UnalignedStoreSupported(type)) { StoreRepresentation rep(type, WriteBarrierKind::kNoWriteBarrier); return jsgraph()->machine()->Store(rep); } UnalignedStoreRepresentation rep(type); return jsgraph()->machine()->UnalignedStore(rep); } Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store, MachineRepresentation rep, Node* index, uint32_t offset, wasm::WasmCodePosition position) { int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi. Node* info = graph()->NewNode( jsgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign)); Node* address = graph()->NewNode(jsgraph()->machine()->Int32Add(), Int32Constant(offset), index); auto store = [&](int offset, MachineRepresentation rep, Node* data) { *effect_ = graph()->NewNode( jsgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)), info, jsgraph()->Int32Constant(offset), data, *effect_, *control_); }; // Store address, is_store, and mem_rep. store(offsetof(wasm::MemoryTracingInfo, address), MachineRepresentation::kWord32, address); store(offsetof(wasm::MemoryTracingInfo, is_store), MachineRepresentation::kWord8, jsgraph()->Int32Constant(is_store ? 1 : 0)); store(offsetof(wasm::MemoryTracingInfo, mem_rep), MachineRepresentation::kWord8, jsgraph()->Int32Constant(static_cast<int>(rep))); Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1); SetSourcePosition(call, position); return call; } Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype, Node* index, uint32_t offset, uint32_t alignment, wasm::WasmCodePosition position) { Node* load; // Wasm semantics throw on OOB. Introduce explicit bounds check and // conditioning when not using the trap handler. index = BoundsCheckMem(wasm::WasmOpcodes::MemSize(memtype), index, offset, position, kCanOmitBoundsCheck); if (memtype.representation() == MachineRepresentation::kWord8 || jsgraph()->machine()->UnalignedLoadSupported(memtype.representation())) { if (use_trap_handler()) { load = graph()->NewNode(jsgraph()->machine()->ProtectedLoad(memtype), MemBuffer(offset), index, *effect_, *control_); SetSourcePosition(load, position); } else { load = graph()->NewNode(jsgraph()->machine()->Load(memtype), MemBuffer(offset), index, *effect_, *control_); } } else { // TODO(eholk): Support unaligned loads with trap handlers. DCHECK(!use_trap_handler()); load = graph()->NewNode(jsgraph()->machine()->UnalignedLoad(memtype), MemBuffer(offset), index, *effect_, *control_); } *effect_ = load; #if defined(V8_TARGET_BIG_ENDIAN) load = BuildChangeEndiannessLoad(load, memtype, type); #endif if (type == wasm::kWasmI64 && ElementSizeLog2Of(memtype.representation()) < 3) { // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes. if (memtype.IsSigned()) { // sign extend load = graph()->NewNode(jsgraph()->machine()->ChangeInt32ToInt64(), load); } else { // zero extend load = graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), load); } } if (FLAG_wasm_trace_memory) { TraceMemoryOperation(false, memtype.representation(), index, offset, position); } return load; } Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index, uint32_t offset, uint32_t alignment, Node* val, wasm::WasmCodePosition position, wasm::ValueType type) { Node* store; index = BoundsCheckMem(wasm::WasmOpcodes::MemSize(mem_rep), index, offset, position, kCanOmitBoundsCheck); #if defined(V8_TARGET_BIG_ENDIAN) val = BuildChangeEndiannessStore(val, mem_rep, type); #endif if (mem_rep == MachineRepresentation::kWord8 || jsgraph()->machine()->UnalignedStoreSupported(mem_rep)) { if (use_trap_handler()) { store = graph()->NewNode(jsgraph()->machine()->ProtectedStore(mem_rep), MemBuffer(offset), index, val, *effect_, *control_); SetSourcePosition(store, position); } else { StoreRepresentation rep(mem_rep, kNoWriteBarrier); store = graph()->NewNode(jsgraph()->machine()->Store(rep), MemBuffer(offset), index, val, *effect_, *control_); } } else { // TODO(eholk): Support unaligned stores with trap handlers. DCHECK(!use_trap_handler()); UnalignedStoreRepresentation rep(mem_rep); store = graph()->NewNode(jsgraph()->machine()->UnalignedStore(rep), MemBuffer(offset), index, val, *effect_, *control_); } *effect_ = store; if (FLAG_wasm_trace_memory) { TraceMemoryOperation(true, mem_rep, index, offset, position); } return store; } namespace { Node* GetAsmJsOOBValue(MachineRepresentation rep, JSGraph* jsgraph) { switch (rep) { case MachineRepresentation::kWord8: case MachineRepresentation::kWord16: case MachineRepresentation::kWord32: return jsgraph->Int32Constant(0); case MachineRepresentation::kWord64: return jsgraph->Int64Constant(0); case MachineRepresentation::kFloat32: return jsgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN()); case MachineRepresentation::kFloat64: return jsgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN()); default: UNREACHABLE(); } } } // namespace Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) { DCHECK_NOT_NULL(context_cache_); Node* mem_start = context_cache_->mem_start; Node* mem_size = context_cache_->mem_size; DCHECK_NOT_NULL(mem_start); DCHECK_NOT_NULL(mem_size); // Asm.js semantics are defined along the lines of typed arrays, hence OOB // reads return {undefined} coerced to the result type (0 for integers, NaN // for float and double). // Note that we check against the memory size ignoring the size of the // stored value, which is conservative if misaligned. Technically, asm.js // should never have misaligned accesses. Diamond bounds_check( graph(), jsgraph()->common(), graph()->NewNode(jsgraph()->machine()->Uint32LessThan(), index, mem_size), BranchHint::kTrue); bounds_check.Chain(*control_); if (untrusted_code_mitigations_) { // Condition the index with the memory mask. Node* mem_mask = context_cache_->mem_mask; DCHECK_NOT_NULL(mem_mask); index = graph()->NewNode(jsgraph()->machine()->Word32And(), index, mem_mask); } index = Uint32ToUintptr(index); Node* load = graph()->NewNode(jsgraph()->machine()->Load(type), mem_start, index, *effect_, bounds_check.if_true); Node* value_phi = bounds_check.Phi(type.representation(), load, GetAsmJsOOBValue(type.representation(), jsgraph())); Node* effect_phi = graph()->NewNode(jsgraph()->common()->EffectPhi(2), load, *effect_, bounds_check.merge); *effect_ = effect_phi; *control_ = bounds_check.merge; return value_phi; } Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) { if (jsgraph()->machine()->Is32()) return node; return graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), node); } Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index, Node* val) { DCHECK_NOT_NULL(context_cache_); Node* mem_start = context_cache_->mem_start; Node* mem_size = context_cache_->mem_size; DCHECK_NOT_NULL(mem_start); DCHECK_NOT_NULL(mem_size); // Asm.js semantics are to ignore OOB writes. // Note that we check against the memory size ignoring the size of the // stored value, which is conservative if misaligned. Technically, asm.js // should never have misaligned accesses. Diamond bounds_check( graph(), jsgraph()->common(), graph()->NewNode(jsgraph()->machine()->Uint32LessThan(), index, mem_size), BranchHint::kTrue); bounds_check.Chain(*control_); if (untrusted_code_mitigations_) { // Condition the index with the memory mask. Node* mem_mask = context_cache_->mem_mask; DCHECK_NOT_NULL(mem_mask); index = graph()->NewNode(jsgraph()->machine()->Word32And(), index, mem_mask); } index = Uint32ToUintptr(index); const Operator* store_op = jsgraph()->machine()->Store(StoreRepresentation( type.representation(), WriteBarrierKind::kNoWriteBarrier)); Node* store = graph()->NewNode(store_op, mem_start, index, val, *effect_, bounds_check.if_true); Node* effect_phi = graph()->NewNode(jsgraph()->common()->EffectPhi(2), store, *effect_, bounds_check.merge); *effect_ = effect_phi; *control_ = bounds_check.merge; return val; } void WasmGraphBuilder::PrintDebugName(Node* node) { PrintF("#%d:%s", node->id(), node->op()->mnemonic()); } Node* WasmGraphBuilder::String(const char* string) { return jsgraph()->Constant( jsgraph()->isolate()->factory()->NewStringFromAsciiChecked(string)); } Graph* WasmGraphBuilder::graph() { return jsgraph()->graph(); } void WasmGraphBuilder::LowerInt64() { if (jsgraph()->machine()->Is64()) return; Int64Lowering r(jsgraph()->graph(), jsgraph()->machine(), jsgraph()->common(), jsgraph()->zone(), sig_); r.LowerGraph(); } void WasmGraphBuilder::SimdScalarLoweringForTesting() { SimdScalarLowering(jsgraph(), sig_).LowerGraph(); } void WasmGraphBuilder::SetSourcePosition(Node* node, wasm::WasmCodePosition position) { DCHECK_NE(position, wasm::kNoCodePosition); if (source_position_table_) source_position_table_->SetSourcePosition(node, SourcePosition(position)); } Node* WasmGraphBuilder::S128Zero() { has_simd_ = true; return graph()->NewNode(jsgraph()->machine()->S128Zero()); } Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) { has_simd_ = true; switch (opcode) { case wasm::kExprF32x4Splat: return graph()->NewNode(jsgraph()->machine()->F32x4Splat(), inputs[0]); case wasm::kExprF32x4SConvertI32x4: return graph()->NewNode(jsgraph()->machine()->F32x4SConvertI32x4(), inputs[0]); case wasm::kExprF32x4UConvertI32x4: return graph()->NewNode(jsgraph()->machine()->F32x4UConvertI32x4(), inputs[0]); case wasm::kExprF32x4Abs: return graph()->NewNode(jsgraph()->machine()->F32x4Abs(), inputs[0]); case wasm::kExprF32x4Neg: return graph()->NewNode(jsgraph()->machine()->F32x4Neg(), inputs[0]); case wasm::kExprF32x4RecipApprox: return graph()->NewNode(jsgraph()->machine()->F32x4RecipApprox(), inputs[0]); case wasm::kExprF32x4RecipSqrtApprox: return graph()->NewNode(jsgraph()->machine()->F32x4RecipSqrtApprox(), inputs[0]); case wasm::kExprF32x4Add: return graph()->NewNode(jsgraph()->machine()->F32x4Add(), inputs[0], inputs[1]); case wasm::kExprF32x4AddHoriz: return graph()->NewNode(jsgraph()->machine()->F32x4AddHoriz(), inputs[0], inputs[1]); case wasm::kExprF32x4Sub: return graph()->NewNode(jsgraph()->machine()->F32x4Sub(), inputs[0], inputs[1]); case wasm::kExprF32x4Mul: return graph()->NewNode(jsgraph()->machine()->F32x4Mul(), inputs[0], inputs[1]); case wasm::kExprF32x4Min: return graph()->NewNode(jsgraph()->machine()->F32x4Min(), inputs[0], inputs[1]); case wasm::kExprF32x4Max: return graph()->NewNode(jsgraph()->machine()->F32x4Max(), inputs[0], inputs[1]); case wasm::kExprF32x4Eq: return graph()->NewNode(jsgraph()->machine()->F32x4Eq(), inputs[0], inputs[1]); case wasm::kExprF32x4Ne: return graph()->NewNode(jsgraph()->machine()->F32x4Ne(), inputs[0], inputs[1]); case wasm::kExprF32x4Lt: return graph()->NewNode(jsgraph()->machine()->F32x4Lt(), inputs[0], inputs[1]); case wasm::kExprF32x4Le: return graph()->NewNode(jsgraph()->machine()->F32x4Le(), inputs[0], inputs[1]); case wasm::kExprF32x4Gt: return graph()->NewNode(jsgraph()->machine()->F32x4Lt(), inputs[1], inputs[0]); case wasm::kExprF32x4Ge: return graph()->NewNode(jsgraph()->machine()->F32x4Le(), inputs[1], inputs[0]); case wasm::kExprI32x4Splat: return graph()->NewNode(jsgraph()->machine()->I32x4Splat(), inputs[0]); case wasm::kExprI32x4SConvertF32x4: return graph()->NewNode(jsgraph()->machine()->I32x4SConvertF32x4(), inputs[0]); case wasm::kExprI32x4UConvertF32x4: return graph()->NewNode(jsgraph()->machine()->I32x4UConvertF32x4(), inputs[0]); case wasm::kExprI32x4SConvertI16x8Low: return graph()->NewNode(jsgraph()->machine()->I32x4SConvertI16x8Low(), inputs[0]); case wasm::kExprI32x4SConvertI16x8High: return graph()->NewNode(jsgraph()->machine()->I32x4SConvertI16x8High(), inputs[0]); case wasm::kExprI32x4Neg: return graph()->NewNode(jsgraph()->machine()->I32x4Neg(), inputs[0]); case wasm::kExprI32x4Add: return graph()->NewNode(jsgraph()->machine()->I32x4Add(), inputs[0], inputs[1]); case wasm::kExprI32x4AddHoriz: return graph()->NewNode(jsgraph()->machine()->I32x4AddHoriz(), inputs[0], inputs[1]); case wasm::kExprI32x4Sub: return graph()->NewNode(jsgraph()->machine()->I32x4Sub(), inputs[0], inputs[1]); case wasm::kExprI32x4Mul: return graph()->NewNode(jsgraph()->machine()->I32x4Mul(), inputs[0], inputs[1]); case wasm::kExprI32x4MinS: return graph()->NewNode(jsgraph()->machine()->I32x4MinS(), inputs[0], inputs[1]); case wasm::kExprI32x4MaxS: return graph()->NewNode(jsgraph()->machine()->I32x4MaxS(), inputs[0], inputs[1]); case wasm::kExprI32x4Eq: return graph()->NewNode(jsgraph()->machine()->I32x4Eq(), inputs[0], inputs[1]); case wasm::kExprI32x4Ne: return graph()->NewNode(jsgraph()->machine()->I32x4Ne(), inputs[0], inputs[1]); case wasm::kExprI32x4LtS: return graph()->NewNode(jsgraph()->machine()->I32x4GtS(), inputs[1], inputs[0]); case wasm::kExprI32x4LeS: return graph()->NewNode(jsgraph()->machine()->I32x4GeS(), inputs[1], inputs[0]); case wasm::kExprI32x4GtS: return graph()->NewNode(jsgraph()->machine()->I32x4GtS(), inputs[0], inputs[1]); case wasm::kExprI32x4GeS: return graph()->NewNode(jsgraph()->machine()->I32x4GeS(), inputs[0], inputs[1]); case wasm::kExprI32x4UConvertI16x8Low: return graph()->NewNode(jsgraph()->machine()->I32x4UConvertI16x8Low(), inputs[0]); case wasm::kExprI32x4UConvertI16x8High: return graph()->NewNode(jsgraph()->machine()->I32x4UConvertI16x8High(), inputs[0]); case wasm::kExprI32x4MinU: return graph()->NewNode(jsgraph()->machine()->I32x4MinU(), inputs[0], inputs[1]); case wasm::kExprI32x4MaxU: return graph()->NewNode(jsgraph()->machine()->I32x4MaxU(), inputs[0], inputs[1]); case wasm::kExprI32x4LtU: return graph()->NewNode(jsgraph()->machine()->I32x4GtU(), inputs[1], inputs[0]); case wasm::kExprI32x4LeU: return graph()->NewNode(jsgraph()->machine()->I32x4GeU(), inputs[1], inputs[0]); case wasm::kExprI32x4GtU: return graph()->NewNode(jsgraph()->machine()->I32x4GtU(), inputs[0], inputs[1]); case wasm::kExprI32x4GeU: return graph()->NewNode(jsgraph()->machine()->I32x4GeU(), inputs[0], inputs[1]); case wasm::kExprI16x8Splat: return graph()->NewNode(jsgraph()->machine()->I16x8Splat(), inputs[0]); case wasm::kExprI16x8SConvertI8x16Low: return graph()->NewNode(jsgraph()->machine()->I16x8SConvertI8x16Low(), inputs[0]); case wasm::kExprI16x8SConvertI8x16High: return graph()->NewNode(jsgraph()->machine()->I16x8SConvertI8x16High(), inputs[0]); case wasm::kExprI16x8Neg: return graph()->NewNode(jsgraph()->machine()->I16x8Neg(), inputs[0]); case wasm::kExprI16x8SConvertI32x4: return graph()->NewNode(jsgraph()->machine()->I16x8SConvertI32x4(), inputs[0], inputs[1]); case wasm::kExprI16x8Add: return graph()->NewNode(jsgraph()->machine()->I16x8Add(), inputs[0], inputs[1]); case wasm::kExprI16x8AddSaturateS: return graph()->NewNode(jsgraph()->machine()->I16x8AddSaturateS(), inputs[0], inputs[1]); case wasm::kExprI16x8AddHoriz: return graph()->NewNode(jsgraph()->machine()->I16x8AddHoriz(), inputs[0], inputs[1]); case wasm::kExprI16x8Sub: return graph()->NewNode(jsgraph()->machine()->I16x8Sub(), inputs[0], inputs[1]); case wasm::kExprI16x8SubSaturateS: return graph()->NewNode(jsgraph()->machine()->I16x8SubSaturateS(), inputs[0], inputs[1]); case wasm::kExprI16x8Mul: return graph()->NewNode(jsgraph()->machine()->I16x8Mul(), inputs[0], inputs[1]); case wasm::kExprI16x8MinS: return graph()->NewNode(jsgraph()->machine()->I16x8MinS(), inputs[0], inputs[1]); case wasm::kExprI16x8MaxS: return graph()->NewNode(jsgraph()->machine()->I16x8MaxS(), inputs[0], inputs[1]); case wasm::kExprI16x8Eq: return graph()->NewNode(jsgraph()->machine()->I16x8Eq(), inputs[0], inputs[1]); case wasm::kExprI16x8Ne: return graph()->NewNode(jsgraph()->machine()->I16x8Ne(), inputs[0], inputs[1]); case wasm::kExprI16x8LtS: return graph()->NewNode(jsgraph()->machine()->I16x8GtS(), inputs[1], inputs[0]); case wasm::kExprI16x8LeS: return graph()->NewNode(jsgraph()->machine()->I16x8GeS(), inputs[1], inputs[0]); case wasm::kExprI16x8GtS: return graph()->NewNode(jsgraph()->machine()->I16x8GtS(), inputs[0], inputs[1]); case wasm::kExprI16x8GeS: return graph()->NewNode(jsgraph()->machine()->I16x8GeS(), inputs[0], inputs[1]); case wasm::kExprI16x8UConvertI8x16Low: return graph()->NewNode(jsgraph()->machine()->I16x8UConvertI8x16Low(), inputs[0]); case wasm::kExprI16x8UConvertI8x16High: return graph()->NewNode(jsgraph()->machine()->I16x8UConvertI8x16High(), inputs[0]); case wasm::kExprI16x8UConvertI32x4: return graph()->NewNode(jsgraph()->machine()->I16x8UConvertI32x4(), inputs[0], inputs[1]); case wasm::kExprI16x8AddSaturateU: return graph()->NewNode(jsgraph()->machine()->I16x8AddSaturateU(), inputs[0], inputs[1]); case wasm::kExprI16x8SubSaturateU: return graph()->NewNode(jsgraph()->machine()->I16x8SubSaturateU(), inputs[0], inputs[1]); case wasm::kExprI16x8MinU: return graph()->NewNode(jsgraph()->machine()->I16x8MinU(), inputs[0], inputs[1]); case wasm::kExprI16x8MaxU: return graph()->NewNode(jsgraph()->machine()->I16x8MaxU(), inputs[0], inputs[1]); case wasm::kExprI16x8LtU: return graph()->NewNode(jsgraph()->machine()->I16x8GtU(), inputs[1], inputs[0]); case wasm::kExprI16x8LeU: return graph()->NewNode(jsgraph()->machine()->I16x8GeU(), inputs[1], inputs[0]); case wasm::kExprI16x8GtU: return graph()->NewNode(jsgraph()->machine()->I16x8GtU(), inputs[0], inputs[1]); case wasm::kExprI16x8GeU: return graph()->NewNode(jsgraph()->machine()->I16x8GeU(), inputs[0], inputs[1]); case wasm::kExprI8x16Splat: return graph()->NewNode(jsgraph()->machine()->I8x16Splat(), inputs[0]); case wasm::kExprI8x16Neg: return graph()->NewNode(jsgraph()->machine()->I8x16Neg(), inputs[0]); case wasm::kExprI8x16SConvertI16x8: return graph()->NewNode(jsgraph()->machine()->I8x16SConvertI16x8(), inputs[0], inputs[1]); case wasm::kExprI8x16Add: return graph()->NewNode(jsgraph()->machine()->I8x16Add(), inputs[0], inputs[1]); case wasm::kExprI8x16AddSaturateS: return graph()->NewNode(jsgraph()->machine()->I8x16AddSaturateS(), inputs[0], inputs[1]); case wasm::kExprI8x16Sub: return graph()->NewNode(jsgraph()->machine()->I8x16Sub(), inputs[0], inputs[1]); case wasm::kExprI8x16SubSaturateS: return graph()->NewNode(jsgraph()->machine()->I8x16SubSaturateS(), inputs[0], inputs[1]); case wasm::kExprI8x16Mul: return graph()->NewNode(jsgraph()->machine()->I8x16Mul(), inputs[0], inputs[1]); case wasm::kExprI8x16MinS: return graph()->NewNode(jsgraph()->machine()->I8x16MinS(), inputs[0], inputs[1]); case wasm::kExprI8x16MaxS: return graph()->NewNode(jsgraph()->machine()->I8x16MaxS(), inputs[0], inputs[1]); case wasm::kExprI8x16Eq: return graph()->NewNode(jsgraph()->machine()->I8x16Eq(), inputs[0], inputs[1]); case wasm::kExprI8x16Ne: return graph()->NewNode(jsgraph()->machine()->I8x16Ne(), inputs[0], inputs[1]); case wasm::kExprI8x16LtS: return graph()->NewNode(jsgraph()->machine()->I8x16GtS(), inputs[1], inputs[0]); case wasm::kExprI8x16LeS: return graph()->NewNode(jsgraph()->machine()->I8x16GeS(), inputs[1], inputs[0]); case wasm::kExprI8x16GtS: return graph()->NewNode(jsgraph()->machine()->I8x16GtS(), inputs[0], inputs[1]); case wasm::kExprI8x16GeS: return graph()->NewNode(jsgraph()->machine()->I8x16GeS(), inputs[0], inputs[1]); case wasm::kExprI8x16UConvertI16x8: return graph()->NewNode(jsgraph()->machine()->I8x16UConvertI16x8(), inputs[0], inputs[1]); case wasm::kExprI8x16AddSaturateU: return graph()->NewNode(jsgraph()->machine()->I8x16AddSaturateU(), inputs[0], inputs[1]); case wasm::kExprI8x16SubSaturateU: return graph()->NewNode(jsgraph()->machine()->I8x16SubSaturateU(), inputs[0], inputs[1]); case wasm::kExprI8x16MinU: return graph()->NewNode(jsgraph()->machine()->I8x16MinU(), inputs[0], inputs[1]); case wasm::kExprI8x16MaxU: return graph()->NewNode(jsgraph()->machine()->I8x16MaxU(), inputs[0], inputs[1]); case wasm::kExprI8x16LtU: return graph()->NewNode(jsgraph()->machine()->I8x16GtU(), inputs[1], inputs[0]); case wasm::kExprI8x16LeU: return graph()->NewNode(jsgraph()->machine()->I8x16GeU(), inputs[1], inputs[0]); case wasm::kExprI8x16GtU: return graph()->NewNode(jsgraph()->machine()->I8x16GtU(), inputs[0], inputs[1]); case wasm::kExprI8x16GeU: return graph()->NewNode(jsgraph()->machine()->I8x16GeU(), inputs[0], inputs[1]); case wasm::kExprS128And: return graph()->NewNode(jsgraph()->machine()->S128And(), inputs[0], inputs[1]); case wasm::kExprS128Or: return graph()->NewNode(jsgraph()->machine()->S128Or(), inputs[0], inputs[1]); case wasm::kExprS128Xor: return graph()->NewNode(jsgraph()->machine()->S128Xor(), inputs[0], inputs[1]); case wasm::kExprS128Not: return graph()->NewNode(jsgraph()->machine()->S128Not(), inputs[0]); case wasm::kExprS128Select: return graph()->NewNode(jsgraph()->machine()->S128Select(), inputs[0], inputs[1], inputs[2]); case wasm::kExprS1x4AnyTrue: return graph()->NewNode(jsgraph()->machine()->S1x4AnyTrue(), inputs[0]); case wasm::kExprS1x4AllTrue: return graph()->NewNode(jsgraph()->machine()->S1x4AllTrue(), inputs[0]); case wasm::kExprS1x8AnyTrue: return graph()->NewNode(jsgraph()->machine()->S1x8AnyTrue(), inputs[0]); case wasm::kExprS1x8AllTrue: return graph()->NewNode(jsgraph()->machine()->S1x8AllTrue(), inputs[0]); case wasm::kExprS1x16AnyTrue: return graph()->NewNode(jsgraph()->machine()->S1x16AnyTrue(), inputs[0]); case wasm::kExprS1x16AllTrue: return graph()->NewNode(jsgraph()->machine()->S1x16AllTrue(), inputs[0]); default: FATAL_UNSUPPORTED_OPCODE(opcode); } } Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane, Node* const* inputs) { has_simd_ = true; switch (opcode) { case wasm::kExprF32x4ExtractLane: return graph()->NewNode(jsgraph()->machine()->F32x4ExtractLane(lane), inputs[0]); case wasm::kExprF32x4ReplaceLane: return graph()->NewNode(jsgraph()->machine()->F32x4ReplaceLane(lane), inputs[0], inputs[1]); case wasm::kExprI32x4ExtractLane: return graph()->NewNode(jsgraph()->machine()->I32x4ExtractLane(lane), inputs[0]); case wasm::kExprI32x4ReplaceLane: return graph()->NewNode(jsgraph()->machine()->I32x4ReplaceLane(lane), inputs[0], inputs[1]); case wasm::kExprI16x8ExtractLane: return graph()->NewNode(jsgraph()->machine()->I16x8ExtractLane(lane), inputs[0]); case wasm::kExprI16x8ReplaceLane: return graph()->NewNode(jsgraph()->machine()->I16x8ReplaceLane(lane), inputs[0], inputs[1]); case wasm::kExprI8x16ExtractLane: return graph()->NewNode(jsgraph()->machine()->I8x16ExtractLane(lane), inputs[0]); case wasm::kExprI8x16ReplaceLane: return graph()->NewNode(jsgraph()->machine()->I8x16ReplaceLane(lane), inputs[0], inputs[1]); default: FATAL_UNSUPPORTED_OPCODE(opcode); } } Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift, Node* const* inputs) { has_simd_ = true; switch (opcode) { case wasm::kExprI32x4Shl: return graph()->NewNode(jsgraph()->machine()->I32x4Shl(shift), inputs[0]); case wasm::kExprI32x4ShrS: return graph()->NewNode(jsgraph()->machine()->I32x4ShrS(shift), inputs[0]); case wasm::kExprI32x4ShrU: return graph()->NewNode(jsgraph()->machine()->I32x4ShrU(shift), inputs[0]); case wasm::kExprI16x8Shl: return graph()->NewNode(jsgraph()->machine()->I16x8Shl(shift), inputs[0]); case wasm::kExprI16x8ShrS: return graph()->NewNode(jsgraph()->machine()->I16x8ShrS(shift), inputs[0]); case wasm::kExprI16x8ShrU: return graph()->NewNode(jsgraph()->machine()->I16x8ShrU(shift), inputs[0]); case wasm::kExprI8x16Shl: return graph()->NewNode(jsgraph()->machine()->I8x16Shl(shift), inputs[0]); case wasm::kExprI8x16ShrS: return graph()->NewNode(jsgraph()->machine()->I8x16ShrS(shift), inputs[0]); case wasm::kExprI8x16ShrU: return graph()->NewNode(jsgraph()->machine()->I8x16ShrU(shift), inputs[0]); default: FATAL_UNSUPPORTED_OPCODE(opcode); } } Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16], Node* const* inputs) { has_simd_ = true; return graph()->NewNode(jsgraph()->machine()->S8x16Shuffle(shuffle), inputs[0], inputs[1]); } #define ATOMIC_BINOP_LIST(V) \ V(I32AtomicAdd, Add, Uint32) \ V(I32AtomicSub, Sub, Uint32) \ V(I32AtomicAnd, And, Uint32) \ V(I32AtomicOr, Or, Uint32) \ V(I32AtomicXor, Xor, Uint32) \ V(I32AtomicExchange, Exchange, Uint32) \ V(I32AtomicAdd8U, Add, Uint8) \ V(I32AtomicSub8U, Sub, Uint8) \ V(I32AtomicAnd8U, And, Uint8) \ V(I32AtomicOr8U, Or, Uint8) \ V(I32AtomicXor8U, Xor, Uint8) \ V(I32AtomicExchange8U, Exchange, Uint8) \ V(I32AtomicAdd16U, Add, Uint16) \ V(I32AtomicSub16U, Sub, Uint16) \ V(I32AtomicAnd16U, And, Uint16) \ V(I32AtomicOr16U, Or, Uint16) \ V(I32AtomicXor16U, Xor, Uint16) \ V(I32AtomicExchange16U, Exchange, Uint16) #define ATOMIC_TERNARY_LIST(V) \ V(I32AtomicCompareExchange, CompareExchange, Uint32) \ V(I32AtomicCompareExchange8U, CompareExchange, Uint8) \ V(I32AtomicCompareExchange16U, CompareExchange, Uint16) #define ATOMIC_LOAD_LIST(V) \ V(I32AtomicLoad, Uint32) \ V(I32AtomicLoad8U, Uint8) \ V(I32AtomicLoad16U, Uint16) #define ATOMIC_STORE_LIST(V) \ V(I32AtomicStore, Uint32, kWord32) \ V(I32AtomicStore8U, Uint8, kWord8) \ V(I32AtomicStore16U, Uint16, kWord16) Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs, uint32_t alignment, uint32_t offset, wasm::WasmCodePosition position) { // TODO(gdeepti): Add alignment validation, traps on misalignment Node* node; switch (opcode) { #define BUILD_ATOMIC_BINOP(Name, Operation, Type) \ case wasm::kExpr##Name: { \ Node* index = \ BoundsCheckMem(wasm::WasmOpcodes::MemSize(MachineType::Type()), \ inputs[0], offset, position, kNeedsBoundsCheck); \ node = graph()->NewNode( \ jsgraph()->machine()->Atomic##Operation(MachineType::Type()), \ MemBuffer(offset), index, inputs[1], *effect_, *control_); \ break; \ } ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP) #undef BUILD_ATOMIC_BINOP #define BUILD_ATOMIC_TERNARY_OP(Name, Operation, Type) \ case wasm::kExpr##Name: { \ Node* index = \ BoundsCheckMem(wasm::WasmOpcodes::MemSize(MachineType::Type()), \ inputs[0], offset, position, kNeedsBoundsCheck); \ node = graph()->NewNode( \ jsgraph()->machine()->Atomic##Operation(MachineType::Type()), \ MemBuffer(offset), index, inputs[1], inputs[2], *effect_, *control_); \ break; \ } ATOMIC_TERNARY_LIST(BUILD_ATOMIC_TERNARY_OP) #undef BUILD_ATOMIC_TERNARY_OP #define BUILD_ATOMIC_LOAD_OP(Name, Type) \ case wasm::kExpr##Name: { \ Node* index = \ BoundsCheckMem(wasm::WasmOpcodes::MemSize(MachineType::Type()), \ inputs[0], offset, position, kNeedsBoundsCheck); \ node = graph()->NewNode( \ jsgraph()->machine()->AtomicLoad(MachineType::Type()), \ MemBuffer(offset), index, *effect_, *control_); \ break; \ } ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP) #undef BUILD_ATOMIC_LOAD_OP #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep) \ case wasm::kExpr##Name: { \ Node* index = \ BoundsCheckMem(wasm::WasmOpcodes::MemSize(MachineType::Type()), \ inputs[0], offset, position, kNeedsBoundsCheck); \ node = graph()->NewNode( \ jsgraph()->machine()->AtomicStore(MachineRepresentation::Rep), \ MemBuffer(offset), index, inputs[1], *effect_, *control_); \ break; \ } ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP) #undef BUILD_ATOMIC_STORE_OP default: FATAL_UNSUPPORTED_OPCODE(opcode); } *effect_ = node; return node; } #undef ATOMIC_BINOP_LIST #undef ATOMIC_TERNARY_LIST #undef ATOMIC_LOAD_LIST #undef ATOMIC_STORE_LIST namespace { bool must_record_function_compilation(Isolate* isolate) { return isolate->logger()->is_logging_code_events() || isolate->is_profiling(); } PRINTF_FORMAT(4, 5) void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag, Isolate* isolate, Handle<Code> code, const char* format, ...) { DCHECK(must_record_function_compilation(isolate)); ScopedVector<char> buffer(128); va_list arguments; va_start(arguments, format); int len = VSNPrintF(buffer, format, arguments); CHECK_LT(0, len); va_end(arguments); Handle<String> name_str = isolate->factory()->NewStringFromAsciiChecked(buffer.start()); Handle<String> script_str = isolate->factory()->NewStringFromAsciiChecked("(wasm)"); Handle<SharedFunctionInfo> shared = isolate->factory()->NewSharedFunctionInfo(name_str, code, false); PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *shared, *script_str, 0, 0)); } } // namespace Handle<Code> CompileJSToWasmWrapper(Isolate* isolate, wasm::WasmModule* module, WasmCodeWrapper wasm_code, uint32_t index, Address wasm_context_address, bool use_trap_handler) { const wasm::WasmFunction* func = &module->functions[index]; //---------------------------------------------------------------------------- // Create the Graph //---------------------------------------------------------------------------- Zone zone(isolate->allocator(), ZONE_NAME); Graph graph(&zone); CommonOperatorBuilder common(&zone); MachineOperatorBuilder machine( &zone, MachineType::PointerRepresentation(), InstructionSelector::SupportedMachineOperatorFlags(), InstructionSelector::AlignmentRequirements()); JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); Node* control = nullptr; Node* effect = nullptr; // TODO(titzer): compile JS to WASM wrappers without a {ModuleEnv}. ModuleEnv env(module, // TODO(mtrofin): remove the Illegal builtin when we don't need // FLAG_wasm_jit_to_native BUILTIN_CODE(isolate, Illegal), // default_function_code use_trap_handler); WasmGraphBuilder builder(&env, &zone, &jsgraph, CEntryStub(isolate, 1).GetCode(), func->sig); builder.set_control_ptr(&control); builder.set_effect_ptr(&effect); builder.BuildJSToWasmWrapper(wasm_code, wasm_context_address); //---------------------------------------------------------------------------- // Run the compilation pipeline. //---------------------------------------------------------------------------- if (FLAG_trace_turbo_graph) { // Simple textual RPO. OFStream os(stdout); os << "-- Graph after change lowering -- " << std::endl; os << AsRPO(graph); } // Schedule and compile to machine code. int params = static_cast<int>(module->functions[index].sig->parameter_count()); CallDescriptor* incoming = Linkage::GetJSCallDescriptor( &zone, false, params + 1, CallDescriptor::kNoFlags); #ifdef DEBUG EmbeddedVector<char, 32> func_name; static unsigned id = 0; func_name.Truncate(SNPrintF(func_name, "js-to-wasm#%d", id++)); #else Vector<const char> func_name = CStrVector("js-to-wasm"); #endif CompilationInfo info(func_name, &zone, Code::JS_TO_WASM_FUNCTION); Handle<Code> code = Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph); #ifdef ENABLE_DISASSEMBLER if (FLAG_print_opt_code && !code.is_null()) { OFStream os(stdout); code->Disassemble(func_name.start(), os); } #endif if (must_record_function_compilation(isolate)) { RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code, "%.*s", func_name.length(), func_name.start()); } return code; } namespace { void ValidateImportWrapperReferencesImmovables(Handle<Code> wrapper) { #ifdef DEBUG // We expect the only embedded objects to be those originating from // a snapshot, which are immovable. DisallowHeapAllocation no_gc; if (wrapper.is_null()) return; static constexpr int kAllGCRefs = (1 << (RelocInfo::LAST_GCED_ENUM + 1)) - 1; for (RelocIterator it(*wrapper, kAllGCRefs); !it.done(); it.next()) { RelocInfo::Mode mode = it.rinfo()->rmode(); Object* target = nullptr; switch (mode) { case RelocInfo::CODE_TARGET: // this would be either one of the stubs or builtins, because // we didn't link yet. target = Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); break; case RelocInfo::EMBEDDED_OBJECT: target = it.rinfo()->target_object(); break; default: UNREACHABLE(); } DCHECK_NOT_NULL(target); bool is_immovable = target->IsSmi() || Heap::IsImmovable(HeapObject::cast(target)); bool is_allowed_stub = false; if (target->IsCode()) { Code* code = Code::cast(target); is_allowed_stub = code->kind() == Code::STUB && CodeStub::MajorKeyFromKey(code->stub_key()) == CodeStub::DoubleToI; } DCHECK(is_immovable || is_allowed_stub); } #endif } } // namespace Handle<Code> CompileWasmToJSWrapper( Isolate* isolate, Handle<JSReceiver> target, wasm::FunctionSig* sig, uint32_t index, wasm::ModuleOrigin origin, bool use_trap_handler, Handle<FixedArray> global_js_imports_table) { //---------------------------------------------------------------------------- // Create the Graph //---------------------------------------------------------------------------- Zone zone(isolate->allocator(), ZONE_NAME); Graph graph(&zone); CommonOperatorBuilder common(&zone); MachineOperatorBuilder machine( &zone, MachineType::PointerRepresentation(), InstructionSelector::SupportedMachineOperatorFlags(), InstructionSelector::AlignmentRequirements()); JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); Node* control = nullptr; Node* effect = nullptr; SourcePositionTable* source_position_table = origin == wasm::kAsmJsOrigin ? new (&zone) SourcePositionTable(&graph) : nullptr; ModuleEnv env(nullptr, Handle<Code>::null(), use_trap_handler); WasmGraphBuilder builder(&env, &zone, &jsgraph, CEntryStub(isolate, 1).GetCode(), sig, source_position_table); builder.set_control_ptr(&control); builder.set_effect_ptr(&effect); if (builder.BuildWasmToJSWrapper(target, global_js_imports_table, index)) { global_js_imports_table->set( OffsetForImportData(index, WasmGraphBuilder::kFunction), *target); if (target->IsJSFunction()) { Handle<JSFunction> function = Handle<JSFunction>::cast(target); global_js_imports_table->set( OffsetForImportData(index, WasmGraphBuilder::kFunctionContext), function->context()); global_js_imports_table->set( OffsetForImportData(index, WasmGraphBuilder::kGlobalProxy), function->context()->global_proxy()); } } if (FLAG_trace_turbo_graph) { // Simple textual RPO. OFStream os(stdout); os << "-- Graph after change lowering -- " << std::endl; os << AsRPO(graph); } // Schedule and compile to machine code. CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig); if (machine.Is32()) { incoming = GetI32WasmCallDescriptor(&zone, incoming); } #ifdef DEBUG EmbeddedVector<char, 32> func_name; static unsigned id = 0; func_name.Truncate(SNPrintF(func_name, "wasm-to-js#%d", id++)); #else Vector<const char> func_name = CStrVector("wasm-to-js"); #endif CompilationInfo info(func_name, &zone, Code::WASM_TO_JS_FUNCTION); Handle<Code> code = Pipeline::GenerateCodeForTesting( &info, isolate, incoming, &graph, nullptr, source_position_table); ValidateImportWrapperReferencesImmovables(code); Handle<FixedArray> deopt_data = isolate->factory()->NewFixedArray(2, TENURED); intptr_t loc = reinterpret_cast<intptr_t>(global_js_imports_table.location()); Handle<Object> loc_handle = isolate->factory()->NewHeapNumberFromBits(loc); deopt_data->set(0, *loc_handle); Handle<Object> index_handle = isolate->factory()->NewNumberFromInt( OffsetForImportData(index, WasmGraphBuilder::kFunction)); deopt_data->set(1, *index_handle); code->set_deoptimization_data(*deopt_data); #ifdef ENABLE_DISASSEMBLER if (FLAG_print_opt_code && !code.is_null()) { OFStream os(stdout); code->Disassemble(func_name.start(), os); } #endif if (must_record_function_compilation(isolate)) { RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code, "%.*s", func_name.length(), func_name.start()); } return code; } Handle<Code> CompileWasmToWasmWrapper(Isolate* isolate, WasmCodeWrapper target, wasm::FunctionSig* sig, Address new_wasm_context_address) { //---------------------------------------------------------------------------- // Create the Graph //---------------------------------------------------------------------------- Zone zone(isolate->allocator(), ZONE_NAME); Graph graph(&zone); CommonOperatorBuilder common(&zone); MachineOperatorBuilder machine( &zone, MachineType::PointerRepresentation(), InstructionSelector::SupportedMachineOperatorFlags(), InstructionSelector::AlignmentRequirements()); JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); Node* control = nullptr; Node* effect = nullptr; ModuleEnv env( nullptr, Handle<Code>::null(), !target.IsCodeObject() && target.GetWasmCode()->HasTrapHandlerIndex()); WasmGraphBuilder builder(&env, &zone, &jsgraph, Handle<Code>(), sig); builder.set_control_ptr(&control); builder.set_effect_ptr(&effect); builder.BuildWasmToWasmWrapper(target, new_wasm_context_address); if (HasInt64ParamOrReturn(sig)) builder.LowerInt64(); if (FLAG_trace_turbo_graph) { // Simple textual RPO. OFStream os(stdout); os << "-- Graph after change lowering -- " << std::endl; os << AsRPO(graph); } // Schedule and compile to machine code. CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig); if (machine.Is32()) { incoming = GetI32WasmCallDescriptor(&zone, incoming); } bool debugging = #if DEBUG true; #else FLAG_print_opt_code || FLAG_trace_turbo || FLAG_trace_turbo_graph; #endif Vector<const char> func_name = ArrayVector("wasm-to-wasm"); static unsigned id = 0; Vector<char> buffer; if (debugging) { buffer = Vector<char>::New(128); int chars = SNPrintF(buffer, "wasm-to-wasm#%d", id); func_name = Vector<const char>::cast(buffer.SubVector(0, chars)); } CompilationInfo info(func_name, &zone, Code::WASM_TO_WASM_FUNCTION); Handle<Code> code = Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph); #ifdef ENABLE_DISASSEMBLER if (FLAG_print_opt_code && !code.is_null()) { OFStream os(stdout); code->Disassemble(buffer.start(), os); } #endif if (debugging) { buffer.Dispose(); } if (isolate->logger()->is_logging_code_events() || isolate->is_profiling()) { RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code, "wasm-to-wasm"); } return code; } Handle<Code> CompileWasmInterpreterEntry(Isolate* isolate, uint32_t func_index, wasm::FunctionSig* sig, Handle<WasmInstanceObject> instance) { //---------------------------------------------------------------------------- // Create the Graph //---------------------------------------------------------------------------- Zone zone(isolate->allocator(), ZONE_NAME); Graph graph(&zone); CommonOperatorBuilder common(&zone); MachineOperatorBuilder machine( &zone, MachineType::PointerRepresentation(), InstructionSelector::SupportedMachineOperatorFlags(), InstructionSelector::AlignmentRequirements()); JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); Node* control = nullptr; Node* effect = nullptr; WasmGraphBuilder builder(nullptr, &zone, &jsgraph, CEntryStub(isolate, 1).GetCode(), sig); builder.set_control_ptr(&control); builder.set_effect_ptr(&effect); builder.BuildWasmInterpreterEntry(func_index); Handle<Code> code = Handle<Code>::null(); { if (FLAG_trace_turbo_graph) { // Simple textual RPO. OFStream os(stdout); os << "-- Wasm interpreter entry graph -- " << std::endl; os << AsRPO(graph); } // Schedule and compile to machine code. CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig); if (machine.Is32()) { incoming = GetI32WasmCallDescriptor(&zone, incoming); } #ifdef DEBUG EmbeddedVector<char, 32> func_name; func_name.Truncate( SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index)); #else Vector<const char> func_name = CStrVector("wasm-interpreter-entry"); #endif CompilationInfo info(func_name, &zone, Code::WASM_INTERPRETER_ENTRY); code = Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph, nullptr); #ifdef ENABLE_DISASSEMBLER if (FLAG_print_opt_code && !code.is_null()) { OFStream os(stdout); code->Disassemble(func_name.start(), os); } #endif if (must_record_function_compilation(isolate)) { RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code, "%.*s", func_name.length(), func_name.start()); } } if (!FLAG_wasm_jit_to_native) { Handle<FixedArray> deopt_data = isolate->factory()->NewFixedArray(1, TENURED); Handle<WeakCell> weak_instance = isolate->factory()->NewWeakCell(instance); deopt_data->set(0, *weak_instance); code->set_deoptimization_data(*deopt_data); } return code; } Handle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig, Address wasm_context_address) { Zone zone(isolate->allocator(), ZONE_NAME); Graph graph(&zone); CommonOperatorBuilder common(&zone); MachineOperatorBuilder machine( &zone, MachineType::PointerRepresentation(), InstructionSelector::SupportedMachineOperatorFlags(), InstructionSelector::AlignmentRequirements()); JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); Node* control = nullptr; Node* effect = nullptr; WasmGraphBuilder builder(nullptr, &zone, &jsgraph, CEntryStub(isolate, 1).GetCode(), sig); builder.set_control_ptr(&control); builder.set_effect_ptr(&effect); builder.BuildCWasmEntry(wasm_context_address); if (FLAG_trace_turbo_graph) { // Simple textual RPO. OFStream os(stdout); os << "-- C Wasm entry graph -- " << std::endl; os << AsRPO(graph); } // Schedule and compile to machine code. CallDescriptor* incoming = Linkage::GetJSCallDescriptor( &zone, false, CWasmEntryParameters::kNumParameters + 1, CallDescriptor::kNoFlags); // Build a name in the form "c-wasm-entry:<params>:<returns>". static constexpr size_t kMaxNameLen = 128; char debug_name[kMaxNameLen] = "c-wasm-entry:"; size_t name_len = strlen(debug_name); auto append_name_char = [&](char c) { if (name_len + 1 < kMaxNameLen) debug_name[name_len++] = c; }; for (wasm::ValueType t : sig->parameters()) { append_name_char(wasm::WasmOpcodes::ShortNameOf(t)); } append_name_char(':'); for (wasm::ValueType t : sig->returns()) { append_name_char(wasm::WasmOpcodes::ShortNameOf(t)); } debug_name[name_len] = '\0'; Vector<const char> debug_name_vec(debug_name, name_len); CompilationInfo info(debug_name_vec, &zone, Code::C_WASM_ENTRY); Handle<Code> code = Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph); #ifdef ENABLE_DISASSEMBLER if (FLAG_print_opt_code && !code.is_null()) { OFStream os(stdout); code->Disassemble(debug_name, os); } #endif return code; } SourcePositionTable* WasmCompilationUnit::BuildGraphForWasmFunction( double* decode_ms) { base::ElapsedTimer decode_timer; if (FLAG_trace_wasm_decode_time) { decode_timer.Start(); } // Create a TF graph during decoding. SourcePositionTable* source_position_table = new (tf_.jsgraph_->zone()) SourcePositionTable(tf_.jsgraph_->graph()); WasmGraphBuilder builder(env_, tf_.jsgraph_->zone(), tf_.jsgraph_, centry_stub_, func_body_.sig, source_position_table, runtime_exception_support_); tf_.graph_construction_result_ = wasm::BuildTFGraph(isolate_->allocator(), &builder, func_body_); if (tf_.graph_construction_result_.failed()) { if (FLAG_trace_wasm_compiler) { OFStream os(stdout); os << "Compilation failed: " << tf_.graph_construction_result_.error_msg() << std::endl; } return nullptr; } builder.LowerInt64(); if (builder.has_simd() && (!CpuFeatures::SupportsWasmSimd128() || lower_simd_)) { SimdScalarLowering(tf_.jsgraph_, func_body_.sig).LowerGraph(); } if (func_index_ >= FLAG_trace_wasm_ast_start && func_index_ < FLAG_trace_wasm_ast_end) { PrintRawWasmCode(isolate_->allocator(), func_body_, env_->module, wasm::kPrintLocals); } if (FLAG_trace_wasm_decode_time) { *decode_ms = decode_timer.Elapsed().InMillisecondsF(); } return source_position_table; } namespace { Vector<const char> GetDebugName(Zone* zone, wasm::WasmName name, int index) { if (!name.is_empty()) { return name; } #ifdef DEBUG constexpr int kBufferLength = 15; EmbeddedVector<char, kBufferLength> name_vector; int name_len = SNPrintF(name_vector, "wasm#%d", index); DCHECK(name_len > 0 && name_len < name_vector.length()); char* index_name = zone->NewArray<char>(name_len); memcpy(index_name, name_vector.start(), name_len); return Vector<const char>(index_name, name_len); #else return {}; #endif } } // namespace // static WasmCompilationUnit::CompilationMode WasmCompilationUnit::GetDefaultCompilationMode() { return FLAG_liftoff ? WasmCompilationUnit::CompilationMode::kLiftoff : WasmCompilationUnit::CompilationMode::kTurbofan; } WasmCompilationUnit::WasmCompilationUnit( Isolate* isolate, ModuleEnv* env, wasm::NativeModule* native_module, wasm::FunctionBody body, wasm::WasmName name, int index, Handle<Code> centry_stub, CompilationMode mode, Counters* counters, RuntimeExceptionSupport exception_support, bool lower_simd) : isolate_(isolate), env_(env), func_body_(body), func_name_(name), counters_(counters ? counters : isolate->counters()), centry_stub_(centry_stub), func_index_(index), runtime_exception_support_(exception_support), native_module_(native_module), lower_simd_(lower_simd), protected_instructions_( new std::vector<trap_handler::ProtectedInstructionData>()), mode_(mode) { switch (mode_) { case WasmCompilationUnit::CompilationMode::kLiftoff: new (&liftoff_) LiftoffData(isolate); break; case WasmCompilationUnit::CompilationMode::kTurbofan: new (&tf_) TurbofanData(); break; } } WasmCompilationUnit::~WasmCompilationUnit() { switch (mode_) { case WasmCompilationUnit::CompilationMode::kLiftoff: liftoff_.~LiftoffData(); break; case WasmCompilationUnit::CompilationMode::kTurbofan: tf_.~TurbofanData(); break; } } void WasmCompilationUnit::ExecuteCompilation() { auto size_histogram = env_->module->is_wasm() ? counters()->wasm_wasm_function_size_bytes() : counters()->wasm_asm_function_size_bytes(); size_histogram->AddSample( static_cast<int>(func_body_.end - func_body_.start)); auto timed_histogram = env_->module->is_wasm() ? counters()->wasm_compile_wasm_function_time() : counters()->wasm_compile_asm_function_time(); TimedHistogramScope wasm_compile_function_time_scope(timed_histogram); if (FLAG_trace_wasm_compiler) { if (func_name_.start() != nullptr) { PrintF("Compiling wasm function %d:'%.*s'\n\n", func_index(), func_name_.length(), func_name_.start()); } else { PrintF("Compiling wasm function %d:<unnamed>\n\n", func_index()); } } switch (mode_) { case WasmCompilationUnit::CompilationMode::kLiftoff: if (ExecuteLiftoffCompilation()) break; // Otherwise, fall back to turbofan. liftoff_.~LiftoffData(); mode_ = WasmCompilationUnit::CompilationMode::kTurbofan; new (&tf_) TurbofanData(); // fall-through case WasmCompilationUnit::CompilationMode::kTurbofan: ExecuteTurbofanCompilation(); break; } } void WasmCompilationUnit::ExecuteTurbofanCompilation() { double decode_ms = 0; size_t node_count = 0; // Scope for the {graph_zone}. { Zone graph_zone(isolate_->allocator(), ZONE_NAME); tf_.jsgraph_ = new (&graph_zone) JSGraph( isolate_, new (&graph_zone) Graph(&graph_zone), new (&graph_zone) CommonOperatorBuilder(&graph_zone), nullptr, nullptr, new (&graph_zone) MachineOperatorBuilder( &graph_zone, MachineType::PointerRepresentation(), InstructionSelector::SupportedMachineOperatorFlags(), InstructionSelector::AlignmentRequirements())); SourcePositionTable* source_positions = BuildGraphForWasmFunction(&decode_ms); if (tf_.graph_construction_result_.failed()) { ok_ = false; return; } base::ElapsedTimer pipeline_timer; if (FLAG_trace_wasm_decode_time) { node_count = tf_.jsgraph_->graph()->NodeCount(); pipeline_timer.Start(); } tf_.compilation_zone_.reset(new Zone(isolate_->allocator(), ZONE_NAME)); // Run the compiler pipeline to generate machine code. CallDescriptor* descriptor = GetWasmCallDescriptor(tf_.compilation_zone_.get(), func_body_.sig); if (tf_.jsgraph_->machine()->Is32()) { descriptor = GetI32WasmCallDescriptor(tf_.compilation_zone_.get(), descriptor); } tf_.info_.reset(new CompilationInfo( GetDebugName(tf_.compilation_zone_.get(), func_name_, func_index_), tf_.compilation_zone_.get(), Code::WASM_FUNCTION)); tf_.job_.reset(Pipeline::NewWasmCompilationJob( tf_.info_.get(), isolate_, tf_.jsgraph_, descriptor, source_positions, protected_instructions_.get(), env_->module->origin())); ok_ = tf_.job_->ExecuteJob() == CompilationJob::SUCCEEDED; // TODO(bradnelson): Improve histogram handling of size_t. counters()->wasm_compile_function_peak_memory_bytes()->AddSample( static_cast<int>(tf_.jsgraph_->graph()->zone()->allocation_size())); if (FLAG_trace_wasm_decode_time) { double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF(); PrintF( "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, " "%0.3f ms pipeline\n", static_cast<unsigned>(func_body_.end - func_body_.start), decode_ms, node_count, pipeline_ms); } // The graph zone is about to get out of scope. Avoid invalid references. tf_.jsgraph_ = nullptr; } // Record the memory cost this unit places on the system until // it is finalized. memory_cost_ = tf_.job_->AllocatedMemory(); } // WasmCompilationUnit::ExecuteLiftoffCompilation() is defined in // liftoff-compiler.cc. WasmCodeWrapper WasmCompilationUnit::FinishCompilation( wasm::ErrorThrower* thrower) { WasmCodeWrapper ret; switch (mode_) { case WasmCompilationUnit::CompilationMode::kLiftoff: ret = FinishLiftoffCompilation(thrower); break; case WasmCompilationUnit::CompilationMode::kTurbofan: ret = FinishTurbofanCompilation(thrower); break; default: UNREACHABLE(); } if (!ret.IsCodeObject() && ret.is_null()) { thrower->RuntimeError("Error finalizing code."); } return ret; } WasmCodeWrapper WasmCompilationUnit::FinishTurbofanCompilation( wasm::ErrorThrower* thrower) { if (!ok_) { if (tf_.graph_construction_result_.failed()) { // Add the function as another context for the exception. EmbeddedVector<char, 128> message; if (func_name_.start() == nullptr) { SNPrintF(message, "Compiling wasm function #%d failed", func_index_); } else { wasm::TruncatedUserString<> trunc_name(func_name_); SNPrintF(message, "Compiling wasm function #%d:%.*s failed", func_index_, trunc_name.length(), trunc_name.start()); } thrower->CompileFailed(message.start(), tf_.graph_construction_result_); } return {}; } base::ElapsedTimer codegen_timer; if (FLAG_trace_wasm_decode_time) { codegen_timer.Start(); } if (tf_.job_->FinalizeJob(isolate_) != CompilationJob::SUCCEEDED) { return {}; } if (!FLAG_wasm_jit_to_native) { Handle<Code> code = tf_.info_->code(); DCHECK(!code.is_null()); if (FLAG_trace_wasm_decode_time) { double codegen_ms = codegen_timer.Elapsed().InMillisecondsF(); PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n", static_cast<unsigned>(func_body_.end - func_body_.start), codegen_ms); } PackProtectedInstructions(code); return WasmCodeWrapper(code); } else { // TODO(mtrofin): when we crystalize a design in lieu of WasmCodeDesc, that // works for both wasm and non-wasm, we can simplify AddCode to just take // that as a parameter. const CodeDesc& desc = tf_.job_->compilation_info()->wasm_code_desc()->code_desc; wasm::WasmCode* code = native_module_->AddCode( desc, tf_.job_->compilation_info()->wasm_code_desc()->frame_slot_count, func_index_, tf_.job_->compilation_info()->wasm_code_desc()->safepoint_table_offset, std::move(protected_instructions_)); if (!code) { return WasmCodeWrapper(code); } // TODO(mtrofin): add CodeEventListener call - see the non-native case. if (FLAG_trace_wasm_decode_time) { double codegen_ms = codegen_timer.Elapsed().InMillisecondsF(); PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n", static_cast<unsigned>(func_body_.end - func_body_.start), codegen_ms); } Handle<ByteArray> source_positions = tf_.job_->compilation_info()->wasm_code_desc()->source_positions_table; MaybeHandle<HandlerTable> handler_table = tf_.job_->compilation_info()->wasm_code_desc()->handler_table; native_module_->compiled_module()->source_positions()->set( func_index_, *source_positions); if (!handler_table.is_null()) { native_module_->compiled_module()->handler_table()->set( func_index_, *handler_table.ToHandleChecked()); } #ifdef ENABLE_DISASSEMBLER // Note: only do this after setting source positions, as this will be // accessed and printed here. if (FLAG_print_code || FLAG_print_wasm_code) { // TODO(wasm): Use proper log files, here and elsewhere. PrintF("--- Native Wasm code ---\n"); code->Print(isolate_); PrintF("--- End code ---\n"); } #endif // TODO(mtrofin): this should probably move up in the common caller, // once liftoff has source positions. Until then, we'd need to handle // undefined values, which is complicating the code. LOG_CODE_EVENT(isolate_, CodeLinePosInfoRecordEvent(code->instructions().start(), *source_positions)); return WasmCodeWrapper(code); } } // TODO(mtrofin): remove when FLAG_wasm_jit_to_native is not needed void WasmCompilationUnit::PackProtectedInstructions(Handle<Code> code) const { if (protected_instructions_->empty()) return; DCHECK_LT(protected_instructions_->size(), std::numeric_limits<int>::max()); const int num_instructions = static_cast<int>(protected_instructions_->size()); Handle<FixedArray> fn_protected = isolate_->factory()->NewFixedArray( num_instructions * Code::kTrapDataSize, TENURED); for (int i = 0; i < num_instructions; ++i) { const trap_handler::ProtectedInstructionData& instruction = protected_instructions_->at(i); fn_protected->set(Code::kTrapDataSize * i + Code::kTrapCodeOffset, Smi::FromInt(instruction.instr_offset)); fn_protected->set(Code::kTrapDataSize * i + Code::kTrapLandingOffset, Smi::FromInt(instruction.landing_offset)); } code->set_protected_instructions(*fn_protected); } WasmCodeWrapper WasmCompilationUnit::FinishLiftoffCompilation( wasm::ErrorThrower* thrower) { CodeDesc desc; liftoff_.asm_.GetCode(isolate_, &desc); Handle<ByteArray> source_positions = liftoff_.source_position_table_builder_.ToSourcePositionTable(isolate_); WasmCodeWrapper ret; if (!FLAG_wasm_jit_to_native) { Handle<Code> code; code = isolate_->factory()->NewCode( desc, Code::WASM_FUNCTION, code, Builtins::kNoBuiltinId, MaybeHandle<HandlerTable>(), source_positions, MaybeHandle<DeoptimizationData>(), kMovable, 0, // stub_key false, // is_turbofanned liftoff_.asm_.GetTotalFrameSlotCount(), // stack_slots liftoff_.safepoint_table_offset_); if (isolate_->logger()->is_logging_code_events() || isolate_->is_profiling()) { RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate_, code, "wasm#%d-liftoff", func_index_); } PackProtectedInstructions(code); ret = WasmCodeWrapper(code); } else { // TODO(mtrofin): figure a way to raise events. // Consider lifting it to FinishCompilation. native_module_->compiled_module()->source_positions()->set( func_index_, *source_positions); ret = WasmCodeWrapper( native_module_->AddCode(desc, liftoff_.asm_.GetTotalFrameSlotCount(), func_index_, liftoff_.safepoint_table_offset_, std::move(protected_instructions_), true)); } #ifdef ENABLE_DISASSEMBLER if (FLAG_print_code || FLAG_print_wasm_code) { // TODO(wasm): Use proper log files, here and elsewhere. OFStream os(stdout); os << "--- Wasm liftoff code ---\n"; EmbeddedVector<char, 64> func_name; if (func_name_.start() != nullptr) { SNPrintF(func_name, "#%d:%.*s", func_index(), func_name_.length(), func_name_.start()); } else { SNPrintF(func_name, "wasm#%d", func_index()); } ret.Disassemble(func_name.start(), isolate_, os); os << "--- End code ---\n"; } #endif return ret; } // static WasmCodeWrapper WasmCompilationUnit::CompileWasmFunction( wasm::NativeModule* native_module, wasm::ErrorThrower* thrower, Isolate* isolate, const wasm::ModuleWireBytes& wire_bytes, ModuleEnv* env, const wasm::WasmFunction* function, CompilationMode mode) { wasm::FunctionBody function_body{ function->sig, function->code.offset(), wire_bytes.start() + function->code.offset(), wire_bytes.start() + function->code.end_offset()}; WasmCompilationUnit unit(isolate, env, native_module, function_body, wire_bytes.GetNameOrNull(function), function->func_index, CEntryStub(isolate, 1).GetCode(), mode); unit.ExecuteCompilation(); return unit.FinishCompilation(thrower); } #undef WASM_64 #undef FATAL_UNSUPPORTED_OPCODE } // namespace compiler } // namespace internal } // namespace v8