[wasm] Move fast path of wasm Table.get/set to builtins

- Reworks the builtins WasmTableGet and WasmTableSet to do the fast
  path, instead of generating this inline in wasm-compiler.

Change-Id: I0a47c09d6f4f6d81c7b362f6f45e95b19e3edf86
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2135864
Commit-Queue: Bill Budge <bbudge@chromium.org>
Reviewed-by: Andreas Haas <ahaas@chromium.org>
Cr-Commit-Position: refs/heads/master@{#67296}

src/builtins/builtins-wasm-gen.cc

@@ -312,50 +312,99 @@ TF_BUILTIN(WasmTableCopy, WasmBuiltinsAssembler) {
 }
 
 TF_BUILTIN(WasmTableGet, WasmBuiltinsAssembler) {
-  TNode<Int32T> entry_index =
-      UncheckedCast<Int32T>(Parameter(Descriptor::kEntryIndex));
   TNode<WasmInstanceObject> instance = LoadInstanceFromFrame();
-  TNode<Context> context = LoadContextFromInstance(instance);
-  Label entry_index_out_of_range(this, Label::kDeferred);
 
-  TNode<BoolT> entry_index_fits_in_smi =
-      IsValidPositiveSmi(ChangeInt32ToIntPtr(entry_index));
-  GotoIfNot(entry_index_fits_in_smi, &entry_index_out_of_range);
+  Label call_runtime(this, Label::kDeferred),
+      index_out_of_range(this, Label::kDeferred);
+
+  TNode<IntPtrT> table_index =
+      UncheckedCast<IntPtrT>(Parameter(Descriptor::kTableIndex));
+  GotoIfNot(IsValidPositiveSmi(table_index), &index_out_of_range);
+  TNode<IntPtrT> entry_index = ChangeInt32ToIntPtr(
+      UncheckedCast<Int32T>(Parameter(Descriptor::kEntryIndex)));
+  GotoIfNot(IsValidPositiveSmi(entry_index), &index_out_of_range);
+
+  TNode<FixedArray> tables_array =
+      LoadObjectField<FixedArray>(instance, WasmInstanceObject::kTablesOffset);
+  TNode<WasmTableObject> table =
+      CAST(LoadFixedArrayElement(tables_array, table_index));
+  TNode<IntPtrT> entries_length =
+      LoadAndUntagObjectField(table, WasmTableObject::kCurrentLengthOffset);
+  GotoIfNot(IntPtrLessThan(entry_index, entries_length), &index_out_of_range);
+
+  TNode<FixedArray> entries_array =
+      LoadObjectField<FixedArray>(table, WasmTableObject::kEntriesOffset);
+
+  TNode<Object> entry = LoadFixedArrayElement(entries_array, entry_index);
 
-  TNode<Smi> entry_index_smi = SmiFromInt32(entry_index);
-  TNode<Smi> table_index_smi = CAST(Parameter(Descriptor::kTableIndex));
+  // If the entry is our placeholder for lazy function initialization, then we
+  // fall back to the runtime call.
+  TNode<Map> map = LoadReceiverMap(entry);
+  GotoIf(IsTuple2Map(map), &call_runtime);
 
-  TailCallRuntime(Runtime::kWasmFunctionTableGet, context, instance,
-                  table_index_smi, entry_index_smi);
+  Return(entry);
 
-  BIND(&entry_index_out_of_range);
+  BIND(&call_runtime);
+  // Fall back to the runtime call for more complex cases.
+  // table_index and entry_index must be in Smi range, due to checks above.
+  TailCallRuntime(Runtime::kWasmFunctionTableGet,
+                  LoadContextFromInstance(instance), instance,
+                  SmiFromIntPtr(table_index), SmiFromIntPtr(entry_index));
+
+  BIND(&index_out_of_range);
   MessageTemplate message_id =
       wasm::WasmOpcodes::TrapReasonToMessageId(wasm::kTrapTableOutOfBounds);
-  TailCallRuntime(Runtime::kThrowWasmError, context,
+  TailCallRuntime(Runtime::kThrowWasmError, LoadContextFromInstance(instance),
                   SmiConstant(static_cast<int>(message_id)));
 }
 
 TF_BUILTIN(WasmTableSet, WasmBuiltinsAssembler) {
-  TNode<Int32T> entry_index =
-      UncheckedCast<Int32T>(Parameter(Descriptor::kEntryIndex));
   TNode<WasmInstanceObject> instance = LoadInstanceFromFrame();
-  TNode<Context> context = LoadContextFromInstance(instance);
-  Label entry_index_out_of_range(this, Label::kDeferred);
 
-  TNode<BoolT> entry_index_fits_in_smi =
-      IsValidPositiveSmi(ChangeInt32ToIntPtr(entry_index));
-  GotoIfNot(entry_index_fits_in_smi, &entry_index_out_of_range);
+  Label call_runtime(this, Label::kDeferred),
+      index_out_of_range(this, Label::kDeferred);
+
+  TNode<IntPtrT> table_index =
+      UncheckedCast<IntPtrT>(Parameter(Descriptor::kTableIndex));
+  GotoIfNot(IsValidPositiveSmi(table_index), &index_out_of_range);
+  TNode<IntPtrT> entry_index = ChangeInt32ToIntPtr(
+      UncheckedCast<Int32T>(Parameter(Descriptor::kEntryIndex)));
+  GotoIfNot(IsValidPositiveSmi(entry_index), &index_out_of_range);
 
-  TNode<Smi> entry_index_smi = SmiFromInt32(entry_index);
-  TNode<Smi> table_index_smi = CAST(Parameter(Descriptor::kTableIndex));
   TNode<Object> value = CAST(Parameter(Descriptor::kValue));
-  TailCallRuntime(Runtime::kWasmFunctionTableSet, context, instance,
-                  table_index_smi, entry_index_smi, value);
 
-  BIND(&entry_index_out_of_range);
+  TNode<FixedArray> tables_array =
+      LoadObjectField<FixedArray>(instance, WasmInstanceObject::kTablesOffset);
+  TNode<WasmTableObject> table =
+      CAST(LoadFixedArrayElement(tables_array, table_index));
+  // Fall back to the runtime to set funcrefs, since we have to update function
+  // dispatch tables.
+  TNode<Smi> table_type =
+      LoadObjectField<Smi>(table, WasmTableObject::kRawTypeOffset);
+  GotoIf(SmiEqual(table_type, SmiConstant(wasm::ValueType::Kind::kFuncRef)),
+         &call_runtime);
+
+  TNode<IntPtrT> entries_length =
+      LoadAndUntagObjectField(table, WasmTableObject::kCurrentLengthOffset);
+  GotoIfNot(IntPtrLessThan(entry_index, entries_length), &index_out_of_range);
+
+  TNode<FixedArray> entries_array =
+      LoadObjectField<FixedArray>(table, WasmTableObject::kEntriesOffset);
+
+  StoreFixedArrayElement(entries_array, entry_index, value);
+  Return(UndefinedConstant());
+
+  BIND(&call_runtime);
+  // Fall back to the runtime call for more complex cases.
+  // table_index and entry_index must be in Smi range, due to checks above.
+  TailCallRuntime(
+      Runtime::kWasmFunctionTableSet, LoadContextFromInstance(instance),
+      instance, SmiFromIntPtr(table_index), SmiFromIntPtr(entry_index), value);
+
+  BIND(&index_out_of_range);
   MessageTemplate message_id =
       wasm::WasmOpcodes::TrapReasonToMessageId(wasm::kTrapTableOutOfBounds);
-  TailCallRuntime(Runtime::kThrowWasmError, context,
+  TailCallRuntime(Runtime::kThrowWasmError, LoadContextFromInstance(instance),
                   SmiConstant(static_cast<int>(message_id)));
 }
 

src/codegen/interface-descriptors.h

@@ -1403,18 +1403,18 @@ class WasmTableCopyDescriptor final : public CallInterfaceDescriptor {
 class WasmTableGetDescriptor final : public CallInterfaceDescriptor {
  public:
   DEFINE_PARAMETERS_NO_CONTEXT(kTableIndex, kEntryIndex)
-  DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::AnyTagged(),     // result 1
-                                    MachineType::TaggedSigned(),  // kTableIndex
-                                    MachineType::Int32())         // kEntryIndex
+  DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::AnyTagged(),  // result
+                                    MachineType::IntPtr(),     // kTableIndex
+                                    MachineType::Int32())      // kEntryIndex
   DECLARE_DESCRIPTOR(WasmTableGetDescriptor, CallInterfaceDescriptor)
 };
 
 class WasmTableSetDescriptor final : public CallInterfaceDescriptor {
  public:
   DEFINE_PARAMETERS_NO_CONTEXT(kTableIndex, kEntryIndex, kValue)
-  DEFINE_PARAMETER_TYPES(MachineType::TaggedSigned(),  // kTableIndex
-                         MachineType::Int32(),         // kEntryIndex
-                         MachineType::AnyTagged())     // kValue
+  DEFINE_PARAMETER_TYPES(MachineType::IntPtr(),     // kTableIndex
+                         MachineType::Int32(),      // kEntryIndex
+                         MachineType::AnyTagged())  // kValue
   DECLARE_DESCRIPTOR(WasmTableSetDescriptor, CallInterfaceDescriptor)
 };
 

src/compiler/wasm-compiler.cc

@@ -1938,6 +1938,19 @@ ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
   }
 }
 
+template <typename BuiltinDescriptor>
+CallDescriptor* GetBuiltinCallDescriptor(WasmGraphBuilder* builder,
+                                         StubCallMode stub_mode) {
+  BuiltinDescriptor interface_descriptor;
+  return Linkage::GetStubCallDescriptor(
+      builder->mcgraph()->zone(),                     // zone
+      interface_descriptor,                           // descriptor
+      interface_descriptor.GetStackParameterCount(),  // stack parameter count
+      CallDescriptor::kNoFlags,                       // flags
+      Operator::kNoProperties,                        // properties
+      stub_mode);                                     // stub call mode
+}
+
 }  // namespace
 
 Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
@@ -3348,73 +3361,10 @@ Node* WasmGraphBuilder::GlobalSet(uint32_t index, Node* val) {
       graph()->NewNode(op, base, offset, val, effect(), control()));
 }
 
-void WasmGraphBuilder::BoundsCheckTable(uint32_t table_index, Node* entry_index,
-                                        wasm::WasmCodePosition position,
-                                        wasm::TrapReason trap_reason,
-                                        Node** base_node) {
-  Node* tables = LOAD_INSTANCE_FIELD(Tables, MachineType::TaggedPointer());
-  Node* table = LOAD_FIXED_ARRAY_SLOT_ANY(tables, table_index);
-
-  int length_field_size = WasmTableObject::kCurrentLengthOffsetEnd -
-                          WasmTableObject::kCurrentLengthOffset + 1;
-  Node* length_smi = gasm_->Load(
-      assert_size(length_field_size, MachineType::TaggedSigned()), table,
-      wasm::ObjectAccess::ToTagged(WasmTableObject::kCurrentLengthOffset));
-  Node* length = BuildChangeSmiToInt32(length_smi);
-
-  // Bounds check against the table size.
-  Node* in_bounds = graph()->NewNode(mcgraph()->machine()->Uint32LessThan(),
-                                     entry_index, length);
-  TrapIfFalse(trap_reason, in_bounds, position);
-
-  if (base_node) {
-    int storage_field_size = WasmTableObject::kEntriesOffsetEnd -
-                             WasmTableObject::kEntriesOffset + 1;
-    *base_node = gasm_->Load(
-        assert_size(storage_field_size, MachineType::TaggedPointer()), table,
-        wasm::ObjectAccess::ToTagged(WasmTableObject::kEntriesOffset));
-  }
-}
-
-void WasmGraphBuilder::GetTableBaseAndOffset(uint32_t table_index,
-                                             Node* entry_index,
-                                             wasm::WasmCodePosition position,
-                                             Node** base_node,
-                                             Node** offset_node) {
-  BoundsCheckTable(table_index, entry_index, position,
-                   wasm::kTrapTableOutOfBounds, base_node);
-  // From the index, calculate the actual offset in the FixeArray. This
-  // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
-  // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
-  Node* index_times_tagged_size = graph()->NewNode(
-      mcgraph()->machine()->IntMul(), Uint32ToUintptr(entry_index),
-      mcgraph()->Int32Constant(kTaggedSize));
-
-  *offset_node = graph()->NewNode(
-      mcgraph()->machine()->IntAdd(), index_times_tagged_size,
-      mcgraph()->IntPtrConstant(
-          wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
-}
-
 Node* WasmGraphBuilder::TableGet(uint32_t table_index, Node* index,
                                  wasm::WasmCodePosition position) {
-  if (env_->module->tables[table_index].type == wasm::kWasmAnyRef ||
-      env_->module->tables[table_index].type == wasm::kWasmNullRef ||
-      env_->module->tables[table_index].type == wasm::kWasmExnRef) {
-    Node* base = nullptr;
-    Node* offset = nullptr;
-    GetTableBaseAndOffset(table_index, index, position, &base, &offset);
-    return gasm_->Load(MachineType::AnyTagged(), base, offset);
-  }
-  // We access funcref tables through runtime calls.
-  WasmTableGetDescriptor interface_descriptor;
-  auto call_descriptor = Linkage::GetStubCallDescriptor(
-      mcgraph()->zone(),                              // zone
-      interface_descriptor,                           // descriptor
-      interface_descriptor.GetStackParameterCount(),  // stack parameter count
-      CallDescriptor::kNoFlags,                       // flags
-      Operator::kNoProperties,                        // properties
-      StubCallMode::kCallWasmRuntimeStub);            // stub call mode
+  auto call_descriptor = GetBuiltinCallDescriptor<WasmTableGetDescriptor>(
+      this, StubCallMode::kCallWasmRuntimeStub);
   // A direct call to a wasm runtime stub defined in this module.
   // Just encode the stub index. This will be patched at relocation.
   Node* call_target = mcgraph()->RelocatableIntPtrConstant(
@@ -3422,40 +3372,21 @@ Node* WasmGraphBuilder::TableGet(uint32_t table_index, Node* index,
 
   return SetEffectControl(graph()->NewNode(
       mcgraph()->common()->Call(call_descriptor), call_target,
-      graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)), index,
-      effect(), control()));
+      IntPtrConstant(table_index), index, effect(), control()));
 }
 
 Node* WasmGraphBuilder::TableSet(uint32_t table_index, Node* index, Node* val,
                                  wasm::WasmCodePosition position) {
-  if (env_->module->tables[table_index].type == wasm::kWasmAnyRef ||
-      env_->module->tables[table_index].type == wasm::kWasmNullRef ||
-      env_->module->tables[table_index].type == wasm::kWasmExnRef) {
-    Node* base = nullptr;
-    Node* offset = nullptr;
-    GetTableBaseAndOffset(table_index, index, position, &base, &offset);
-    return STORE_RAW_NODE_OFFSET(
-        base, offset, val, MachineRepresentation::kTagged, kFullWriteBarrier);
-  } else {
-    // We access funcref tables through runtime calls.
-    WasmTableSetDescriptor interface_descriptor;
-    auto call_descriptor = Linkage::GetStubCallDescriptor(
-        mcgraph()->zone(),                              // zone
-        interface_descriptor,                           // descriptor
-        interface_descriptor.GetStackParameterCount(),  // stack parameter count
-        CallDescriptor::kNoFlags,                       // flags
-        Operator::kNoProperties,                        // properties
-        StubCallMode::kCallWasmRuntimeStub);            // stub call mode
-    // A direct call to a wasm runtime stub defined in this module.
-    // Just encode the stub index. This will be patched at relocation.
-    Node* call_target = mcgraph()->RelocatableIntPtrConstant(
-        wasm::WasmCode::kWasmTableSet, RelocInfo::WASM_STUB_CALL);
+  auto call_descriptor = GetBuiltinCallDescriptor<WasmTableSetDescriptor>(
+      this, StubCallMode::kCallWasmRuntimeStub);
+  // A direct call to a wasm runtime stub defined in this module.
+  // Just encode the stub index. This will be patched at relocation.
+  Node* call_target = mcgraph()->RelocatableIntPtrConstant(
+      wasm::WasmCode::kWasmTableSet, RelocInfo::WASM_STUB_CALL);
 
-    return SetEffectControl(graph()->NewNode(
-        mcgraph()->common()->Call(call_descriptor), call_target,
-        graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
-        index, val, effect(), control()));
-  }
+  return SetEffectControl(graph()->NewNode(
+      mcgraph()->common()->Call(call_descriptor), call_target,
+      IntPtrConstant(table_index), index, val, effect(), control()));
 }
 
 Node* WasmGraphBuilder::CheckBoundsAndAlignment(
@@ -4029,19 +3960,6 @@ Signature<MachineRepresentation>* CreateMachineSignature(
   return builder.Build();
 }
 
-template <typename BuiltinDescriptor>
-CallDescriptor* GetBuiltinCallDescriptor(WasmGraphBuilder* builder,
-                                         StubCallMode stub_mode) {
-  BuiltinDescriptor interface_descriptor;
-  return Linkage::GetStubCallDescriptor(
-      builder->mcgraph()->zone(),                     // zone
-      interface_descriptor,                           // descriptor
-      interface_descriptor.GetStackParameterCount(),  // stack parameter count
-      CallDescriptor::kNoFlags,                       // flags
-      Operator::kNoProperties,                        // properties
-      stub_mode);                                     // stub call mode
-}
-
 }  // namespace
 
 void WasmGraphBuilder::AddInt64LoweringReplacement(

src/compiler/wasm-compiler.h

@@ -321,14 +321,6 @@ class WasmGraphBuilder {
   void GetBaseAndOffsetForImportedMutableAnyRefGlobal(
       const wasm::WasmGlobal& global, Node** base, Node** offset);
 
-  void BoundsCheckTable(uint32_t table_index, Node* index,
-                        wasm::WasmCodePosition position,
-                        wasm::TrapReason trap_reason, Node** base_node);
-
-  void GetTableBaseAndOffset(uint32_t table_index, Node* index,
-                             wasm::WasmCodePosition position, Node** base_node,
-                             Node** offset_node);
-
   // Utilities to manipulate sets of instance cache nodes.
   void InitInstanceCache(WasmInstanceCacheNodes* instance_cache);
   void PrepareInstanceCacheForLoop(WasmInstanceCacheNodes* instance_cache,

src/runtime/runtime-wasm.cc

@@ -487,6 +487,8 @@ RUNTIME_FUNCTION(Runtime_WasmFunctionTableGet) {
   DCHECK_LT(table_index, instance->tables().length());
   auto table = handle(
       WasmTableObject::cast(instance->tables().get(table_index)), isolate);
+  // We only use the runtime call for lazily initialized function references.
+  DCHECK_EQ(table->type(), wasm::kWasmFuncRef);
 
   if (!WasmTableObject::IsInBounds(isolate, table, entry_index)) {
     return ThrowWasmError(isolate, MessageTemplate::kWasmTrapTableOutOfBounds);
@@ -508,6 +510,8 @@ RUNTIME_FUNCTION(Runtime_WasmFunctionTableSet) {
   DCHECK_LT(table_index, instance->tables().length());
   auto table = handle(
       WasmTableObject::cast(instance->tables().get(table_index)), isolate);
+  // We only use the runtime call for function references.
+  DCHECK_EQ(table->type(), wasm::kWasmFuncRef);
 
   if (!WasmTableObject::IsInBounds(isolate, table, entry_index)) {
     return ThrowWasmError(isolate, MessageTemplate::kWasmTrapTableOutOfBounds);