[wasm-gc] Speculative inlining for call_ref (off by default)

This patch adds infrastructure for collecting feedback about call_ref
call targets in Liftoff code, and using that feedback for turning
such calls into inlineable direct calls when building Turbofan graphs.
The feature is considered experimental quality and hence off by default,
--wasm-speculative-inlining turns it on.

Bug: v8:7748
Change-Id: I0d0d776f8a71c3dd2c9124d3731f3cb06d4f5821
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3205902
Commit-Queue: Jakob Kummerow <jkummerow@chromium.org>
Reviewed-by: Manos Koukoutos <manoskouk@chromium.org>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Maya Lekova <mslekova@chromium.org>
Cr-Commit-Position: refs/heads/main@{#77287}

src/builtins/wasm.tq

@@ -453,6 +453,111 @@ builtin WasmI64AtomicWait64(
   }
 }
 
+// Type feedback collection support for `call_ref`.
+
+extern macro GetCodeEntry(Code): RawPtr;
+extern macro GetCodeEntry(CodeDataContainer): RawPtr;
+
+struct TargetAndInstance {
+  target: RawPtr;
+  instance: HeapObject;  // WasmInstanceObject or Tuple2
+}
+
+macro GetTargetAndInstance(funcref: JSFunction): TargetAndInstance {
+  const sfi = funcref.shared_function_info;
+  dcheck(Is<WasmFunctionData>(sfi.function_data));
+  const funcData = UnsafeCast<WasmFunctionData>(sfi.function_data);
+  const ref = funcData.ref;
+  if (Is<Tuple2>(ref)) {
+    const instance: WasmInstanceObject = LoadInstanceFromFrame();
+    UnsafeCast<Tuple2>(ref).value1 = instance;
+  }
+  let target = funcData.foreign_address_ptr;
+  if (Signed(target) == IntPtrConstant(0)) {
+    const wrapper =
+        UnsafeCast<WasmJSFunctionData>(funcData).wasm_to_js_wrapper_code;
+    target = GetCodeEntry(wrapper);
+  }
+  return TargetAndInstance{target: target, instance: ref};
+}
+
+// Vector format:
+// Two slots per call_ref instruction. These slots' values can be:
+// - uninitialized: (undefined, <unused>). Note: we use {undefined} as the
+//   sentinel as an optimization, as it's the default value for FixedArrays.
+// - monomorphic: (funcref, call_ref_data)
+// - polymorphic: (fixed_array, <unused>). In this case, the array
+//   contains 2..4 pairs (funcref, call_ref_data) (like monomorphic data).
+// - megamorphic: ("megamorphic" sentinel, <unused>)
+
+builtin CallRefIC(
+    vector: FixedArray, index: intptr, funcref: JSFunction): TargetAndInstance {
+  const value = vector.objects[index];
+  if (value == funcref) {
+    // Monomorphic hit. Check for this case first to maximize its performance.
+    const data = UnsafeCast<CallRefData>(vector.objects[index + 1]);
+    data.count = data.count + 1;
+    return TargetAndInstance{target: data.target, instance: data.instance};
+  }
+  // Check for polymorphic hit; its performance is second-most-important.
+  if (Is<FixedArray>(value)) {
+    const entries = UnsafeCast<FixedArray>(value);
+    for (let i: intptr = 0; i < entries.length_intptr; i += 2) {
+      if (entries.objects[i] == funcref) {
+        // Polymorphic hit.
+        const data = UnsafeCast<CallRefData>(entries.objects[i + 1]);
+        data.count = data.count + 1;
+        return TargetAndInstance{target: data.target, instance: data.instance};
+      }
+    }
+  }
+  // All other cases are some sort of miss and must compute the target/
+  // instance. They all fall through to returning the computed data.
+  const result = GetTargetAndInstance(funcref);
+  if (TaggedEqual(value, Undefined)) {
+    const data = new
+    CallRefData{instance: result.instance, target: result.target, count: 1};
+    vector.objects[index] = funcref;
+    vector.objects[index + 1] = data;
+  } else if (Is<FixedArray>(value)) {
+    // Polymorphic miss.
+    const entries = UnsafeCast<FixedArray>(value);
+    if (entries.length == SmiConstant(8)) {  // 4 entries, 2 slots each.
+      vector.objects[index] = ic::kMegamorphicSymbol;
+      vector.objects[index + 1] = ic::kMegamorphicSymbol;
+    } else {
+      const data = new
+      CallRefData{instance: result.instance, target: result.target, count: 1};
+      const newEntries = UnsafeCast<FixedArray>(AllocateFixedArray(
+          ElementsKind::PACKED_ELEMENTS, entries.length_intptr + 2,
+          AllocationFlag::kNone));
+      for (let i: intptr = 0; i < entries.length_intptr; i++) {
+        newEntries.objects[i] = entries.objects[i];
+      }
+      const newIndex = entries.length_intptr;
+      newEntries.objects[newIndex] = funcref;
+      newEntries.objects[newIndex + 1] = data;
+      vector.objects[index] = newEntries;
+    }
+  } else if (Is<JSFunction>(value)) {
+    // Monomorphic miss.
+    const data = new
+    CallRefData{instance: result.instance, target: result.target, count: 1};
+    const newEntries = UnsafeCast<FixedArray>(AllocateFixedArray(
+        ElementsKind::PACKED_ELEMENTS, 4, AllocationFlag::kNone));
+    newEntries.objects[0] = value;
+    newEntries.objects[1] = vector.objects[index + 1];
+    newEntries.objects[2] = funcref;
+    newEntries.objects[3] = data;
+    vector.objects[index] = newEntries;
+    // Clear the old pointer to the first entry's data object; the specific
+    // value we write doesn't matter.
+    vector.objects[index + 1] = Undefined;
+  }
+  // The "ic::IsMegamorphic(value)" case doesn't need to do anything.
+  return result;
+}
+
 extern macro TryHasOwnProperty(HeapObject, Map, InstanceType, Name): never
     labels Found, NotFound, Bailout;
 type OnNonExistent constexpr 'OnNonExistent';

src/compiler/code-assembler.cc

@@ -490,6 +490,15 @@ void CodeAssembler::Return(TNode<WordT> value1, TNode<WordT> value2) {
   return raw_assembler()->Return(value1, value2);
 }
 
+void CodeAssembler::Return(TNode<WordT> value1, TNode<Object> value2) {
+  DCHECK_EQ(2, raw_assembler()->call_descriptor()->ReturnCount());
+  DCHECK_EQ(
+      MachineType::PointerRepresentation(),
+      raw_assembler()->call_descriptor()->GetReturnType(0).representation());
+  DCHECK(raw_assembler()->call_descriptor()->GetReturnType(1).IsTagged());
+  return raw_assembler()->Return(value1, value2);
+}
+
 void CodeAssembler::PopAndReturn(Node* pop, Node* value) {
   DCHECK_EQ(1, raw_assembler()->call_descriptor()->ReturnCount());
   return raw_assembler()->PopAndReturn(pop, value);

src/compiler/code-assembler.h

@@ -627,6 +627,7 @@ class V8_EXPORT_PRIVATE CodeAssembler {
   void Return(TNode<Float32T> value);
   void Return(TNode<Float64T> value);
   void Return(TNode<WordT> value1, TNode<WordT> value2);
+  void Return(TNode<WordT> value1, TNode<Object> value2);
   void PopAndReturn(Node* pop, Node* value);
 
   void ReturnIf(TNode<BoolT> condition, TNode<Object> value);

src/compiler/pipeline.cc

@@ -1696,8 +1696,8 @@ struct WasmInliningPhase {
         data->jsgraph()->Dead(), data->observe_node_manager());
     DeadCodeElimination dead(&graph_reducer, data->graph(),
                              data->mcgraph()->common(), temp_zone);
-    // For now, hard-code inlining the function at index 0.
-    InlineByIndex heuristics({0});
+    // For now, inline the first few functions;
+    InlineFirstFew heuristics(FLAG_wasm_inlining_budget);
     WasmInliner inliner(&graph_reducer, env, data->source_positions(),
                         data->node_origins(), data->mcgraph(), wire_bytes,
                         &heuristics);

src/compiler/wasm-inlining.cc

@@ -27,6 +27,11 @@ Reduction WasmInliner::Reduce(Node* node) {
   }
 }
 
+#define TRACE(...)                            \
+  if (FLAG_trace_wasm_speculative_inlining) { \
+    PrintF(__VA_ARGS__);                      \
+  }
+
 // TODO(12166): Save inlined frames for trap/--trace-wasm purposes. Consider
 //              tail calls.
 // TODO(12166): Inline indirect calls/call_ref.
@@ -40,10 +45,22 @@ Reduction WasmInliner::ReduceCall(Node* call) {
   if (callee->opcode() != reloc_opcode) return NoChange();
   auto info = OpParameter<RelocatablePtrConstantInfo>(callee->op());
   uint32_t inlinee_index = static_cast<uint32_t>(info.value());
+  TRACE("[considering call to %d... ", inlinee_index)
+  if (info.rmode() != RelocInfo::WASM_CALL) {
+    TRACE("not a wasm call]\n")
+    return NoChange();
+  }
+  if (inlinee_index < module()->num_imported_functions) {
+    TRACE("imported function]\n")
+    return NoChange();
+  }
+
   if (!heuristics_->DoInline(source_positions_->GetSourcePosition(call),
                              inlinee_index)) {
+    TRACE("heuristics say no]\n")
     return NoChange();
   }
+  TRACE("inlining!]\n")
 
   CHECK_LT(inlinee_index, module()->functions.size());
   const wasm::WasmFunction* inlinee = &module()->functions[inlinee_index];

src/compiler/wasm-inlining.h

@@ -33,26 +33,18 @@ class SourcePositionTable;
 // Parent class for classes that provide heuristics on how to inline in wasm.
 class WasmInliningHeuristics {
  public:
-  virtual bool DoInline(SourcePosition position,
-                        uint32_t function_index) const = 0;
+  virtual bool DoInline(SourcePosition position, uint32_t function_index) = 0;
 };
 
-// A simple inlining heuristic that inlines all function calls to a set of given
-// function indices.
-class InlineByIndex : public WasmInliningHeuristics {
+class InlineFirstFew : public WasmInliningHeuristics {
  public:
-  explicit InlineByIndex(uint32_t function_index)
-      : WasmInliningHeuristics(), function_indices_(function_index) {}
-  InlineByIndex(std::initializer_list<uint32_t> function_indices)
-      : WasmInliningHeuristics(), function_indices_(function_indices) {}
-
-  bool DoInline(SourcePosition position,
-                uint32_t function_index) const override {
-    return function_indices_.count(function_index) > 0;
+  explicit InlineFirstFew(int count) : count_(count) {}
+  bool DoInline(SourcePosition position, uint32_t function_index) override {
+    return count_-- > 0;
   }
 
  private:
-  std::unordered_set<uint32_t> function_indices_;
+  int count_;
 };
 
 // The WasmInliner provides the core graph inlining machinery for Webassembly
@@ -65,7 +57,7 @@ class WasmInliner final : public AdvancedReducer {
               SourcePositionTable* source_positions,
               NodeOriginTable* node_origins, MachineGraph* mcgraph,
               const wasm::WireBytesStorage* wire_bytes,
-              const WasmInliningHeuristics* heuristics)
+              WasmInliningHeuristics* heuristics)
       : AdvancedReducer(editor),
         env_(env),
         source_positions_(source_positions),
@@ -98,7 +90,7 @@ class WasmInliner final : public AdvancedReducer {
   NodeOriginTable* const node_origins_;
   MachineGraph* const mcgraph_;
   const wasm::WireBytesStorage* const wire_bytes_;
-  const WasmInliningHeuristics* const heuristics_;
+  WasmInliningHeuristics* heuristics_;
 };
 
 }  // namespace compiler

src/diagnostics/objects-printer.cc

@@ -1868,6 +1868,11 @@ void WasmInstanceObject::WasmInstanceObjectPrint(std::ostream& os) {
     os << "\n - managed_native_allocations: "
        << Brief(managed_native_allocations());
   }
+  if (has_tags_table()) {
+    os << "\n - tags table: " << Brief(tags_table());
+  }
+  os << "\n - managed object maps: " << Brief(managed_object_maps());
+  os << "\n - feedback vectors: " << Brief(feedback_vectors());
   os << "\n - memory_start: " << static_cast<void*>(memory_start());
   os << "\n - memory_size: " << memory_size();
   os << "\n - imported_function_targets: "

src/flags/flag-definitions.h

@@ -1046,8 +1046,16 @@ DEFINE_BOOL(wasm_math_intrinsics, true,
 DEFINE_BOOL(
     wasm_inlining, false,
     "enable inlining of wasm functions into wasm functions (experimental)")
+DEFINE_INT(wasm_inlining_budget, 3,
+           "maximum number of call targets to inline into a Wasm function")
 DEFINE_BOOL(wasm_speculative_inlining, false,
             "enable speculative inlining of call_ref targets (experimental)")
+DEFINE_BOOL(trace_wasm_speculative_inlining, false,
+            "trace wasm speculative inlining")
+DEFINE_IMPLICATION(wasm_speculative_inlining, experimental_wasm_typed_funcref)
+DEFINE_IMPLICATION(wasm_speculative_inlining, wasm_inlining)
+DEFINE_IMPLICATION(wasm_speculative_inlining, wasm_dynamic_tiering)
+DEFINE_NEG_IMPLICATION(wasm_speculative_inlining, wasm_tier_up)
 DEFINE_BOOL(wasm_loop_unrolling, true,
             "enable loop unrolling for wasm functions")
 DEFINE_BOOL(wasm_fuzzer_gen_test, false,

src/runtime/runtime-wasm.cc

@@ -208,15 +208,15 @@ RUNTIME_FUNCTION(Runtime_WasmCompileLazy) {
 
   DCHECK(isolate->context().is_null());
   isolate->set_context(instance->native_context());
-  Handle<WasmModuleObject> module_object{instance->module_object(), isolate};
-  bool success = wasm::CompileLazy(isolate, module_object, func_index);
+  bool success = wasm::CompileLazy(isolate, instance, func_index);
   if (!success) {
     DCHECK(isolate->has_pending_exception());
     return ReadOnlyRoots(isolate).exception();
   }
 
   Address entrypoint =
-      module_object->native_module()->GetCallTargetForFunction(func_index);
+      instance->module_object().native_module()->GetCallTargetForFunction(
+          func_index);
 
   return Object(entrypoint);
 }
@@ -295,7 +295,7 @@ RUNTIME_FUNCTION(Runtime_WasmTriggerTierUp) {
   int func_index = frame_finder.frame()->function_index();
   auto* native_module = instance->module_object().native_module();
 
-  wasm::TriggerTierUp(isolate, native_module, func_index);
+  wasm::TriggerTierUp(isolate, native_module, func_index, instance);
 
   return ReadOnlyRoots(isolate).undefined_value();
 }

src/wasm/function-compiler.h

@@ -57,6 +57,7 @@ struct WasmCompilationResult {
   ExecutionTier result_tier;
   Kind kind = kFunction;
   ForDebugging for_debugging = kNoDebugging;
+  int feedback_vector_slots = 0;
 };
 
 class V8_EXPORT_PRIVATE WasmCompilationUnit final {

src/wasm/graph-builder-interface.cc

@@ -116,12 +116,25 @@ class WasmGraphBuildingInterface {
         inlined_status_(inlined_status) {}
 
   void StartFunction(FullDecoder* decoder) {
-    // Get the branch hints map for this function (if available)
+    // Get the branch hints map and type feedback for this function (if
+    // available).
     if (decoder->module_) {
       auto branch_hints_it = decoder->module_->branch_hints.find(func_index_);
       if (branch_hints_it != decoder->module_->branch_hints.end()) {
         branch_hints_ = &branch_hints_it->second;
       }
+      TypeFeedbackStorage& feedbacks = decoder->module_->type_feedback;
+      base::MutexGuard mutex_guard(&feedbacks.mutex);
+      auto feedback = feedbacks.feedback_for_function.find(func_index_);
+      if (feedback != feedbacks.feedback_for_function.end()) {
+        type_feedback_ = std::move(feedback->second);
+        // Erasing the map entry means that if the same function later gets
+        // inlined, its inlined copy won't have any type feedback available.
+        // However, if we don't erase the entry now, we'll be stuck with it
+        // forever.
+        // TODO(jkummerow): Reconsider our options here.
+        feedbacks.feedback_for_function.erase(func_index_);
+      }
     }
     // The first '+ 1' is needed by TF Start node, the second '+ 1' is for the
     // instance parameter.
@@ -657,7 +670,15 @@ class WasmGraphBuildingInterface {
   void CallRef(FullDecoder* decoder, const Value& func_ref,
                const FunctionSig* sig, uint32_t sig_index, const Value args[],
                Value returns[]) {
-    if (!FLAG_wasm_inlining) {
+    int maybe_feedback = -1;
+    // TODO(jkummerow): The way we currently prepare type feedback means that
+    // we won't have any for inlined functions. Figure out how to change that.
+    if (FLAG_wasm_speculative_inlining && type_feedback_.size() > 0) {
+      DCHECK_LT(feedback_instruction_index_, type_feedback_.size());
+      maybe_feedback = type_feedback_[feedback_instruction_index_];
+      feedback_instruction_index_++;
+    }
+    if (maybe_feedback == -1) {
       DoCall(decoder, CallInfo::CallRef(func_ref, NullCheckFor(func_ref.type)),
              sig, args, returns);
       return;
@@ -665,9 +686,14 @@ class WasmGraphBuildingInterface {
 
     // Check for equality against a function at a specific index, and if
     // successful, just emit a direct call.
-    // TODO(12166): For now, we check against function 0. Decide the index based
-    // on liftoff feedback.
-    const uint32_t expected_function_index = 0;
+    DCHECK_GE(maybe_feedback, 0);
+    const uint32_t expected_function_index = maybe_feedback;
+
+    if (FLAG_trace_wasm_speculative_inlining) {
+      PrintF("[Function #%d call #%d: graph support for inlining target #%d]\n",
+             func_index_, feedback_instruction_index_ - 1,
+             expected_function_index);
+    }
 
     TFNode* success_control;
     TFNode* failure_control;
@@ -715,7 +741,13 @@ class WasmGraphBuildingInterface {
   void ReturnCallRef(FullDecoder* decoder, const Value& func_ref,
                      const FunctionSig* sig, uint32_t sig_index,
                      const Value args[]) {
-    if (!FLAG_wasm_inlining) {
+    int maybe_feedback = -1;
+    if (FLAG_wasm_speculative_inlining) {
+      DCHECK_LE(feedback_instruction_index_, type_feedback_.size());
+      maybe_feedback = type_feedback_[feedback_instruction_index_];
+      feedback_instruction_index_++;
+    }
+    if (maybe_feedback == -1) {
       DoReturnCall(decoder,
                    CallInfo::CallRef(func_ref, NullCheckFor(func_ref.type)),
                    sig, args);
@@ -724,9 +756,14 @@ class WasmGraphBuildingInterface {
 
     // Check for equality against a function at a specific index, and if
     // successful, just emit a direct call.
-    // TODO(12166): For now, we check against function 0. Decide the index based
-    // on liftoff feedback.
-    const uint32_t expected_function_index = 0;
+    DCHECK_GE(maybe_feedback, 0);
+    const uint32_t expected_function_index = maybe_feedback;
+
+    if (FLAG_trace_wasm_speculative_inlining) {
+      PrintF("[Function #%d call #%d: graph support for inlining target #%d]\n",
+             func_index_, feedback_instruction_index_ - 1,
+             expected_function_index);
+    }
 
     TFNode* success_control;
     TFNode* failure_control;
@@ -1251,6 +1288,10 @@ class WasmGraphBuildingInterface {
   // Tracks loop data for loop unrolling.
   std::vector<compiler::WasmLoopInfo> loop_infos_;
   InlinedStatus inlined_status_;
+  // The entries in {type_feedback_} are indexed by the position of feedback-
+  // consuming instructions (currently only call_ref).
+  int feedback_instruction_index_ = 0;
+  std::vector<int> type_feedback_;
 
   TFNode* effect() { return builder_->effect(); }
 

src/wasm/module-compiler.cc

@@ -1134,8 +1134,9 @@ bool IsLazyModule(const WasmModule* module) {
 
 }  // namespace
 
-bool CompileLazy(Isolate* isolate, Handle<WasmModuleObject> module_object,
+bool CompileLazy(Isolate* isolate, Handle<WasmInstanceObject> instance,
                  int func_index) {
+  Handle<WasmModuleObject> module_object(instance->module_object(), isolate);
   NativeModule* native_module = module_object->native_module();
   const WasmModule* module = native_module->module();
   auto enabled_features = native_module->enabled_features();
@@ -1197,6 +1198,15 @@ bool CompileLazy(Isolate* isolate, Handle<WasmModuleObject> module_object,
     return false;
   }
 
+  // Allocate feedback vector if needed.
+  if (result.feedback_vector_slots > 0) {
+    DCHECK(FLAG_wasm_speculative_inlining);
+    Handle<FixedArray> vector =
+        isolate->factory()->NewFixedArray(result.feedback_vector_slots);
+    instance->feedback_vectors().set(
+        declared_function_index(module, func_index), *vector);
+  }
+
   WasmCodeRefScope code_ref_scope;
   WasmCode* code;
   {
@@ -1228,16 +1238,100 @@ bool CompileLazy(Isolate* isolate, Handle<WasmModuleObject> module_object,
   return true;
 }
 
+std::vector<int> ProcessTypeFeedback(Isolate* isolate,
+                                     Handle<WasmInstanceObject> instance,
+                                     int func_index) {
+  int which_vector = declared_function_index(instance->module(), func_index);
+  Object maybe_feedback = instance->feedback_vectors().get(which_vector);
+  if (!maybe_feedback.IsFixedArray()) return {};
+  FixedArray feedback = FixedArray::cast(maybe_feedback);
+  std::vector<int> result(feedback.length() / 2);
+  int imported_functions =
+      static_cast<int>(instance->module()->num_imported_functions);
+  for (int i = 0; i < feedback.length(); i += 2) {
+    Object value = feedback.get(i);
+    if (WasmExportedFunction::IsWasmExportedFunction(value)) {
+      // Monomorphic. Mark the target for inlining if it's defined in the
+      // same module.
+      WasmExportedFunction target = WasmExportedFunction::cast(value);
+      if (target.instance() == *instance &&
+          target.function_index() >= imported_functions) {
+        if (FLAG_trace_wasm_speculative_inlining) {
+          PrintF("[Function #%d call_ref #%d inlineable (monomorphic)]\n",
+                 func_index, i / 2);
+        }
+        result[i / 2] = target.function_index();
+        continue;
+      }
+    } else if (value.IsFixedArray()) {
+      // Polymorphic. Pick a target for inlining if there is one that was
+      // seen for most calls, and matches the requirements of the monomorphic
+      // case.
+      FixedArray polymorphic = FixedArray::cast(value);
+      size_t total_count = 0;
+      for (int j = 0; j < polymorphic.length(); j += 2) {
+        total_count += CallRefData::cast(polymorphic.get(j + 1)).count();
+      }
+      int found_target = -1;
+      double best_frequency = 0;
+      for (int j = 0; j < polymorphic.length(); j += 2) {
+        uint32_t this_count = CallRefData::cast(polymorphic.get(j + 1)).count();
+        double frequency = static_cast<double>(this_count) / total_count;
+        if (frequency > best_frequency) best_frequency = frequency;
+        if (frequency < 0.8) continue;
+        Object maybe_target = polymorphic.get(j);
+        if (!WasmExportedFunction::IsWasmExportedFunction(maybe_target)) {
+          continue;
+        }
+        WasmExportedFunction target =
+            WasmExportedFunction::cast(polymorphic.get(j));
+        if (target.instance() != *instance ||
+            target.function_index() < imported_functions) {
+          continue;
+        }
+        found_target = target.function_index();
+        if (FLAG_trace_wasm_speculative_inlining) {
+          PrintF("[Function #%d call_ref #%d inlineable (polymorphic %f)]\n",
+                 func_index, i / 2, frequency);
+        }
+        break;
+      }
+      if (found_target >= 0) {
+        result[i / 2] = found_target;
+        continue;
+      } else if (FLAG_trace_wasm_speculative_inlining) {
+        PrintF("[Function #%d call_ref #%d: best frequency %f]\n", func_index,
+               i / 2, best_frequency);
+      }
+    }
+    // If we fall through to here, then this call isn't eligible for inlining.
+    // Possible reasons: uninitialized or megamorphic feedback; or monomorphic
+    // or polymorphic that didn't meet our requirements.
+    result[i / 2] = -1;
+  }
+  return result;
+}
+
 void TriggerTierUp(Isolate* isolate, NativeModule* native_module,
-                   int func_index) {
+                   int func_index, Handle<WasmInstanceObject> instance) {
   CompilationStateImpl* compilation_state =
       Impl(native_module->compilation_state());
   WasmCompilationUnit tiering_unit{func_index, ExecutionTier::kTurbofan,
                                    kNoDebugging};
 
+  const WasmModule* module = native_module->module();
+  if (FLAG_wasm_speculative_inlining) {
+    auto feedback = ProcessTypeFeedback(isolate, instance, func_index);
+    base::MutexGuard mutex_guard(&module->type_feedback.mutex);
+    // TODO(jkummerow): we could have collisions here if two different instances
+    // of the same module schedule tier-ups of the same function at the same
+    // time. If that ever becomes a problem, figure out a solution.
+    module->type_feedback.feedback_for_function[func_index] =
+        std::move(feedback);
+  }
+
   uint32_t* call_array = native_module->num_liftoff_function_calls_array();
-  int offset =
-      wasm::declared_function_index(native_module->module(), func_index);
+  int offset = wasm::declared_function_index(module, func_index);
 
   size_t priority =
       base::Relaxed_Load(reinterpret_cast<int*>(&call_array[offset]));

src/wasm/module-compiler.h

@@ -73,9 +73,10 @@ WasmCode* CompileImportWrapper(
 // Triggered by the WasmCompileLazy builtin. The return value indicates whether
 // compilation was successful. Lazy compilation can fail only if validation is
 // also lazy.
-bool CompileLazy(Isolate*, Handle<WasmModuleObject>, int func_index);
+bool CompileLazy(Isolate*, Handle<WasmInstanceObject>, int func_index);
 
-void TriggerTierUp(Isolate*, NativeModule*, int func_index);
+void TriggerTierUp(Isolate*, NativeModule*, int func_index,
+                   Handle<WasmInstanceObject> instance);
 
 template <typename Key, typename Hash>
 class WrapperQueue {

src/wasm/module-decoder.cc

@@ -711,6 +711,7 @@ class ModuleDecoderImpl : public Decoder {
                                         import->index,  // func_index
                                         0,              // sig_index
                                         {0, 0},         // code
+                                        0,              // feedback slots
                                         true,           // imported
                                         false,          // exported
                                         false});        // declared
@@ -805,6 +806,7 @@ class ModuleDecoderImpl : public Decoder {
                                     func_index,  // func_index
                                     0,           // sig_index
                                     {0, 0},      // code
+                                    0,           // feedback slots
                                     false,       // imported
                                     false,       // exported
                                     false});     // declared

src/wasm/module-instantiate.cc

@@ -705,6 +705,27 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() {
     instance->set_managed_object_maps(*maps);
   }
 
+  //--------------------------------------------------------------------------
+  // Allocate type feedback vectors for functions.
+  //--------------------------------------------------------------------------
+  if (FLAG_wasm_speculative_inlining) {
+    int num_functions = static_cast<int>(module_->num_declared_functions);
+    Handle<FixedArray> vectors =
+        isolate_->factory()->NewFixedArray(num_functions, AllocationType::kOld);
+    instance->set_feedback_vectors(*vectors);
+    for (int i = 0; i < num_functions; i++) {
+      int func_index = module_->num_imported_functions + i;
+      int slots = module_->functions[func_index].feedback_slots;
+      if (slots == 0) continue;
+      if (FLAG_trace_wasm_speculative_inlining) {
+        PrintF("[Function %d (declared %d): allocating %d feedback slots]\n",
+               func_index, i, slots);
+      }
+      Handle<FixedArray> feedback = isolate_->factory()->NewFixedArray(slots);
+      vectors->set(i, *feedback);
+    }
+  }
+
   //--------------------------------------------------------------------------
   // Process the initialization for the module's globals.
   //--------------------------------------------------------------------------

src/wasm/wasm-code-manager.cc

@@ -2266,6 +2266,13 @@ std::vector<std::unique_ptr<WasmCode>> NativeModule::AddCompiledCode(
   for (auto& result : results) {
     DCHECK(result.succeeded());
     total_code_space += RoundUp<kCodeAlignment>(result.code_desc.instr_size);
+    if (result.result_tier == ExecutionTier::kLiftoff) {
+      int index = result.func_index;
+      DCHECK(module()->functions[index].feedback_slots == 0 ||
+             module()->functions[index].feedback_slots ==
+                 result.feedback_vector_slots);
+      module()->functions[index].feedback_slots = result.feedback_vector_slots;
+    }
   }
   base::Vector<byte> code_space;
   NativeModule::JumpTablesRef jump_tables;

src/wasm/wasm-code-manager.h

@@ -86,6 +86,7 @@ struct WasmModule;
   V(WasmTraceMemory)                      \
   V(BigIntToI32Pair)                      \
   V(BigIntToI64)                          \
+  V(CallRefIC)                            \
   V(DoubleToI)                            \
   V(I32PairToBigInt)                      \
   V(I64ToBigInt)                          \

src/wasm/wasm-constants.h

@@ -161,7 +161,7 @@ constexpr int kAnonymousFuncIndex = -1;
 constexpr uint32_t kGenericWrapperBudget = 1000;
 
 #if V8_TARGET_ARCH_X64
-constexpr int32_t kOSRTargetOffset = 3 * kSystemPointerSize;
+constexpr int32_t kOSRTargetOffset = 4 * kSystemPointerSize;
 #endif
 
 }  // namespace wasm

src/wasm/wasm-module.h

@@ -9,6 +9,7 @@
 #ifndef V8_WASM_WASM_MODULE_H_
 #define V8_WASM_WASM_MODULE_H_
 
+#include <map>
 #include <memory>
 
 #include "src/base/optional.h"
@@ -62,6 +63,10 @@ struct WasmFunction {
   uint32_t func_index;     // index into the function table.
   uint32_t sig_index;      // index into the signature table.
   WireBytesRef code;       // code of this function.
+  // Required number of slots in a feedback vector. Marked {mutable} because
+  // this is computed late (by Liftoff compilation), when the rest of the
+  // {WasmFunction} is typically considered {const}.
+  mutable int feedback_slots;
   bool imported;
   bool exported;
   bool declared;
@@ -257,6 +262,12 @@ struct V8_EXPORT_PRIVATE WasmDebugSymbols {
   WireBytesRef external_url;
 };
 
+struct TypeFeedbackStorage {
+  std::map<uint32_t, std::vector<int>> feedback_for_function;
+  // Accesses to {feedback_for_function} are guarded by this mutex.
+  base::Mutex mutex;
+};
+
 struct WasmTable;
 
 // End of a chain of explicit supertypes.
@@ -364,6 +375,9 @@ struct V8_EXPORT_PRIVATE WasmModule {
   std::vector<WasmCompilationHint> compilation_hints;
   BranchHintInfo branch_hints;
   SignatureMap signature_map;  // canonicalizing map for signature indexes.
+  // Entries in this storage are short-lived: when tier-up of a function is
+  // scheduled, an entry is placed; the Turbofan graph builder consumes it.
+  mutable TypeFeedbackStorage type_feedback;
 
   ModuleOrigin origin = kWasmOrigin;  // origin of the module
   LazilyGeneratedNames lazily_generated_names;

src/wasm/wasm-objects-inl.h

@@ -253,6 +253,8 @@ OPTIONAL_ACCESSORS(WasmInstanceObject, wasm_external_functions, FixedArray,
                    kWasmExternalFunctionsOffset)
 ACCESSORS(WasmInstanceObject, managed_object_maps, FixedArray,
           kManagedObjectMapsOffset)
+ACCESSORS(WasmInstanceObject, feedback_vectors, FixedArray,
+          kFeedbackVectorsOffset)
 
 void WasmInstanceObject::clear_padding() {
   if (FIELD_SIZE(kOptionalPaddingOffset) != 0) {

src/wasm/wasm-objects.cc

@@ -1311,6 +1311,7 @@ Handle<WasmInstanceObject> WasmInstanceObject::New(
   instance->set_hook_on_function_call_address(
       isolate->debug()->hook_on_function_call_address());
   instance->set_managed_object_maps(*isolate->factory()->empty_fixed_array());
+  instance->set_feedback_vectors(*isolate->factory()->empty_fixed_array());
   instance->set_num_liftoff_function_calls_array(
       module_object->native_module()->num_liftoff_function_calls_array());
   instance->set_break_on_entry(module_object->script().break_on_entry());

src/wasm/wasm-objects.h

@@ -349,6 +349,7 @@ class V8_EXPORT_PRIVATE WasmInstanceObject : public JSObject {
   DECL_OPTIONAL_ACCESSORS(tags_table, FixedArray)
   DECL_OPTIONAL_ACCESSORS(wasm_external_functions, FixedArray)
   DECL_ACCESSORS(managed_object_maps, FixedArray)
+  DECL_ACCESSORS(feedback_vectors, FixedArray)
   DECL_PRIMITIVE_ACCESSORS(memory_start, byte*)
   DECL_PRIMITIVE_ACCESSORS(memory_size, size_t)
   DECL_PRIMITIVE_ACCESSORS(isolate_root, Address)
@@ -425,6 +426,7 @@ class V8_EXPORT_PRIVATE WasmInstanceObject : public JSObject {
   V(kTagsTableOffset, kTaggedSize)                                        \
   V(kWasmExternalFunctionsOffset, kTaggedSize)                            \
   V(kManagedObjectMapsOffset, kTaggedSize)                                \
+  V(kFeedbackVectorsOffset, kTaggedSize)                                  \
   V(kBreakOnEntryOffset, kUInt8Size)                                      \
   /* More padding to make the header pointer-size aligned */              \
   V(kHeaderPaddingOffset, POINTER_SIZE_PADDING(kHeaderPaddingOffset))     \
@@ -460,7 +462,8 @@ class V8_EXPORT_PRIVATE WasmInstanceObject : public JSObject {
       kManagedNativeAllocationsOffset,
       kTagsTableOffset,
       kWasmExternalFunctionsOffset,
-      kManagedObjectMapsOffset};
+      kManagedObjectMapsOffset,
+      kFeedbackVectorsOffset};
 
   const wasm::WasmModule* module();
 

src/wasm/wasm-objects.tq

@@ -157,3 +157,10 @@ extern class WasmArray extends WasmObject {
   @if(TAGGED_SIZE_8_BYTES) optional_padding: uint32;
   @ifnot(TAGGED_SIZE_8_BYTES) optional_padding: void;
 }
+
+@export
+class CallRefData extends HeapObject {
+  instance: HeapObject;
+  target: RawPtr;
+  count: uint32;
+}

test/cctest/wasm/wasm-run-utils.cc

@@ -148,6 +148,7 @@ uint32_t TestingModuleBuilder::AddFunction(const FunctionSig* sig,
                                      index,    // func_index
                                      0,        // sig_index
                                      {0, 0},   // code
+                                     0,        // feedback slots
                                      false,    // imported
                                      false,    // exported
                                      false});  // declared

test/common/wasm/wasm-interpreter.cc

@@ -4219,6 +4219,7 @@ ControlTransferMap WasmInterpreter::ComputeControlTransfersForTesting(
                         0,       // func_index
                         0,       // sig_index
                         {0, 0},  // code
+                        0,       // feedback slots
                         false,   // imported
                         false,   // exported
                         false};  // declared

test/unittests/wasm/function-body-decoder-unittest.cc

@@ -98,6 +98,7 @@ class TestModuleBuilder {
          static_cast<uint32_t>(mod.functions.size()),  // func_index
          sig_index,                                    // sig_index
          {0, 0},                                       // code
+         0,                                            // feedback slots
          false,                                        // import
          false,                                        // export
          declared});                                   // declared