[wasm][pgo] Introduce flags for PGO via local files

This adds two flags to dump "type feedback" (call targets and
frequencies) to a local file, or load it from there. This is meant for
experimentation only.

Some implications are removed, as (speculative) inlining now does not
require Liftoff any more, but can also use information from PGO.

R=jkummerow@chromium.org

Bug: v8:13209
Change-Id: I2d34233ce4077db61f5c237b1941136ac61d3b73
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3870470Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Commit-Queue: Clemens Backes <clemensb@chromium.org>
Cr-Commit-Position: refs/heads/main@{#82979}

src/flags/flag-definitions.h

@@ -1036,6 +1036,14 @@ DEFINE_INT(wasm_tier_mask_for_testing, 0,
 DEFINE_INT(wasm_debug_mask_for_testing, 0,
            "bitmask of functions to compile for debugging, only applies if the "
            "tier is Liftoff")
+// TODO(clemensb): Introduce experimental_wasm_pgo to read from a custom section
+// instead of from a local file.
+DEFINE_BOOL(
+    experimental_wasm_pgo_to_file, false,
+    "experimental: dump Wasm PGO information to a local file (for testing)")
+DEFINE_BOOL(
+    experimental_wasm_pgo_from_file, false,
+    "experimental: read and use Wasm PGO data from a local file (for testing)")
 
 DEFINE_BOOL(validate_asm, true, "validate asm.js modules before compiling")
 // asm.js validation is disabled since it triggers wasm code generation.
@@ -1102,15 +1110,10 @@ DEFINE_BOOL(trace_wasm_speculative_inlining, false,
 DEFINE_BOOL(trace_wasm_typer, false, "trace wasm typer")
 DEFINE_BOOL(wasm_type_canonicalization, false,
             "apply isorecursive canonicalization on wasm types")
-DEFINE_IMPLICATION(wasm_speculative_inlining, wasm_dynamic_tiering)
 DEFINE_IMPLICATION(wasm_speculative_inlining, wasm_inlining)
 DEFINE_WEAK_IMPLICATION(experimental_wasm_gc, wasm_speculative_inlining)
 DEFINE_WEAK_IMPLICATION(experimental_wasm_typed_funcref,
                         wasm_type_canonicalization)
-// Speculative inlining needs type feedback from Liftoff and compilation in
-// Turbofan.
-DEFINE_NEG_NEG_IMPLICATION(liftoff, wasm_speculative_inlining)
-DEFINE_NEG_IMPLICATION(liftoff_only, wasm_speculative_inlining)
 
 DEFINE_BOOL(wasm_loop_unrolling, true,
             "enable loop unrolling for wasm functions")

src/wasm/graph-builder-interface.cc

@@ -196,6 +196,9 @@ class WasmGraphBuildingInterface {
   void StartFunctionBody(FullDecoder* decoder, Control* block) {}
 
   void FinishFunction(FullDecoder*) {
+    if (v8_flags.wasm_speculative_inlining) {
+      DCHECK_EQ(feedback_instruction_index_, type_feedback_.size());
+    }
     if (inlined_status_ == kRegularFunction) {
       builder_->PatchInStackCheckIfNeeded();
     }

src/wasm/module-compiler.cc

@@ -1422,6 +1422,7 @@ void TransitiveTypeFeedbackProcessor::ProcessFunction(int func_index) {
   base::Vector<uint32_t> call_direct_targets =
       module_->type_feedback.feedback_for_function[func_index]
           .call_targets.as_vector();
+  DCHECK_EQ(feedback.length(), call_direct_targets.size() * 2);
   FeedbackMaker fm(instance_, func_index, feedback.length() / 2);
   for (int i = 0; i < feedback.length(); i += 2) {
     Object value = feedback.get(i);

src/wasm/module-instantiate.cc

@@ -452,7 +452,8 @@ MaybeHandle<WasmInstanceObject> InstantiateToInstanceObject(
                           memory_buffer);
   auto instance = builder.Build();
   if (!instance.is_null()) {
-    // Post tasks for lazy compilation metrics before we call the start function
+    // Post tasks for lazy compilation metrics before we call the start
+    // function.
     if (v8_flags.wasm_lazy_compilation &&
         module_object->native_module()
             ->ShouldLazyCompilationMetricsBeReported()) {

src/wasm/wasm-code-manager.cc

@@ -1884,6 +1884,11 @@ NativeModule::~NativeModule() {
   // {owned_code_}. The destructor of {WasmImportWrapperCache} still needs to
   // decrease reference counts on the {WasmCode} objects.
   import_wrapper_cache_.reset();
+
+  // If experimental PGO support is enabled, serialize the PGO data now.
+  if (V8_UNLIKELY(FLAG_experimental_wasm_pgo_to_file)) {
+    DumpProfileToFile(module_.get(), wire_bytes());
+  }
 }
 
 WasmCodeManager::WasmCodeManager()

src/wasm/wasm-engine.cc

@@ -530,21 +530,30 @@ MaybeHandle<WasmModuleObject> WasmEngine::SyncCompile(
   TRACE_EVENT1("v8.wasm", "wasm.SyncCompile", "id", compilation_id);
   v8::metrics::Recorder::ContextId context_id =
       isolate->GetOrRegisterRecorderContextId(isolate->native_context());
-  ModuleResult result =
-      DecodeWasmModule(enabled, bytes.start(), bytes.end(), false, kWasmOrigin,
-                       isolate->counters(), isolate->metrics_recorder(),
-                       context_id, DecodingMethod::kSync, allocator());
-  if (result.failed()) {
-    thrower->CompileFailed(result.error());
-    return {};
+  std::shared_ptr<WasmModule> module;
+  {
+    ModuleResult result = DecodeWasmModule(
+        enabled, bytes.start(), bytes.end(), false, kWasmOrigin,
+        isolate->counters(), isolate->metrics_recorder(), context_id,
+        DecodingMethod::kSync, allocator());
+    if (result.failed()) {
+      thrower->CompileFailed(result.error());
+      return {};
+    }
+    module = std::move(result).value();
+  }
+
+  // If experimental PGO via files is enabled, load profile information now.
+  if (V8_UNLIKELY(FLAG_experimental_wasm_pgo_from_file)) {
+    LoadProfileFromFile(module.get(), bytes.module_bytes());
   }
 
   // Transfer ownership of the WasmModule to the {Managed<WasmModule>} generated
   // in {CompileToNativeModule}.
   Handle<FixedArray> export_wrappers;
-  std::shared_ptr<NativeModule> native_module = CompileToNativeModule(
-      isolate, enabled, thrower, std::move(result).value(), bytes,
-      &export_wrappers, compilation_id, context_id);
+  std::shared_ptr<NativeModule> native_module =
+      CompileToNativeModule(isolate, enabled, thrower, std::move(module), bytes,
+                            &export_wrappers, compilation_id, context_id);
   if (!native_module) return {};
 
 #ifdef DEBUG

src/wasm/wasm-module.cc

@@ -16,6 +16,7 @@
 #include "src/wasm/wasm-code-manager.h"
 #include "src/wasm/wasm-init-expr.h"
 #include "src/wasm/wasm-js.h"
+#include "src/wasm/wasm-module-builder.h"  // For {ZoneBuffer}.
 #include "src/wasm/wasm-objects-inl.h"
 #include "src/wasm/wasm-result.h"
 #include "src/wasm/wasm-subtyping.h"
@@ -688,4 +689,146 @@ size_t GetWireBytesHash(base::Vector<const uint8_t> wire_bytes) {
       kZeroHashSeed);
 }
 
+base::OwnedVector<uint8_t> GetProfileData(const WasmModule* module) {
+  const TypeFeedbackStorage& type_feedback = module->type_feedback;
+  AccountingAllocator allocator;
+  Zone zone{&allocator, "wasm::GetProfileData"};
+  ZoneBuffer buffer{&zone};
+  base::MutexGuard mutex_guard{&type_feedback.mutex};
+
+  // Get an ordered list of function indexes, so we generate deterministic data.
+  std::vector<uint32_t> ordered_func_indexes;
+  ordered_func_indexes.reserve(type_feedback.feedback_for_function.size());
+  for (const auto& entry : type_feedback.feedback_for_function) {
+    ordered_func_indexes.push_back(entry.first);
+  }
+  std::sort(ordered_func_indexes.begin(), ordered_func_indexes.end());
+
+  buffer.write_u32v(static_cast<uint32_t>(ordered_func_indexes.size()));
+  for (const uint32_t func_index : ordered_func_indexes) {
+    buffer.write_u32v(func_index);
+    // Serialize {feedback_vector}.
+    const FunctionTypeFeedback& feedback =
+        type_feedback.feedback_for_function.at(func_index);
+    buffer.write_u32v(static_cast<uint32_t>(feedback.feedback_vector.size()));
+    for (const CallSiteFeedback& call_site_feedback :
+         feedback.feedback_vector) {
+      int cases = call_site_feedback.num_cases();
+      buffer.write_i32v(cases);
+      for (int i = 0; i < cases; ++i) {
+        buffer.write_i32v(call_site_feedback.function_index(i));
+        buffer.write_i32v(call_site_feedback.call_count(i));
+      }
+    }
+    // Serialize {call_targets}.
+    buffer.write_u32v(static_cast<uint32_t>(feedback.call_targets.size()));
+    for (uint32_t call_target : feedback.call_targets) {
+      buffer.write_u32v(call_target);
+    }
+  }
+  return base::OwnedVector<uint8_t>::Of(buffer);
+}
+
+void RestoreProfileData(WasmModule* module,
+                        base::Vector<uint8_t> profile_data) {
+  TypeFeedbackStorage& type_feedback = module->type_feedback;
+  Decoder decoder{profile_data.begin(), profile_data.end()};
+  uint32_t num_entries = decoder.consume_u32v("num function entries");
+  CHECK_LE(num_entries, module->num_declared_functions);
+  for (uint32_t missing_entries = num_entries; missing_entries > 0;
+       --missing_entries) {
+    uint32_t function_index = decoder.consume_u32v("function index");
+    CHECK(!type_feedback.feedback_for_function.count(function_index));
+    FunctionTypeFeedback& feedback =
+        type_feedback.feedback_for_function[function_index];
+    // Deserialize {feedback_vector}.
+    uint32_t feedback_vector_size =
+        decoder.consume_u32v("feedback vector size");
+    feedback.feedback_vector.resize(feedback_vector_size);
+    for (CallSiteFeedback& feedback : feedback.feedback_vector) {
+      int num_cases = decoder.consume_i32v("num cases");
+      if (num_cases == 0) continue;  // no feedback
+      if (num_cases == 1) {          // monomorphic
+        int called_function_index = decoder.consume_i32v("function index");
+        int call_count = decoder.consume_i32v("call count");
+        feedback = CallSiteFeedback{called_function_index, call_count};
+      } else {  // polymorphic
+        auto* polymorphic = new CallSiteFeedback::PolymorphicCase[num_cases];
+        for (int i = 0; i < num_cases; ++i) {
+          polymorphic[i].function_index =
+              decoder.consume_i32v("function index");
+          polymorphic[i].absolute_call_frequency =
+              decoder.consume_i32v("call count");
+        }
+        feedback = CallSiteFeedback{polymorphic, num_cases};
+      }
+    }
+    // Deserialize {call_targets}.
+    uint32_t num_call_targets = decoder.consume_u32v("num call targets");
+    feedback.call_targets =
+        base::OwnedVector<uint32_t>::NewForOverwrite(num_call_targets);
+    for (uint32_t& call_target : feedback.call_targets) {
+      call_target = decoder.consume_u32v("call target");
+    }
+  }
+  CHECK(decoder.ok());
+  CHECK_EQ(decoder.pc(), decoder.end());
+}
+
+void DumpProfileToFile(const WasmModule* module,
+                       base::Vector<const uint8_t> wire_bytes) {
+  CHECK(!wire_bytes.empty());
+  // File are named `profile-wasm-<hash>`.
+  // We use the same hash as for reported scripts, to make it easier to
+  // correlate files to wasm modules (see {CreateWasmScript}).
+  uint32_t hash = static_cast<uint32_t>(GetWireBytesHash(wire_bytes));
+  base::EmbeddedVector<char, 32> filename;
+  SNPrintF(filename, "profile-wasm-%08x", hash);
+  base::OwnedVector<uint8_t> profile_data = GetProfileData(module);
+  PrintF("Dumping Wasm PGO data to file '%s' (%zu bytes)\n", filename.begin(),
+         profile_data.size());
+  if (FILE* file = base::OS::FOpen(filename.begin(), "wb")) {
+    CHECK_EQ(profile_data.size(),
+             fwrite(profile_data.begin(), 1, profile_data.size(), file));
+    base::Fclose(file);
+  }
+}
+
+void LoadProfileFromFile(WasmModule* module,
+                         base::Vector<const uint8_t> wire_bytes) {
+  CHECK(!wire_bytes.empty());
+  // File are named `profile-wasm-<hash>`.
+  // We use the same hash as for reported scripts, to make it easier to
+  // correlate files to wasm modules (see {CreateWasmScript}).
+  uint32_t hash = static_cast<uint32_t>(GetWireBytesHash(wire_bytes));
+  base::EmbeddedVector<char, 32> filename;
+  SNPrintF(filename, "profile-wasm-%08x", hash);
+
+  FILE* file = base::OS::FOpen(filename.begin(), "rb");
+  if (!file) {
+    PrintF("No Wasm PGO data found: Cannot open file '%s'\n", filename.begin());
+    return;
+  }
+
+  fseek(file, 0, SEEK_END);
+  size_t size = ftell(file);
+  rewind(file);
+
+  PrintF("Loading Wasm PGO data from file '%s' (%zu bytes)\n", filename.begin(),
+         size);
+  base::OwnedVector<uint8_t> profile_data =
+      base::OwnedVector<uint8_t>::NewForOverwrite(size);
+  for (size_t read = 0; read < size;) {
+    read += fread(profile_data.begin() + read, 1, size - read, file);
+    CHECK(!ferror(file));
+  }
+
+  base::Fclose(file);
+
+  RestoreProfileData(module, profile_data.as_vector());
+
+  // Check that the generated profile is deterministic.
+  DCHECK_EQ(profile_data.as_vector(), GetProfileData(module).as_vector());
+}
+
 }  // namespace v8::internal::wasm

src/wasm/wasm-module.h

@@ -450,6 +450,7 @@ class CallSiteFeedback {
   bool is_polymorphic() const { return index_or_count_ <= -2; }
   bool is_invalid() const { return index_or_count_ == -1; }
   const PolymorphicCase* polymorphic_storage() const {
+    DCHECK(is_polymorphic());
     return reinterpret_cast<PolymorphicCase*>(frequency_or_ool_);
   }
 
@@ -477,7 +478,7 @@ struct FunctionTypeFeedback {
 struct TypeFeedbackStorage {
   std::unordered_map<uint32_t, FunctionTypeFeedback> feedback_for_function;
   // Accesses to {feedback_for_function} are guarded by this mutex.
-  base::Mutex mutex;
+  mutable base::Mutex mutex;
 };
 
 struct WasmTable;
@@ -794,6 +795,12 @@ size_t PrintSignature(base::Vector<char> buffer, const wasm::FunctionSig*,
 V8_EXPORT_PRIVATE size_t
 GetWireBytesHash(base::Vector<const uint8_t> wire_bytes);
 
+void DumpProfileToFile(const WasmModule* module,
+                       base::Vector<const uint8_t> wire_bytes);
+
+void LoadProfileFromFile(WasmModule* module,
+                         base::Vector<const uint8_t> wire_bytes);
+
 }  // namespace v8::internal::wasm
 
 #endif  // V8_WASM_WASM_MODULE_H_

test/mjsunit/regress/wasm/regress-12874.js

@@ -2,7 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Flags: --experimental-wasm-gc --no-liftoff --no-wasm-inlining
+// Flags: --experimental-wasm-gc
+// Flags: --no-liftoff --no-wasm-speculative-inlining --no-wasm-inlining
 
 d8.file.execute("test/mjsunit/wasm/wasm-module-builder.js");