[wasm] Lazy compilation after deserialization

The serialization format contains one boolean flag per function which
specifies whether the function code exists in the serialized module or
not. With this CL, this boolean flag is extended to a three-value flag
which indicates whether the function exists, and if not, whether the
function was executed before serialization. This information can then be
used upon deserialization to compile only those functions that were
executed before serialization.

Design doc: https://docs.google.com/document/d/1U3uqq4njqLqFhr1G2sU_bmpQxY-3bvfG55udSb-DvA4/edit?usp=sharing

Bug: v8:12281
Change-Id: I465e31e5422fa45163256be0e6594045865f0174
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3364089Reviewed-by: Clemens Backes <clemensb@chromium.org>
Commit-Queue: Andreas Haas <ahaas@chromium.org>
Cr-Commit-Position: refs/heads/main@{#78545}

src/flags/flag-definitions.h

@@ -987,6 +987,7 @@ DEFINE_BOOL(wasm_tier_up, true,
             "have an effect)")
 DEFINE_BOOL(wasm_dynamic_tiering, false,
             "enable dynamic tier up to the optimizing compiler")
+DEFINE_NEG_NEG_IMPLICATION(liftoff, wasm_dynamic_tiering)
 DEFINE_WEAK_IMPLICATION(future, wasm_dynamic_tiering)
 DEFINE_INT(wasm_tiering_budget, 1800000,
            "budget for dynamic tiering (rough approximation of bytes executed")

src/runtime/runtime-test-wasm.cc

@@ -319,15 +319,12 @@ RUNTIME_FUNCTION(Runtime_GetWasmExceptionValues) {
   return *isolate->factory()->NewJSArrayWithElements(values);
 }
 
-// Wait until the given module is fully tiered up, then serialize it into an
-// array buffer.
 RUNTIME_FUNCTION(Runtime_SerializeWasmModule) {
   HandleScope scope(isolate);
   DCHECK_EQ(1, args.length());
   CONVERT_ARG_HANDLE_CHECKED(WasmModuleObject, module_obj, 0);
 
   wasm::NativeModule* native_module = module_obj->native_module();
-  native_module->compilation_state()->WaitForTopTierFinished();
   DCHECK(!native_module->compilation_state()->failed());
 
   wasm::WasmSerializer wasm_serializer(native_module);
@@ -470,6 +467,21 @@ RUNTIME_FUNCTION(Runtime_IsLiftoffFunction) {
   return isolate->heap()->ToBoolean(code && code->is_liftoff());
 }
 
+RUNTIME_FUNCTION(Runtime_IsTurboFanFunction) {
+  HandleScope scope(isolate);
+  DCHECK_EQ(1, args.length());
+  CONVERT_ARG_HANDLE_CHECKED(JSFunction, function, 0);
+  CHECK(WasmExportedFunction::IsWasmExportedFunction(*function));
+  Handle<WasmExportedFunction> exp_fun =
+      Handle<WasmExportedFunction>::cast(function);
+  wasm::NativeModule* native_module =
+      exp_fun->instance().module_object().native_module();
+  uint32_t func_index = exp_fun->function_index();
+  wasm::WasmCodeRefScope code_ref_scope;
+  wasm::WasmCode* code = native_module->GetCode(func_index);
+  return isolate->heap()->ToBoolean(code && code->is_turbofan());
+}
+
 RUNTIME_FUNCTION(Runtime_FreezeWasmLazyCompilation) {
   DCHECK_EQ(1, args.length());
   DisallowGarbageCollection no_gc;

src/runtime/runtime.h

@@ -608,6 +608,7 @@ namespace internal {
   F(GetWasmRecoveredTrapCount, 0, 1)       \
   F(IsAsmWasmCode, 1, 1)                   \
   F(IsLiftoffFunction, 1, 1)               \
+  F(IsTurboFanFunction, 1, 1)              \
   F(IsThreadInWasm, 0, 1)                  \
   F(IsWasmCode, 1, 1)                      \
   F(IsWasmTrapHandlerEnabled, 0, 1)        \

src/wasm/compilation-environment.h

@@ -158,7 +158,8 @@ class V8_EXPORT_PRIVATE CompilationState {
   void AddCallback(std::unique_ptr<CompilationEventCallback> callback);
 
   void InitializeAfterDeserialization(
-      base::Vector<const int> missing_functions);
+      base::Vector<const int> lazy_functions,
+      base::Vector<const int> liftoff_functions);
 
   // Wait until top tier compilation finished, or compilation failed.
   void WaitForTopTierFinished();

src/wasm/module-compiler.cc

@@ -556,7 +556,8 @@ class CompilationStateImpl {
   // Initialize the compilation progress after deserialization. This is needed
   // for recompilation (e.g. for tier down) to work later.
   void InitializeCompilationProgressAfterDeserialization(
-      base::Vector<const int> missing_functions);
+      base::Vector<const int> lazy_functions,
+      base::Vector<const int> liftoff_functions);
 
   // Initializes compilation units based on the information encoded in the
   // {compilation_progress_}.
@@ -666,7 +667,7 @@ class CompilationStateImpl {
 
  private:
   uint8_t SetupCompilationProgressForFunction(
-      bool lazy_module, NativeModule* module,
+      bool lazy_function, NativeModule* module,
       const WasmFeatures& enabled_features, int func_index);
 
   // Returns the potentially-updated {function_progress}.
@@ -849,9 +850,10 @@ void CompilationState::WaitForTopTierFinished() {
 void CompilationState::SetHighPriority() { Impl(this)->SetHighPriority(); }
 
 void CompilationState::InitializeAfterDeserialization(
-    base::Vector<const int> missing_functions) {
+    base::Vector<const int> lazy_functions,
+    base::Vector<const int> liftoff_functions) {
   Impl(this)->InitializeCompilationProgressAfterDeserialization(
-      missing_functions);
+      lazy_functions, liftoff_functions);
 }
 
 bool CompilationState::failed() const { return Impl(this)->failed(); }
@@ -3031,12 +3033,12 @@ bool CompilationStateImpl::cancelled() const {
 }
 
 uint8_t CompilationStateImpl::SetupCompilationProgressForFunction(
-    bool lazy_module, NativeModule* native_module,
+    bool lazy_function, NativeModule* native_module,
     const WasmFeatures& enabled_features, int func_index) {
   ExecutionTierPair requested_tiers =
       GetRequestedExecutionTiers(native_module, enabled_features, func_index);
   CompileStrategy strategy = GetCompileStrategy(
-      native_module->module(), enabled_features, func_index, lazy_module);
+      native_module->module(), enabled_features, func_index, lazy_function);
 
   bool required_for_baseline = strategy == CompileStrategy::kEager;
   bool required_for_top_tier = strategy != CompileStrategy::kLazy;
@@ -3200,9 +3202,11 @@ void CompilationStateImpl::AddCompilationUnit(CompilationUnitBuilder* builder,
 }
 
 void CompilationStateImpl::InitializeCompilationProgressAfterDeserialization(
-    base::Vector<const int> missing_functions) {
-  TRACE_EVENT1("v8.wasm", "wasm.CompilationAfterDeserialization",
-               "num_missing_functions", missing_functions.size());
+    base::Vector<const int> lazy_functions,
+    base::Vector<const int> liftoff_functions) {
+  TRACE_EVENT2("v8.wasm", "wasm.CompilationAfterDeserialization",
+               "num_lazy_functions", lazy_functions.size(),
+               "num_liftoff_functions", liftoff_functions.size());
   TimedHistogramScope lazy_compile_time_scope(
       counters()->wasm_compile_after_deserialize());
 
@@ -3212,21 +3216,35 @@ void CompilationStateImpl::InitializeCompilationProgressAfterDeserialization(
   {
     base::MutexGuard guard(&callbacks_mutex_);
     DCHECK(compilation_progress_.empty());
-    constexpr uint8_t kProgressAfterDeserialization =
+    constexpr uint8_t kProgressAfterTurbofanDeserialization =
         RequiredBaselineTierField::encode(ExecutionTier::kTurbofan) |
         RequiredTopTierField::encode(ExecutionTier::kTurbofan) |
         ReachedTierField::encode(ExecutionTier::kTurbofan);
     finished_events_.Add(CompilationEvent::kFinishedExportWrappers);
-    if (missing_functions.empty() || FLAG_wasm_lazy_compilation) {
+    if (liftoff_functions.empty() || lazy_module) {
+      // We have to trigger the compilation events to finish compilation.
+      // Typically the events get triggered when a CompilationUnit finishes, but
+      // with lazy compilation there are no compilation units.
       finished_events_.Add(CompilationEvent::kFinishedBaselineCompilation);
-      finished_events_.Add(CompilationEvent::kFinishedTopTierCompilation);
     }
     compilation_progress_.assign(module->num_declared_functions,
-                                 kProgressAfterDeserialization);
-    for (auto func_index : missing_functions) {
-      if (FLAG_wasm_lazy_compilation) {
+                                 kProgressAfterTurbofanDeserialization);
+    for (auto func_index : lazy_functions) {
+      native_module_->UseLazyStub(func_index);
+
+      compilation_progress_[declared_function_index(module, func_index)] =
+          SetupCompilationProgressForFunction(/*lazy_function =*/true,
+                                              native_module_, enabled_features,
+                                              func_index);
+    }
+    for (auto func_index : liftoff_functions) {
+      if (lazy_module) {
         native_module_->UseLazyStub(func_index);
       }
+      // Check that {func_index} is not contained in {lazy_functions}.
+      DCHECK_EQ(
+          compilation_progress_[declared_function_index(module, func_index)],
+          kProgressAfterTurbofanDeserialization);
       compilation_progress_[declared_function_index(module, func_index)] =
           SetupCompilationProgressForFunction(lazy_module, native_module_,
                                               enabled_features, func_index);

src/wasm/wasm-code-manager.h

@@ -292,7 +292,11 @@ class V8_EXPORT_PRIVATE WasmCode final {
   uint32_t raw_tagged_parameter_slots_for_serialization() const {
     return tagged_parameter_slots_;
   }
+
   bool is_liftoff() const { return tier() == ExecutionTier::kLiftoff; }
+
+  bool is_turbofan() const { return tier() == ExecutionTier::kTurbofan; }
+
   bool contains(Address pc) const {
     return reinterpret_cast<Address>(instructions_) <= pc &&
            pc < reinterpret_cast<Address>(instructions_ + instructions_size_);

src/wasm/wasm-serialization.cc

@@ -30,6 +30,9 @@ namespace internal {
 namespace wasm {
 
 namespace {
+constexpr uint8_t kLazyFunction = 2;
+constexpr uint8_t kLiftoffFunction = 3;
+constexpr uint8_t kTurboFanFunction = 4;
 
 // TODO(bbudge) Try to unify the various implementations of readers and writers
 // in Wasm, e.g. StreamProcessor and ZoneBuffer, with these.
@@ -189,17 +192,17 @@ uint32_t GetWasmCalleeTag(RelocInfo* rinfo) {
 
 constexpr size_t kHeaderSize = sizeof(size_t);  // total code size
 
-constexpr size_t kCodeHeaderSize = sizeof(bool) +  // whether code is present
-                                   sizeof(int) +   // offset of constant pool
-                                   sizeof(int) +   // offset of safepoint table
-                                   sizeof(int) +   // offset of handler table
-                                   sizeof(int) +   // offset of code comments
-                                   sizeof(int) +   // unpadded binary size
-                                   sizeof(int) +   // stack slots
-                                   sizeof(int) +   // tagged parameter slots
-                                   sizeof(int) +   // code size
-                                   sizeof(int) +   // reloc size
-                                   sizeof(int) +   // source positions size
+constexpr size_t kCodeHeaderSize = sizeof(uint8_t) +  // code kind
+                                   sizeof(int) +      // offset of constant pool
+                                   sizeof(int) +  // offset of safepoint table
+                                   sizeof(int) +  // offset of handler table
+                                   sizeof(int) +  // offset of code comments
+                                   sizeof(int) +  // unpadded binary size
+                                   sizeof(int) +  // stack slots
+                                   sizeof(int) +  // tagged parameter slots
+                                   sizeof(int) +  // code size
+                                   sizeof(int) +  // reloc size
+                                   sizeof(int) +  // source positions size
                                    sizeof(int) +  // protected instructions size
                                    sizeof(WasmCode::Kind) +  // code kind
                                    sizeof(ExecutionTier);    // tier
@@ -303,10 +306,10 @@ NativeModuleSerializer::NativeModuleSerializer(
 }
 
 size_t NativeModuleSerializer::MeasureCode(const WasmCode* code) const {
-  if (code == nullptr) return sizeof(bool);
+  if (code == nullptr) return sizeof(uint8_t);
   DCHECK_EQ(WasmCode::kWasmFunction, code->kind());
   if (code->tier() != ExecutionTier::kTurbofan) {
-    return sizeof(bool);
+    return sizeof(uint8_t);
   }
   return kCodeHeaderSize + code->instructions().size() +
          code->reloc_info().size() + code->source_positions().size() +
@@ -330,20 +333,30 @@ void NativeModuleSerializer::WriteHeader(Writer* writer,
 }
 
 bool NativeModuleSerializer::WriteCode(const WasmCode* code, Writer* writer) {
-  DCHECK_IMPLIES(!FLAG_wasm_lazy_compilation, code != nullptr);
   if (code == nullptr) {
-    writer->Write(false);
+    writer->Write(kLazyFunction);
     return true;
   }
   DCHECK_EQ(WasmCode::kWasmFunction, code->kind());
   // Only serialize TurboFan code, as Liftoff code can contain breakpoints or
   // non-relocatable constants.
   if (code->tier() != ExecutionTier::kTurbofan) {
-    writer->Write(false);
+    // We check if the function has been executed already. If so, we serialize
+    // it as {kLiftoffFunction} so that upon deserialization the function will
+    // get compiled with Liftoff eagerly. If the function has not been executed
+    // yet, we serialize it as {kLazyFunction}, and the function will not get
+    // compiled upon deserialization.
+    NativeModule* native_module = code->native_module();
+    uint32_t budget =
+        native_module->tiering_budget_array()[declared_function_index(
+            native_module->module(), code->index())];
+    writer->Write(budget == static_cast<uint32_t>(FLAG_wasm_tiering_budget)
+                      ? kLazyFunction
+                      : kLiftoffFunction);
     return true;
   }
   ++num_turbofan_functions_;
-  writer->Write(true);
+  writer->Write(kTurboFanFunction);
   // Write the size of the entire code section, followed by the code header.
   writer->Write(code->constant_pool_offset());
   writer->Write(code->safepoint_table_offset());
@@ -537,8 +550,12 @@ class V8_EXPORT_PRIVATE NativeModuleDeserializer {
 
   bool Read(Reader* reader);
 
-  base::Vector<const int> missing_functions() {
-    return base::VectorOf(missing_functions_);
+  base::Vector<const int> lazy_functions() {
+    return base::VectorOf(lazy_functions_);
+  }
+
+  base::Vector<const int> liftoff_functions() {
+    return base::VectorOf(liftoff_functions_);
   }
 
  private:
@@ -559,7 +576,8 @@ class V8_EXPORT_PRIVATE NativeModuleDeserializer {
   size_t remaining_code_size_ = 0;
   base::Vector<byte> current_code_space_;
   NativeModule::JumpTablesRef current_jump_tables_;
-  std::vector<int> missing_functions_;
+  std::vector<int> lazy_functions_;
+  std::vector<int> liftoff_functions_;
 };
 
 class CopyAndRelocTask : public JobTask {
@@ -692,11 +710,16 @@ void NativeModuleDeserializer::ReadHeader(Reader* reader) {
 
 DeserializationUnit NativeModuleDeserializer::ReadCode(int fn_index,
                                                        Reader* reader) {
-  bool has_code = reader->Read<bool>();
-  if (!has_code) {
-    missing_functions_.push_back(fn_index);
+  uint8_t code_kind = reader->Read<uint8_t>();
+  if (code_kind == kLazyFunction) {
+    lazy_functions_.push_back(fn_index);
     return {};
   }
+  if (code_kind == kLiftoffFunction) {
+    liftoff_functions_.push_back(fn_index);
+    return {};
+  }
+
   int constant_pool_offset = reader->Read<int>();
   int safepoint_table_offset = reader->Read<int>();
   int handler_table_offset = reader->Read<int>();
@@ -873,7 +896,7 @@ MaybeHandle<WasmModuleObject> DeserializeNativeModule(
       return {};
     }
     shared_native_module->compilation_state()->InitializeAfterDeserialization(
-        deserializer.missing_functions());
+        deserializer.lazy_functions(), deserializer.liftoff_functions());
     wasm_engine->UpdateNativeModuleCache(error, &shared_native_module, isolate);
   }
 

test/mjsunit/regress/wasm/regress-11024.js

@@ -5,7 +5,6 @@
 // The test needs --no-liftoff because we can't serialize and deserialize
 // Liftoff code.
 // Flags: --expose-wasm --allow-natives-syntax --expose-gc --no-liftoff
-// Flags: --no-wasm-dynamic-tiering
 
 d8.file.execute("test/mjsunit/wasm/wasm-module-builder.js");
 

test/mjsunit/regress/wasm/regress-7785.js

@@ -2,9 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// The test needs --wasm-tier-up because we can't serialize and deserialize
+// The test needs --no-liftoff because we can't serialize and deserialize
 // Liftoff code.
-// Flags: --allow-natives-syntax --wasm-tier-up --no-wasm-dynamic-tiering
+// Flags: --allow-natives-syntax --no-liftoff
 
 d8.file.execute("test/mjsunit/wasm/wasm-module-builder.js");
 

test/mjsunit/regress/wasm/regress-808848.js

@@ -2,9 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// The test needs --wasm-tier-up because we can't serialize and deserialize
+// The test needs --no-liftoff because we can't serialize and deserialize
 // Liftoff code.
-// Flags: --allow-natives-syntax --wasm-tier-up --no-wasm-dynamic-tiering
+// Flags: --allow-natives-syntax --no-liftoff
 
 d8.file.execute('test/mjsunit/wasm/wasm-module-builder.js');
 

test/mjsunit/regress/wasm/regress-808980.js

@@ -2,10 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// The test needs --wasm-tier-up because we can't serialize and deserialize
+// The test needs --no-liftoff because we can't serialize and deserialize
 // Liftoff code.
-// Flags: --allow-natives-syntax --throws --wasm-tier-up
-// Flags: --no-wasm-dynamic-tiering
+// Flags: --allow-natives-syntax --throws --no-liftoff
 
 d8.file.execute('test/mjsunit/wasm/wasm-module-builder.js');
 let kTableSize = 3;

test/mjsunit/regress/wasm/regress-8896.js

@@ -4,7 +4,7 @@
 
 // Flags: --experimental-wasm-eh --allow-natives-syntax
 // Disable Liftoff so we can serialize the module.
-// Flags: --no-liftoff --no-wasm-dynamic-tiering
+// Flags: --no-liftoff
 
 d8.file.execute("test/mjsunit/wasm/wasm-module-builder.js");
 

test/mjsunit/wasm/compiled-module-serialization.js

@@ -4,8 +4,7 @@
 
 // The test needs --wasm-tier-up because we can't serialize and deserialize
 // Liftoff code.
-// Flags: --expose-wasm --allow-natives-syntax --expose-gc --wasm-tier-up
-// Flags: --no-wasm-dynamic-tiering
+// Flags: --expose-wasm --allow-natives-syntax --expose-gc --no-liftoff
 
 d8.file.execute("test/mjsunit/wasm/wasm-module-builder.js");
 

test/mjsunit/wasm/print-code.js

@@ -2,10 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// The test needs --wasm-tier-up because we can't serialize and deserialize
+// The test needs --no-liftoff because we can't serialize and deserialize
 // Liftoff code.
-// Flags: --allow-natives-syntax --print-wasm-code --wasm-tier-up
-// Flags: --no-wasm-dynamic-tiering
+// Flags: --allow-natives-syntax --print-wasm-code --no-liftoff
 
 // Just test that printing the code of the following wasm modules does not
 // crash.

test/mjsunit/wasm/serialization-with-compilation-hints.js

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --allow-natives-syntax --expose-gc --wasm-dynamic-tiering --liftoff
+// Make the test faster:
+// Flags: --wasm-tiering-budget=1000
+
+// This test busy-waits for tier-up to be complete, hence it does not work in
+// predictable mode where we only have a single thread.
+// Flags: --no-predictable
+
+d8.file.execute('test/mjsunit/wasm/wasm-module-builder.js');
+
+const num_functions = 3;
+
+function create_builder() {
+  const builder = new WasmModuleBuilder();
+  builder.addImport("foo", "bar", kSig_i_v);
+  for (let i = 0; i < num_functions; ++i) {
+    builder.addFunction('f' + i, kSig_i_v)
+        .addBody(wasmI32Const(i))
+        .exportFunc();
+  }
+  return builder;
+}
+
+const wire_bytes = create_builder().toBuffer();
+
+function serializeModule() {
+  const module = new WebAssembly.Module(wire_bytes);
+  let instance = new WebAssembly.Instance(module, {foo: {bar: () => 1}});
+  // Execute {f1} until it gets tiered up.
+  while (%IsLiftoffFunction(instance.exports.f1)) {
+    instance.exports.f1();
+  }
+  // Execute {f2} once, so that the module knows that this is a used function.
+  instance.exports.f2();
+  const buff = %SerializeWasmModule(module);
+  return buff;
+};
+
+const serialized_module = serializeModule();
+// Do some GCs to make sure the first module got collected and removed from the
+// module cache.
+gc();
+gc();
+gc();
+
+(function testSerializedModule() {
+  print(arguments.callee.name);
+  const module = %DeserializeWasmModule(serialized_module, wire_bytes);
+
+  const instance = new WebAssembly.Instance(module, {foo: {bar: () => 1}});
+
+  assertTrue(%IsTurboFanFunction(instance.exports.f1));
+  assertTrue(%IsLiftoffFunction(instance.exports.f2));
+  assertTrue(
+      !%IsLiftoffFunction(instance.exports.f0) &&
+      !%IsTurboFanFunction(instance.exports.f0));
+})();

test/mjsunit/wasm/serialize-lazy-module.js

@@ -5,7 +5,7 @@
 // The test needs --no-liftoff because we can't serialize and deserialize
 // Liftoff code.
 // Flags: --wasm-lazy-compilation --allow-natives-syntax --expose-gc
-// Flags: --no-liftoff --no-wasm-dynamic-tiering
+// Flags: --no-liftoff
 
 d8.file.execute('test/mjsunit/wasm/wasm-module-builder.js');
 

test/mjsunit/wasm/test-partial-serialization.js

@@ -23,9 +23,9 @@ function create_builder() {
 
 function check(instance) {
   for (let i = 0; i < num_functions; ++i) {
-    const expect_liftoff = i != 0 && i != 2;
+    const expect_turbofan = i == 0 || i == 2;
     assertEquals(
-        expect_liftoff, %IsLiftoffFunction(instance.exports['f' + i]),
+        expect_turbofan, %IsTurboFanFunction(instance.exports['f' + i]),
         'function ' + i);
   }
 }

test/mjsunit/wasm/test-serialization-with-lazy-compilation.js

@@ -5,7 +5,7 @@
 // The test needs --no-liftoff because we can't serialize and deserialize
 // Liftoff code.
 // Flags: --allow-natives-syntax --wasm-lazy-compilation --expose-gc
-// Flags: --no-liftoff --no-wasm-dynamic-tiering
+// Flags: --no-liftoff
 
 d8.file.execute('test/mjsunit/wasm/wasm-module-builder.js');
 

tools/testrunner/local/variants.py

@@ -72,7 +72,8 @@ INCOMPATIBLE_FLAGS_PER_VARIANT = {
   "stress": ["--always-opt", "--no-always-opt",
              "--max-inlined-bytecode-size=*",
              "--max-inlined-bytecode-size-cumulative=*", "--stress-inline",
-             "--liftoff-only", "--wasm-speculative-inlining"],
+             "--liftoff-only", "--wasm-speculative-inlining",
+             "--wasm-dynamic-tiering"],
   "sparkplug": ["--jitless"],
   "always_sparkplug": ["--jitless"],
   "code_serializer": ["--cache=after-execute", "--cache=full-code-cache",