[wasm] Basic wasm tier-up

Wasm tier-up first compiles the whole module using Liftoff, and then
using Turbofan. The idea is to achieve fast start-up times by first
running Liftoff-compiled code. In the meantime we finish compilation
with Turbofan, and replace the Liftoff-compiled code as soon
as Turbofan finished compilation, thus achieving high performance.
Tier-up is enabled through the flag FLAG_wasm_tier_up.

Bug: v8:6600
Change-Id: I70552969c53d909a591666a1e7ce1ee1419b2f34
Reviewed-on: https://chromium-review.googlesource.com/1010422
Commit-Queue: Kim-Anh Tran <kimanh@google.com>
Reviewed-by: Andreas Haas <ahaas@chromium.org>
Reviewed-by: Clemens Hammacher <clemensh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#52759}

src/wasm/baseline/liftoff-compiler.cc

@@ -344,6 +344,11 @@ class LiftoffCompiler {
   }
 
   void StartFunctionBody(Decoder* decoder, Control* block) {
+    for (uint32_t i = 0; i < __ num_locals(); ++i) {
+      if (!CheckSupportedType(decoder, kTypes_ilfd, __ local_type(i), "param"))
+        return;
+    }
+
     // Input 0 is the call target, the instance is at 1.
     constexpr int kInstanceParameterIndex = 1;
     // Store the instance parameter to a special stack slot.
@@ -383,10 +388,6 @@ class LiftoffCompiler {
     // finish compilation without errors even if we hit unimplemented
     // LiftoffAssembler methods.
     if (DidAssemblerBailout(decoder)) return;
-    for (uint32_t i = 0; i < __ num_locals(); ++i) {
-      if (!CheckSupportedType(decoder, kTypes_ilfd, __ local_type(i), "param"))
-        return;
-    }
 
     __ SpillInstance(instance_reg);
     // Input 0 is the code target, 1 is the instance. First parameter at 2.

src/wasm/function-compiler.cc

@@ -14,6 +14,19 @@ namespace v8 {
 namespace internal {
 namespace wasm {
 
+namespace {
+const char* GetCompilationModeAsString(
+    WasmCompilationUnit::CompilationMode mode) {
+  switch (mode) {
+    case WasmCompilationUnit::CompilationMode::kLiftoff:
+      return "liftoff";
+    case WasmCompilationUnit::CompilationMode::kTurbofan:
+      return "turbofan";
+  }
+  UNREACHABLE();
+}
+}  // namespace
+
 // static
 WasmCompilationUnit::CompilationMode
 WasmCompilationUnit::GetDefaultCompilationMode() {
@@ -56,7 +69,8 @@ void WasmCompilationUnit::ExecuteCompilation() {
   TimedHistogramScope wasm_compile_function_time_scope(timed_histogram);
 
   if (FLAG_trace_wasm_compiler) {
-    PrintF("Compiling wasm function %d\n\n", func_index_);
+    PrintF("Compiling wasm function %d with %s\n\n", func_index_,
+           GetCompilationModeAsString(mode_));
   }
 
   switch (mode_) {

src/wasm/function-compiler.h

@@ -80,6 +80,7 @@ class WasmCompilationUnit final {
 
   size_t memory_cost() const { return memory_cost_; }
   wasm::NativeModule* native_module() const { return native_module_; }
+  CompilationMode mode() const { return mode_; }
 
  private:
   friend class LiftoffCompilationUnit;

src/wasm/module-compiler.h

@@ -22,7 +22,6 @@ namespace wasm {
 
 class CompilationState;
 class ModuleCompiler;
-struct ModuleEnv;
 class WasmCode;
 
 struct CompilationStateDeleter {
@@ -32,7 +31,9 @@ struct CompilationStateDeleter {
 // Wrapper to create a CompilationState exists in order to avoid having
 // the the CompilationState in the header file.
 std::unique_ptr<CompilationState, CompilationStateDeleter> NewCompilationState(
-    Isolate* isolate);
+    Isolate* isolate, ModuleEnv& env);
+
+ModuleEnv* GetModuleEnv(CompilationState* compilation_state);
 
 MaybeHandle<WasmModuleObject> CompileToModuleObject(
     Isolate* isolate, ErrorThrower* thrower, std::unique_ptr<WasmModule> module,
@@ -99,6 +100,7 @@ class AsyncCompileJob {
   class CompileWrappers;
   class FinishModule;
   class AbortCompilation;
+  class UpdateToTopTierCompiledCode;
 
   const std::shared_ptr<Counters>& async_counters() const {
     return async_counters_;
@@ -138,7 +140,6 @@ class AsyncCompileJob {
   ModuleWireBytes wire_bytes_;
   Handle<Context> context_;
   Handle<JSPromise> module_promise_;
-  std::unique_ptr<ModuleEnv> module_env_;
   std::unique_ptr<WasmModule> module_;
 
   std::vector<DeferredHandles*> deferred_handles_;
@@ -172,6 +173,8 @@ class AsyncCompileJob {
   // compilation. The AsyncCompileJob does not actively use the
   // StreamingDecoder.
   std::shared_ptr<StreamingDecoder> stream_;
+
+  bool tiering_completed_ = false;
 };
 }  // namespace wasm
 }  // namespace internal

src/wasm/wasm-code-manager.cc

@@ -330,10 +330,10 @@ WasmCode::~WasmCode() {
 // {source_native_module} into a {cloning_native_module}.
 class NativeModule::CloneCodeHelper {
  public:
-  explicit CloneCodeHelper(NativeModule* source_native_module,
+  explicit CloneCodeHelper(const NativeModule* source_native_module,
                            NativeModule* cloning_native_module);
 
-  void SelectForCloning(int32_t code_index);
+  void SelectForCloning(uint32_t code_index);
 
   void CloneAndPatchCode();
 
@@ -342,14 +342,34 @@ class NativeModule::CloneCodeHelper {
                                    WasmCode::FlushICache flush_icache);
 
  private:
-  NativeModule* source_native_module_;
+  const NativeModule* source_native_module_;
   NativeModule* cloning_native_module_;
   std::vector<uint32_t> selection_;
   std::unordered_map<Address, Address, AddressHasher> reverse_lookup_;
 };
 
+void NativeModule::CloneHigherTierCodeFrom(
+    const NativeModule* source_native_module) {
+  NativeModule::CloneCodeHelper helper(source_native_module, this);
+
+  for (uint32_t i = num_imported_functions_, e = FunctionCount(); i < e; ++i) {
+    WasmCode* wasm_code = GetCode(i);
+    if (!wasm_code->is_liftoff()) continue;
+    helper.SelectForCloning(i);
+  }
+
+  helper.CloneAndPatchCode();
+
+  // Sanity check: after cloning the code, every function in the
+  // code table should not be Liftoff-compiled code anymore.
+  for (uint32_t i = num_imported_functions(), e = FunctionCount(); i < e; ++i) {
+    DCHECK(!GetCode(i)->is_liftoff());
+  }
+}
+
 NativeModule::CloneCodeHelper::CloneCodeHelper(
-    NativeModule* source_native_module, NativeModule* cloning_native_module)
+    const NativeModule* source_native_module,
+    NativeModule* cloning_native_module)
     : source_native_module_(source_native_module),
       cloning_native_module_(cloning_native_module) {
   for (auto& pair : source_native_module_->trampolines_) {
@@ -361,7 +381,7 @@ NativeModule::CloneCodeHelper::CloneCodeHelper(
   }
 }
 
-void NativeModule::CloneCodeHelper::SelectForCloning(int32_t code_index) {
+void NativeModule::CloneCodeHelper::SelectForCloning(uint32_t code_index) {
   selection_.emplace_back(code_index);
 }
 
@@ -438,12 +458,12 @@ base::AtomicNumber<size_t> NativeModule::next_id_;
 
 NativeModule::NativeModule(uint32_t num_functions, uint32_t num_imports,
                            bool can_request_more, VirtualMemory* mem,
-                           WasmCodeManager* code_manager)
+                           WasmCodeManager* code_manager, ModuleEnv& env)
     : instance_id(next_id_.Increment(1)),
       code_table_(num_functions),
       num_imported_functions_(num_imports),
       compilation_state_(NewCompilationState(
-          reinterpret_cast<Isolate*>(code_manager->isolate_))),
+          reinterpret_cast<Isolate*>(code_manager->isolate_), env)),
       free_memory_(mem->address(), mem->end()),
       wasm_code_manager_(code_manager),
       can_request_more_memory_(can_request_more) {
@@ -1021,24 +1041,25 @@ size_t WasmCodeManager::GetAllocationChunk(const WasmModule& module) {
 }
 
 std::unique_ptr<NativeModule> WasmCodeManager::NewNativeModule(
-    const WasmModule& module) {
+    const WasmModule& module, ModuleEnv& env) {
   size_t code_size = GetAllocationChunk(module);
   return NewNativeModule(
       code_size, static_cast<uint32_t>(module.functions.size()),
-      module.num_imported_functions, kModuleCanAllocateMoreMemory);
+      module.num_imported_functions, kModuleCanAllocateMoreMemory, env);
 }
 
 std::unique_ptr<NativeModule> WasmCodeManager::NewNativeModule(
     size_t size_estimate, uint32_t num_functions,
-    uint32_t num_imported_functions, bool can_request_more) {
+    uint32_t num_imported_functions, bool can_request_more, ModuleEnv& env) {
   VirtualMemory mem;
   TryAllocate(size_estimate, &mem);
   if (mem.IsReserved()) {
     Address start = mem.address();
     size_t size = mem.size();
     Address end = mem.end();
-    std::unique_ptr<NativeModule> ret(new NativeModule(
-        num_functions, num_imported_functions, can_request_more, &mem, this));
+    std::unique_ptr<NativeModule> ret(
+        new NativeModule(num_functions, num_imported_functions,
+                         can_request_more, &mem, this, env));
     TRACE_HEAP("New Module: ID:%zu. Mem: %p,+%zu\n", ret->instance_id,
                reinterpret_cast<void*>(start), size);
     AssignRanges(start, end, ret.get());
@@ -1098,9 +1119,10 @@ bool NativeModule::SetExecutable(bool executable) {
 }
 
 std::unique_ptr<NativeModule> NativeModule::Clone() {
+  ModuleEnv* module_env = GetModuleEnv(compilation_state());
   std::unique_ptr<NativeModule> ret = wasm_code_manager_->NewNativeModule(
       owned_memory_.front().size(), FunctionCount(), num_imported_functions(),
-      can_request_more_memory_);
+      can_request_more_memory_, *module_env);
   TRACE_HEAP("%zu cloned from %zu\n", ret->instance_id, instance_id);
   if (!ret) return ret;
 

src/wasm/wasm-code-manager.h

@@ -247,6 +247,10 @@ class V8_EXPORT_PRIVATE NativeModule final {
   WasmCode* GetCode(uint32_t index) const;
   void SetCode(uint32_t index, WasmCode* wasm_code);
 
+  // Clones higher tier code from a {source_native_module} to
+  // this native module.
+  void CloneHigherTierCodeFrom(const NativeModule* source_native_module);
+
   // Register/release the protected instructions in all code objects with the
   // global trap handler for this process.
   void UnpackAndRegisterProtectedInstructions();
@@ -297,7 +301,7 @@ class V8_EXPORT_PRIVATE NativeModule final {
   static base::AtomicNumber<size_t> next_id_;
   NativeModule(uint32_t num_functions, uint32_t num_imports,
                bool can_request_more, VirtualMemory* vmem,
-               WasmCodeManager* code_manager);
+               WasmCodeManager* code_manager, ModuleEnv& env);
 
   WasmCode* AddAnonymousCode(Handle<Code>, WasmCode::Kind kind);
   Address AllocateForCode(size_t size);
@@ -367,11 +371,13 @@ class V8_EXPORT_PRIVATE WasmCodeManager final {
   // which will be page size aligned. The size of the initial memory
   // is determined with a heuristic based on the total size of wasm
   // code. The native module may later request more memory.
-  std::unique_ptr<NativeModule> NewNativeModule(const WasmModule&);
+  std::unique_ptr<NativeModule> NewNativeModule(const WasmModule& module,
+                                                ModuleEnv& env);
   std::unique_ptr<NativeModule> NewNativeModule(size_t memory_estimate,
                                                 uint32_t num_functions,
                                                 uint32_t num_imported_functions,
-                                                bool can_request_more);
+                                                bool can_request_more,
+                                                ModuleEnv& env);
 
   WasmCode* LookupCode(Address pc) const;
   WasmCode* GetCodeFromStartAddress(Address pc) const;

src/wasm/wasm-code-specialization.cc

@@ -205,6 +205,7 @@ bool CodeSpecialization::ApplyToWasmCode(wasm::WasmCode* code,
       } break;
       case RelocInfo::WASM_CODE_TABLE_ENTRY: {
         DCHECK(FLAG_wasm_tier_up);
+        DCHECK(code->is_liftoff());
         WasmCode* const* code_table_entry =
             native_module->code_table().data() + code->index();
         it.rinfo()->set_wasm_code_table_entry(

src/wasm/wasm-objects.cc

@@ -1357,13 +1357,13 @@ MaybeHandle<FixedArray> WasmSharedModuleData::CheckBreakPoints(
 
 Handle<WasmCompiledModule> WasmCompiledModule::New(
     Isolate* isolate, WasmModule* module, Handle<FixedArray> export_wrappers,
-    bool use_trap_handler) {
+    wasm::ModuleEnv& env) {
   Handle<WasmCompiledModule> compiled_module = Handle<WasmCompiledModule>::cast(
       isolate->factory()->NewStruct(WASM_COMPILED_MODULE_TYPE, TENURED));
   Handle<WeakCell> weak_native_context =
       isolate->factory()->NewWeakCell(isolate->native_context());
   compiled_module->set_weak_native_context(*weak_native_context);
-  compiled_module->set_use_trap_handler(use_trap_handler);
+  compiled_module->set_use_trap_handler(env.use_trap_handler);
   if (!export_wrappers.is_null()) {
     compiled_module->set_export_wrappers(*export_wrappers);
   }
@@ -1371,7 +1371,7 @@ Handle<WasmCompiledModule> WasmCompiledModule::New(
   wasm::NativeModule* native_module = nullptr;
   {
     std::unique_ptr<wasm::NativeModule> native_module_ptr =
-        isolate->wasm_engine()->code_manager()->NewNativeModule(*module);
+        isolate->wasm_engine()->code_manager()->NewNativeModule(*module, env);
     native_module = native_module_ptr.release();
     Handle<Foreign> native_module_wrapper =
         Managed<wasm::NativeModule>::From(isolate, native_module);

src/wasm/wasm-objects.h

@@ -26,6 +26,7 @@ namespace internal {
 namespace wasm {
 class InterpretedFrame;
 class NativeModule;
+struct ModuleEnv;
 class WasmCode;
 struct WasmModule;
 class SignatureMap;
@@ -545,10 +546,10 @@ class WasmCompiledModule : public Struct {
   WCM_SMALL_CONST_NUMBER(bool, use_trap_handler)
 
  public:
-  static Handle<WasmCompiledModule> New(
-      Isolate* isolate, wasm::WasmModule* module,
-      Handle<FixedArray> export_wrappers,
-      bool use_trap_hander);
+  static Handle<WasmCompiledModule> New(Isolate* isolate,
+                                        wasm::WasmModule* module,
+                                        Handle<FixedArray> export_wrappers,
+                                        wasm::ModuleEnv& env);
 
   static Handle<WasmCompiledModule> Clone(Isolate* isolate,
                                           Handle<WasmCompiledModule> module);

src/wasm/wasm-serialization.cc

@@ -13,6 +13,7 @@
 #include "src/snapshot/serializer-common.h"
 #include "src/utils.h"
 #include "src/version.h"
+#include "src/wasm/function-compiler.h"
 #include "src/wasm/module-compiler.h"
 #include "src/wasm/module-decoder.h"
 #include "src/wasm/wasm-code-manager.h"
@@ -702,9 +703,14 @@ MaybeHandle<WasmCompiledModule> DeserializeNativeModule(
   Handle<FixedArray> export_wrappers = isolate->factory()->NewFixedArray(
       static_cast<int>(export_wrappers_size), TENURED);
 
+  // TODO(eholk): We need to properly preserve the flag whether the trap
+  // handler was used or not when serializing.
+  UseTrapHandler use_trap_handler =
+      trap_handler::IsTrapHandlerEnabled() ? kUseTrapHandler : kNoTrapHandler;
+  wasm::ModuleEnv env(shared->module(), use_trap_handler,
+                      wasm::RuntimeExceptionSupport::kRuntimeExceptionSupport);
   Handle<WasmCompiledModule> compiled_module =
-      WasmCompiledModule::New(isolate, shared->module(), export_wrappers,
-                              trap_handler::IsTrapHandlerEnabled());
+      WasmCompiledModule::New(isolate, shared->module(), export_wrappers, env);
   compiled_module->set_shared(*shared);
   script->set_wasm_compiled_module(*compiled_module);
   NativeModuleDeserializer deserializer(isolate,

test/cctest/compiler/test-multiple-return.cc

@@ -17,6 +17,7 @@
 #include "src/macro-assembler-inl.h"
 #include "src/macro-assembler.h"
 #include "src/objects-inl.h"
+#include "src/wasm/function-compiler.h"
 #include "src/wasm/wasm-engine.h"
 #include "src/wasm/wasm-objects-inl.h"
 #include "src/wasm/wasm-opcodes.h"
@@ -123,12 +124,15 @@ Node* ToInt32(RawMachineAssembler& m, MachineType type, Node* a) {
 
 std::unique_ptr<wasm::NativeModule> AllocateNativeModule(Isolate* isolate,
                                                          size_t code_size) {
+  wasm::ModuleEnv env(
+      nullptr, wasm::UseTrapHandler::kNoTrapHandler,
+      wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport);
   // We have to add the code object to a NativeModule, because the
   // WasmCallDescriptor assumes that code is on the native heap and not
   // within a code object.
   std::unique_ptr<wasm::NativeModule> module =
       isolate->wasm_engine()->code_manager()->NewNativeModule(code_size, 1, 0,
-                                                              false);
+                                                              false, env);
   return module;
 }
 

test/cctest/wasm/wasm-run-utils.cc

@@ -225,9 +225,9 @@ Handle<WasmInstanceObject> TestingModuleBuilder::InitInstanceObject() {
       WasmSharedModuleData::New(isolate_, module_wrapper, empty_string, script,
                                 Handle<ByteArray>::null());
   Handle<FixedArray> export_wrappers = isolate_->factory()->NewFixedArray(0);
+  ModuleEnv env = CreateModuleEnv();
   Handle<WasmCompiledModule> compiled_module =
-      WasmCompiledModule::New(isolate_, test_module_ptr_, export_wrappers,
-                              trap_handler::IsTrapHandlerEnabled());
+      WasmCompiledModule::New(isolate_, test_module_ptr_, export_wrappers, env);
   compiled_module->set_shared(*shared_module_data);
   // This method is called when we initialize TestEnvironment. We don't
   // have a memory yet, so we won't create it here. We'll update the

test/fuzzer/multi-return.cc

@@ -151,12 +151,16 @@ CallDescriptor* CreateRandomCallDescriptor(Zone* zone, size_t return_count,
 
 std::unique_ptr<wasm::NativeModule> AllocateNativeModule(i::Isolate* isolate,
                                                          size_t code_size) {
+  wasm::ModuleEnv env(
+      nullptr, wasm::UseTrapHandler::kNoTrapHandler,
+      wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport);
+
   // We have to add the code object to a NativeModule, because the
   // WasmCallDescriptor assumes that code is on the native heap and not
   // within a code object.
   std::unique_ptr<wasm::NativeModule> module =
       isolate->wasm_engine()->code_manager()->NewNativeModule(code_size, 1, 0,
-                                                              false);
+                                                              false, env);
   return module;
 }
 

test/unittests/wasm/wasm-code-manager-unittest.cc

@@ -5,6 +5,7 @@
 #include "test/unittests/test-utils.h"
 #include "testing/gmock/include/gmock/gmock.h"
 
+#include "src/wasm/function-compiler.h"
 #include "src/wasm/wasm-code-manager.h"
 
 namespace v8 {
@@ -161,11 +162,15 @@ class WasmCodeManagerTest : public TestWithContext,
   // We pretend all our modules have 10 functions and no imports, just so
   // we can size up the code_table.
   NativeModulePtr AllocFixedModule(WasmCodeManager* manager, size_t size) {
-    return manager->NewNativeModule(size, 10, 0, false);
+    wasm::ModuleEnv env(nullptr, UseTrapHandler::kNoTrapHandler,
+                        RuntimeExceptionSupport::kNoRuntimeExceptionSupport);
+    return manager->NewNativeModule(size, 10, 0, false, env);
   }
 
   NativeModulePtr AllocGrowableModule(WasmCodeManager* manager, size_t size) {
-    return manager->NewNativeModule(size, 10, 0, true);
+    wasm::ModuleEnv env(nullptr, UseTrapHandler::kNoTrapHandler,
+                        RuntimeExceptionSupport::kNoRuntimeExceptionSupport);
+    return manager->NewNativeModule(size, 10, 0, true, env);
   }
 
   NativeModulePtr AllocModule(WasmCodeManager* manager, size_t size,