[wasm] Add prologue to Liftoff-compiled code for tiering

The prologue checks if optimized code exists, and if not, continues
execution of the current function. Otherwise, it jumps to the address
specified in the native module's code_table.

Also-by: clemensh@chromium.org
Change-Id: If3e76de02115f44ab7758590a949c3f0965a11ca
Reviewed-on: https://chromium-review.googlesource.com/985837
Commit-Queue: Kim-Anh Tran <kimanh@google.com>
Reviewed-by: Clemens Hammacher <clemensh@chromium.org>
Reviewed-by: Andreas Haas <ahaas@chromium.org>
Cr-Commit-Position: refs/heads/master@{#52471}

src/arm/assembler-arm-inl.h

@@ -144,6 +144,12 @@ Address RelocInfo::target_internal_reference_address() {
   return reinterpret_cast<Address>(pc_);
 }
 
+void RelocInfo::set_wasm_code_table_entry(Address target,
+                                          ICacheFlushMode icache_flush_mode) {
+  DCHECK(rmode_ == RelocInfo::WASM_CODE_TABLE_ENTRY);
+  Assembler::set_target_address_at(pc_, constant_pool_, target,
+                                   icache_flush_mode);
+}
 
 Address RelocInfo::target_runtime_entry(Assembler* origin) {
   DCHECK(IsRuntimeEntry(rmode_));

src/arm64/assembler-arm64-inl.h

@@ -674,6 +674,12 @@ Address RelocInfo::target_internal_reference_address() {
   return reinterpret_cast<Address>(pc_);
 }
 
+void RelocInfo::set_wasm_code_table_entry(Address target,
+                                          ICacheFlushMode icache_flush_mode) {
+  DCHECK(rmode_ == RelocInfo::WASM_CODE_TABLE_ENTRY);
+  Assembler::set_target_address_at(pc_, constant_pool_, target,
+                                   icache_flush_mode);
+}
 
 Address RelocInfo::target_runtime_entry(Assembler* origin) {
   DCHECK(IsRuntimeEntry(rmode_));

src/assembler.cc

@@ -539,6 +539,8 @@ const char* RelocInfo::RelocModeName(RelocInfo::Mode rmode) {
       return "global handle";
     case WASM_CALL:
       return "internal wasm call";
+    case WASM_CODE_TABLE_ENTRY:
+      return "wasm code table entry";
     case JS_TO_WASM_CALL:
       return "js to wasm call";
     case NUMBER_OF_MODES:
@@ -640,6 +642,7 @@ void RelocInfo::Verify(Isolate* isolate) {
     case WASM_GLOBAL_HANDLE:
     case WASM_CALL:
     case JS_TO_WASM_CALL:
+    case WASM_CODE_TABLE_ENTRY:
     case NONE:
       break;
     case NUMBER_OF_MODES:

src/assembler.h

@@ -396,6 +396,9 @@ class RelocInfo {
     // cannot be encoded as part of another record.
     PC_JUMP,
 
+    // Points to a wasm code table entry.
+    WASM_CODE_TABLE_ENTRY,
+
     // Pseudo-types
     NUMBER_OF_MODES,
     NONE,  // never recorded value
@@ -538,6 +541,8 @@ class RelocInfo {
   INLINE(void set_target_cell(
       Cell* cell, WriteBarrierMode write_barrier_mode = UPDATE_WRITE_BARRIER,
       ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED));
+  INLINE(void set_wasm_code_table_entry(
+      Address, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED));
   INLINE(void set_target_external_reference(
       Address, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED));
 

src/flag-definitions.h

@@ -543,6 +543,9 @@ DEFINE_UINT(wasm_max_mem_pages, v8::internal::wasm::kV8MaxWasmMemoryPages,
             "maximum memory size of a wasm instance")
 DEFINE_UINT(wasm_max_table_size, v8::internal::wasm::kV8MaxWasmTableSize,
             "maximum table size of a wasm instance")
+DEFINE_BOOL(wasm_tier_up, false,
+            "enable basic tiering up to the optimizing compiler")
+DEFINE_IMPLICATION(wasm_tier_up, liftoff)
 DEFINE_DEBUG_BOOL(trace_wasm_decoder, false, "trace decoding of wasm code")
 DEFINE_DEBUG_BOOL(trace_wasm_decode_time, false,
                   "trace decoding time of wasm code")

src/ia32/assembler-ia32-inl.h

@@ -138,6 +138,14 @@ Address RelocInfo::target_internal_reference_address() {
   return reinterpret_cast<Address>(pc_);
 }
 
+void RelocInfo::set_wasm_code_table_entry(Address target,
+                                          ICacheFlushMode icache_flush_mode) {
+  DCHECK(rmode_ == RelocInfo::WASM_CODE_TABLE_ENTRY);
+  Memory::Address_at(pc_) = target;
+  if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+    Assembler::FlushICache(pc_, sizeof(Address));
+  }
+}
 
 Address RelocInfo::target_runtime_entry(Assembler* origin) {
   DCHECK(IsRuntimeEntry(rmode_));

src/mips/assembler-mips-inl.h

@@ -247,6 +247,12 @@ Address RelocInfo::target_internal_reference_address() {
   return reinterpret_cast<Address>(pc_);
 }
 
+void RelocInfo::set_wasm_code_table_entry(Address target,
+                                          ICacheFlushMode icache_flush_mode) {
+  DCHECK(rmode_ == RelocInfo::WASM_CODE_TABLE_ENTRY);
+  Assembler::set_target_address_at(pc_, constant_pool_, target,
+                                   icache_flush_mode);
+}
 
 Address RelocInfo::target_runtime_entry(Assembler* origin) {
   DCHECK(IsRuntimeEntry(rmode_));

src/mips64/assembler-mips64-inl.h

@@ -213,6 +213,12 @@ Address RelocInfo::target_internal_reference_address() {
   return reinterpret_cast<Address>(pc_);
 }
 
+void RelocInfo::set_wasm_code_table_entry(Address target,
+                                          ICacheFlushMode icache_flush_mode) {
+  DCHECK(rmode_ == RelocInfo::WASM_CODE_TABLE_ENTRY);
+  Assembler::set_target_address_at(pc_, constant_pool_, target,
+                                   icache_flush_mode);
+}
 
 Address RelocInfo::target_runtime_entry(Assembler* origin) {
   DCHECK(IsRuntimeEntry(rmode_));

src/ppc/assembler-ppc-inl.h

@@ -200,6 +200,13 @@ void RelocInfo::set_target_external_reference(
                                    icache_flush_mode);
 }
 
+void RelocInfo::set_wasm_code_table_entry(Address target,
+                                          ICacheFlushMode icache_flush_mode) {
+  DCHECK(rmode_ == RelocInfo::WASM_CODE_TABLE_ENTRY);
+  Assembler::set_target_address_at(pc_, constant_pool_, target,
+                                   icache_flush_mode);
+}
+
 Address RelocInfo::target_runtime_entry(Assembler* origin) {
   DCHECK(IsRuntimeEntry(rmode_));
   return target_address();

src/s390/assembler-s390-inl.h

@@ -182,6 +182,13 @@ void RelocInfo::set_target_external_reference(
                                    icache_flush_mode);
 }
 
+void RelocInfo::set_wasm_code_table_entry(Address target,
+                                          ICacheFlushMode icache_flush_mode) {
+  DCHECK(rmode_ == RelocInfo::WASM_CODE_TABLE_ENTRY);
+  Assembler::set_target_address_at(pc_, constant_pool_, target,
+                                   icache_flush_mode);
+}
+
 Address RelocInfo::target_runtime_entry(Assembler* origin) {
   DCHECK(IsRuntimeEntry(rmode_));
   return target_address();

src/wasm/baseline/arm/liftoff-assembler-arm.h

@@ -190,6 +190,8 @@ bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
 
 void LiftoffAssembler::emit_jump(Label* label) { BAILOUT("emit_jump"); }
 
+void LiftoffAssembler::emit_jump(Register target) { BAILOUT("emit_jump"); }
+
 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
                                       ValueType type, Register lhs,
                                       Register rhs) {

src/wasm/baseline/arm64/liftoff-assembler-arm64.h

@@ -193,6 +193,8 @@ bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
 
 void LiftoffAssembler::emit_jump(Label* label) { BAILOUT("emit_jump"); }
 
+void LiftoffAssembler::emit_jump(Register target) { BAILOUT("emit_jump"); }
+
 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
                                       ValueType type, Register lhs,
                                       Register rhs) {

src/wasm/baseline/ia32/liftoff-assembler-ia32.h

@@ -932,6 +932,8 @@ bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
 
 void LiftoffAssembler::emit_jump(Label* label) { jmp(label); }
 
+void LiftoffAssembler::emit_jump(Register target) { jmp(target); }
+
 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
                                       ValueType type, Register lhs,
                                       Register rhs) {

src/wasm/baseline/liftoff-assembler.h

@@ -465,6 +465,8 @@ class LiftoffAssembler : public TurboAssembler {
                                    LiftoffRegister src);
 
   inline void emit_jump(Label*);
+  inline void emit_jump(Register);
+
   inline void emit_cond_jump(Condition, Label*, ValueType value, Register lhs,
                              Register rhs = no_reg);
   // Set {dst} to 1 if condition holds, 0 otherwise.

src/wasm/baseline/liftoff-compiler.cc

@@ -135,7 +135,8 @@ class LiftoffCompiler {
                   compiler::ModuleEnv* env,
                   SourcePositionTableBuilder* source_position_table_builder,
                   WasmCompilationData* wasm_compilation_data,
-                  Zone* compilation_zone, std::unique_ptr<Zone>* codegen_zone)
+                  Zone* compilation_zone, std::unique_ptr<Zone>* codegen_zone,
+                  WasmCode* const* code_table_entry)
       : asm_(liftoff_asm),
         descriptor_(
             GetLoweredCallDescriptor(compilation_zone, call_descriptor)),
@@ -149,7 +150,8 @@ class LiftoffCompiler {
         wasm_compilation_data_(wasm_compilation_data),
         compilation_zone_(compilation_zone),
         codegen_zone_(codegen_zone),
-        safepoint_table_builder_(compilation_zone_) {}
+        safepoint_table_builder_(compilation_zone_),
+        code_table_entry_(code_table_entry) {}
 
   ~LiftoffCompiler() { BindUnboundLabels(nullptr); }
 
@@ -271,6 +273,49 @@ class LiftoffCompiler {
     if (ool.continuation) __ bind(ool.continuation.get());
   }
 
+  // Inserts a check whether the optimized version of this code already exists.
+  // If so, it redirects execution to the optimized code.
+  void JumpToOptimizedCodeIfExisting() {
+    // Check whether we have an optimized function before
+    // continuing to execute the Liftoff-compiled code.
+    // TODO(clemensh): Reduce number of temporary registers.
+    LiftoffRegList pinned;
+    LiftoffRegister wasm_code_addr =
+        pinned.set(__ GetUnusedRegister(kGpReg, pinned));
+    LiftoffRegister target_code_addr =
+        pinned.set(__ GetUnusedRegister(kGpReg, pinned));
+    LiftoffRegister code_start_address =
+        pinned.set(__ GetUnusedRegister(kGpReg, pinned));
+
+    // Get the current code's target address ({instructions_.start()}).
+    __ ComputeCodeStartAddress(code_start_address.gp());
+
+    static LoadType kPointerLoadType =
+        LoadType::ForValueType(LiftoffAssembler::kWasmIntPtr);
+    using int_t = std::conditional<kPointerSize == 8, uint64_t, uint32_t>::type;
+    static_assert(sizeof(int_t) == sizeof(uintptr_t), "weird uintptr_t");
+    // Get the address of the WasmCode* currently stored in the code table.
+    __ LoadConstant(target_code_addr,
+                    WasmValue(reinterpret_cast<int_t>(code_table_entry_)),
+                    RelocInfo::WASM_CODE_TABLE_ENTRY);
+    // Load the corresponding WasmCode*.
+    __ Load(wasm_code_addr, target_code_addr.gp(), Register::no_reg(), 0,
+            kPointerLoadType, pinned);
+    // Load its target address ({instuctions_.start()}).
+    __ Load(target_code_addr, wasm_code_addr.gp(), Register::no_reg(),
+            WasmCode::kInstructionStartOffset, kPointerLoadType, pinned);
+
+    // If the current code's target address is the same as the
+    // target address of the stored WasmCode, then continue executing, otherwise
+    // jump to the updated WasmCode.
+    Label cont;
+    __ emit_cond_jump(kEqual, &cont, LiftoffAssembler::kWasmIntPtr,
+                      target_code_addr.gp(), code_start_address.gp());
+    __ LeaveFrame(StackFrame::WASM_COMPILED);
+    __ emit_jump(target_code_addr.gp());
+    __ bind(&cont);
+  }
+
   void StartFunctionBody(Decoder* decoder, Control* block) {
     __ EnterFrame(StackFrame::WASM_COMPILED);
     __ set_has_frame(true);
@@ -339,6 +384,14 @@ class LiftoffCompiler {
     StackCheck(0);
 
     DCHECK_EQ(__ num_locals(), __ cache_state()->stack_height());
+
+    // TODO(kimanh): if possible, we want to move this check further up,
+    // in order to avoid unnecessary overhead each time we enter
+    // a Liftoff-compiled function that will jump to a Turbofan-compiled
+    // function.
+    if (FLAG_wasm_tier_up) {
+      JumpToOptimizedCodeIfExisting();
+    }
   }
 
   void GenerateOutOfLineCode(OutOfLineCode& ool) {
@@ -1519,6 +1572,10 @@ class LiftoffCompiler {
   // patch the actually needed stack size in the end.
   uint32_t pc_offset_stack_frame_construction_ = 0;
 
+  // Points to the cell within the {code_table_} of the NativeModule,
+  // which  corresponds to the currently compiled function
+  WasmCode* const* code_table_entry_ = nullptr;
+
   void TraceCacheState(Decoder* decoder) const {
 #ifdef DEBUG
     if (!FLAG_trace_liftoff || !FLAG_trace_wasm_decoder) return;
@@ -1555,10 +1612,12 @@ bool compiler::WasmCompilationUnit::ExecuteLiftoffCompilation() {
   auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, func_body_.sig);
   base::Optional<TimedHistogramScope> liftoff_compile_time_scope(
       base::in_place, counters()->liftoff_compile_time());
+  wasm::WasmCode* const* code_table_entry =
+      native_module_->code_table().data() + func_index_;
   wasm::WasmFullDecoder<wasm::Decoder::kValidate, wasm::LiftoffCompiler>
       decoder(&zone, module, func_body_, &liftoff_.asm_, call_descriptor, env_,
               &liftoff_.source_position_table_builder_, &wasm_compilation_data_,
-              &zone, &liftoff_.codegen_zone_);
+              &zone, &liftoff_.codegen_zone_, code_table_entry);
   decoder.Decode();
   liftoff_compile_time_scope.reset();
   if (!decoder.interface().ok()) {

src/wasm/baseline/mips/liftoff-assembler-mips.h

@@ -620,6 +620,8 @@ void LiftoffAssembler::emit_jump(Label* label) {
   TurboAssembler::Branch(label);
 }
 
+void LiftoffAssembler::emit_jump(Register target) { BAILOUT("emit_jump"); }
+
 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
                                       ValueType type, Register lhs,
                                       Register rhs) {

src/wasm/baseline/mips64/liftoff-assembler-mips64.h

@@ -515,6 +515,8 @@ void LiftoffAssembler::emit_jump(Label* label) {
   TurboAssembler::Branch(label);
 }
 
+void LiftoffAssembler::emit_jump(Register target) { BAILOUT("emit_jump"); }
+
 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
                                       ValueType type, Register lhs,
                                       Register rhs) {

src/wasm/baseline/ppc/liftoff-assembler-ppc.h

@@ -190,6 +190,8 @@ bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
 
 void LiftoffAssembler::emit_jump(Label* label) { BAILOUT("emit_jump"); }
 
+void LiftoffAssembler::emit_jump(Register target) { BAILOUT("emit_jump"); }
+
 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
                                       ValueType type, Register lhs,
                                       Register rhs) {

src/wasm/baseline/s390/liftoff-assembler-s390.h

@@ -190,6 +190,8 @@ bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
 
 void LiftoffAssembler::emit_jump(Label* label) { BAILOUT("emit_jump"); }
 
+void LiftoffAssembler::emit_jump(Register target) { BAILOUT("emit_jump"); }
+
 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
                                       ValueType type, Register lhs,
                                       Register rhs) {

src/wasm/baseline/x64/liftoff-assembler-x64.h

@@ -842,6 +842,8 @@ bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
 
 void LiftoffAssembler::emit_jump(Label* label) { jmp(label); }
 
+void LiftoffAssembler::emit_jump(Register target) { jmp(target); }
+
 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
                                       ValueType type, Register lhs,
                                       Register rhs) {

src/wasm/wasm-code-manager.cc

@@ -462,10 +462,24 @@ NativeModule::NativeModule(uint32_t num_functions, uint32_t num_imports,
   owned_code_.reserve(num_functions);
 }
 
-void NativeModule::ResizeCodeTableForTest(size_t last_index) {
-  size_t new_size = last_index + 1;
-  if (new_size > FunctionCount()) {
-    code_table_.resize(new_size);
+void NativeModule::ResizeCodeTableForTesting(size_t num_functions,
+                                             size_t max_functions) {
+  DCHECK_LE(num_functions, max_functions);
+  if (num_imported_functions_ == num_functions) {
+    // For some tests, the code table might have been initialized to store
+    // a number of imported functions on creation. If that is the case,
+    // we need to retroactively reserve the space.
+    DCHECK_EQ(code_table_.capacity(), num_imported_functions_);
+    DCHECK_EQ(code_table_.size(), num_imported_functions_);
+    DCHECK_EQ(num_functions, 1);
+    code_table_.reserve(max_functions);
+  } else {
+    DCHECK_GT(num_functions, FunctionCount());
+    if (code_table_.capacity() == 0) {
+      code_table_.reserve(max_functions);
+    }
+    DCHECK_EQ(code_table_.capacity(), max_functions);
+    code_table_.resize(num_functions);
   }
 }
 

src/wasm/wasm-code-manager.h

@@ -141,6 +141,11 @@ class V8_EXPORT_PRIVATE WasmCode final {
 
   enum FlushICache : bool { kFlushICache = true, kNoFlushICache = false };
 
+  // Offset of {instructions_.start()}. It is used for tiering, when
+  // we check if optimized code is available during the prologue
+  // of Liftoff-compiled code.
+  static constexpr int kInstructionStartOffset = 0;
+
  private:
   friend class NativeModule;
 
@@ -169,6 +174,7 @@ class V8_EXPORT_PRIVATE WasmCode final {
     DCHECK_LE(safepoint_table_offset, instructions.size());
     DCHECK_LE(constant_pool_offset, instructions.size());
     DCHECK_LE(handler_table_offset, instructions.size());
+    DCHECK_EQ(kInstructionStartOffset, OFFSET_OF(WasmCode, instructions_));
   }
 
   Vector<byte> instructions_;
@@ -253,7 +259,7 @@ class V8_EXPORT_PRIVATE NativeModule final {
 
   // For cctests, where we build both WasmModule and the runtime objects
   // on the fly, and bypass the instance builder pipeline.
-  void ResizeCodeTableForTest(size_t);
+  void ResizeCodeTableForTesting(size_t num_functions, size_t max_functions);
 
   CompilationState* compilation_state() { return compilation_state_.get(); }
 
@@ -264,6 +270,8 @@ class V8_EXPORT_PRIVATE NativeModule final {
 
   uint32_t num_imported_functions() const { return num_imported_functions_; }
 
+  const std::vector<WasmCode*>& code_table() const { return code_table_; }
+
   size_t committed_memory() const { return committed_memory_; }
   const size_t instance_id = 0;
   ~NativeModule();

src/wasm/wasm-code-specialization.cc

@@ -166,6 +166,10 @@ bool CodeSpecialization::ApplyToWasmCode(wasm::WasmCode* code,
   };
   add_mode(reloc_direct_calls, RelocInfo::WASM_CALL);
 
+  // Always patch the code table entry address which is used in Liftoff
+  // prologue to jump to optimized code if existent.
+  reloc_mode |= RelocInfo::ModeMask(RelocInfo::WASM_CODE_TABLE_ENTRY);
+
   base::Optional<PatchDirectCallsHelper> patch_direct_calls_helper;
   bool changed = false;
 
@@ -198,6 +202,16 @@ bool CodeSpecialization::ApplyToWasmCode(wasm::WasmCode* code,
                                           icache_flush_mode);
         changed = true;
       } break;
+      case RelocInfo::WASM_CODE_TABLE_ENTRY: {
+        DCHECK(FLAG_wasm_tier_up);
+        WasmCode* const* code_table_entry =
+            native_module->code_table().data() + code->index();
+        it.rinfo()->set_wasm_code_table_entry(
+            const_cast<Address>(
+                reinterpret_cast<byte const*>(code_table_entry)),
+            icache_flush_mode);
+      } break;
+
       default:
         UNREACHABLE();
     }

src/x64/assembler-x64-inl.h

@@ -336,6 +336,14 @@ Handle<HeapObject> RelocInfo::target_object_handle(Assembler* origin) {
   }
 }
 
+void RelocInfo::set_wasm_code_table_entry(Address target,
+                                          ICacheFlushMode icache_flush_mode) {
+  DCHECK(rmode_ == RelocInfo::WASM_CODE_TABLE_ENTRY);
+  Memory::Address_at(pc_) = target;
+  if (icache_flush_mode != SKIP_ICACHE_FLUSH) {
+    Assembler::FlushICache(pc_, sizeof(Address));
+  }
+}
 
 Address RelocInfo::target_external_reference() {
   DCHECK(rmode_ == RelocInfo::EXTERNAL_REFERENCE);

test/cctest/wasm/wasm-run-utils.cc

@@ -45,7 +45,8 @@ TestingModuleBuilder::TestingModuleBuilder(
         isolate_, maybe_import->js_function, maybe_import->sig,
         maybe_import_index, test_module_.origin(),
         trap_handler::IsTrapHandlerEnabled());
-    native_module_->ResizeCodeTableForTest(maybe_import_index);
+    native_module_->ResizeCodeTableForTesting(maybe_import_index + 1,
+                                              kMaxFunctions);
     auto wasm_to_js_wrapper = native_module_->AddCodeCopy(
         code, wasm::WasmCode::kWasmToJsWrapper, maybe_import_index);
 
@@ -99,7 +100,7 @@ uint32_t TestingModuleBuilder::AddFunction(FunctionSig* sig, const char* name) {
   }
   uint32_t index = static_cast<uint32_t>(test_module_.functions.size());
   if (native_module_) {
-    native_module_->ResizeCodeTableForTest(index);
+    native_module_->ResizeCodeTableForTesting(index + 1, kMaxFunctions);
   }
   test_module_.functions.push_back({sig, index, 0, {0, 0}, false, false});
   if (name) {
@@ -420,7 +421,6 @@ void WasmFunctionCompiler::Build(const byte* start, const byte* end) {
   Handle<WasmCompiledModule> compiled_module(
       builder_->instance_object()->compiled_module(), isolate());
   NativeModule* native_module = compiled_module->GetNativeModule();
-  native_module->ResizeCodeTableForTest(function_->func_index);
   Handle<SeqOneByteString> wire_bytes(compiled_module->shared()->module_bytes(),
                                       isolate());