[liftoff] Cache the instance in a register

This CL avoids redundant loads of the instance from the frame by caching
it in a register if possible. This register will be the first one to be
cleared once we run out of registers (hence it's called a "volatile
register"). On local tests, this seems to reduce most redundant loads
within a function, and it also reduces the load for the stack check in
the function prologue.
After the stack check, we need to discard the cached instance though,
since the potential runtime call for the stack check might clobber it.
This will be addressed in a follow-up CL by re-loading the cached
instance after the stack check. This is expected to remove another good
chunk of instance loads, because the instance would initially be
available in a register when starting the function code.

R=thibaudm@chromium.org

Bug: v8:11336
Change-Id: Ie65ab81263fb9d972f4b7a6daaef86cf704874ef
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2695401
Commit-Queue: Clemens Backes <clemensb@chromium.org>
Reviewed-by: Thibaud Michaud <thibaudm@chromium.org>
Cr-Commit-Position: refs/heads/master@{#72779}

src/wasm/baseline/arm/liftoff-assembler-arm.h

@@ -574,10 +574,14 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
   }
 }
 
-void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) {
-  DCHECK_LE(0, offset);
+void LiftoffAssembler::LoadInstanceFromFrame(Register dst) {
   ldr(dst, liftoff::GetInstanceOperand());
-  MemOperand src{dst, offset};
+}
+
+void LiftoffAssembler::LoadFromInstance(Register dst, Register instance,
+                                        int offset, int size) {
+  DCHECK_LE(0, offset);
+  MemOperand src{instance, offset};
   switch (size) {
     case 1:
       ldrb(dst, src);
@@ -590,8 +594,11 @@ void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) {
   }
 }
 
-void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) {
-  LoadFromInstance(dst, offset, kTaggedSize);
+void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst,
+                                                     Register instance,
+                                                     int offset) {
+  STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
+  ldr(dst, MemOperand{instance, offset});
 }
 
 void LiftoffAssembler::SpillInstance(Register instance) {

src/wasm/baseline/arm64/liftoff-assembler-arm64.h

@@ -392,10 +392,14 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
   }
 }
 
-void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) {
-  DCHECK_LE(0, offset);
+void LiftoffAssembler::LoadInstanceFromFrame(Register dst) {
   Ldr(dst, liftoff::GetInstanceOperand());
-  MemOperand src{dst, offset};
+}
+
+void LiftoffAssembler::LoadFromInstance(Register dst, Register instance,
+                                        int offset, int size) {
+  DCHECK_LE(0, offset);
+  MemOperand src{instance, offset};
   switch (size) {
     case 1:
       Ldrb(dst.W(), src);
@@ -411,10 +415,11 @@ void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) {
   }
 }
 
-void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) {
+void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst,
+                                                     Register instance,
+                                                     int offset) {
   DCHECK_LE(0, offset);
-  Ldr(dst, liftoff::GetInstanceOperand());
-  LoadTaggedPointerField(dst, MemOperand(dst, offset));
+  LoadTaggedPointerField(dst, MemOperand{instance, offset});
 }
 
 void LiftoffAssembler::SpillInstance(Register instance) {

src/wasm/baseline/ia32/liftoff-assembler-ia32.h

@@ -295,10 +295,14 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
   }
 }
 
-void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) {
-  DCHECK_LE(0, offset);
+void LiftoffAssembler::LoadInstanceFromFrame(Register dst) {
   mov(dst, liftoff::GetInstanceOperand());
-  Operand src{dst, offset};
+}
+
+void LiftoffAssembler::LoadFromInstance(Register dst, Register instance,
+                                        int offset, int size) {
+  DCHECK_LE(0, offset);
+  Operand src{instance, offset};
   switch (size) {
     case 1:
       movzx_b(dst, src);
@@ -311,8 +315,11 @@ void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) {
   }
 }
 
-void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) {
-  LoadFromInstance(dst, offset, kTaggedSize);
+void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst,
+                                                     Register instance,
+                                                     int offset) {
+  STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
+  mov(dst, Operand{instance, offset});
 }
 
 void LiftoffAssembler::SpillInstance(Register instance) {

src/wasm/baseline/liftoff-assembler.cc

@@ -456,6 +456,10 @@ void LiftoffAssembler::CacheState::InitMerge(const CacheState& source,
   // |------locals------|---(in between)----|--(discarded)--|----merge----|
   //  <-- num_locals --> <-- stack_depth -->^stack_base      <-- arity -->
 
+  if (source.cached_instance != no_reg) {
+    SetInstanceCacheRegister(source.cached_instance);
+  }
+
   uint32_t stack_base = stack_depth + num_locals;
   uint32_t target_height = stack_base + arity;
   uint32_t discarded = source.stack_height() - target_height;
@@ -687,7 +691,7 @@ void LiftoffAssembler::MaterializeMergedConstants(uint32_t arity) {
   }
 }
 
-void LiftoffAssembler::MergeFullStackWith(const CacheState& target,
+void LiftoffAssembler::MergeFullStackWith(CacheState& target,
                                           const CacheState& source) {
   DCHECK_EQ(source.stack_height(), target.stack_height());
   // TODO(clemensb): Reuse the same StackTransferRecipe object to save some
@@ -696,10 +700,16 @@ void LiftoffAssembler::MergeFullStackWith(const CacheState& target,
   for (uint32_t i = 0, e = source.stack_height(); i < e; ++i) {
     transfers.TransferStackSlot(target.stack_state[i], source.stack_state[i]);
   }
+
+  if (cache_state_.cached_instance != target.cached_instance) {
+    // Backward jumps (to loop headers) do not have a cached instance anyway, so
+    // ignore this. On forward jumps, jump reset the cached instance in the
+    // target state.
+    target.ClearCachedInstanceRegister();
+  }
 }
 
-void LiftoffAssembler::MergeStackWith(const CacheState& target,
-                                      uint32_t arity) {
+void LiftoffAssembler::MergeStackWith(CacheState& target, uint32_t arity) {
   // Before: ----------------|----- (discarded) ----|--- arity ---|
   //                         ^target_stack_height   ^stack_base   ^stack_height
   // After:  ----|-- arity --|
@@ -720,6 +730,13 @@ void LiftoffAssembler::MergeStackWith(const CacheState& target,
     transfers.TransferStackSlot(target.stack_state[target_stack_base + i],
                                 cache_state_.stack_state[stack_base + i]);
   }
+
+  if (cache_state_.cached_instance != target.cached_instance) {
+    // Backward jumps (to loop headers) do not have a cached instance anyway, so
+    // ignore this. On forward jumps, jump reset the cached instance in the
+    // target state.
+    target.ClearCachedInstanceRegister();
+  }
 }
 
 void LiftoffAssembler::Spill(VarState* slot) {
@@ -750,12 +767,17 @@ void LiftoffAssembler::SpillAllRegisters() {
     Spill(slot.offset(), slot.reg(), slot.type());
     slot.MakeStack();
   }
+  cache_state_.ClearCachedInstanceRegister();
   cache_state_.reset_used_registers();
 }
 
 void LiftoffAssembler::ClearRegister(
     Register reg, std::initializer_list<Register*> possible_uses,
     LiftoffRegList pinned) {
+  if (reg == cache_state()->cached_instance) {
+    cache_state()->ClearCachedInstanceRegister();
+    return;
+  }
   if (cache_state()->is_used(LiftoffRegister(reg))) {
     SpillRegister(LiftoffRegister(reg));
   }
@@ -850,6 +872,7 @@ void LiftoffAssembler::PrepareCall(const FunctionSig* sig,
   constexpr size_t kInputShift = 1;
 
   // Spill all cache slots which are not being used as parameters.
+  cache_state_.ClearCachedInstanceRegister();
   for (VarState* it = cache_state_.stack_state.end() - 1 - num_params;
        it >= cache_state_.stack_state.begin() &&
        !cache_state_.used_registers.is_empty();
@@ -1080,6 +1103,14 @@ bool LiftoffAssembler::ValidateCacheState() const {
     }
     used_regs.set(reg);
   }
+  if (cache_state_.cached_instance != no_reg) {
+    DCHECK(!used_regs.has(cache_state_.cached_instance));
+    int liftoff_code =
+        LiftoffRegister{cache_state_.cached_instance}.liftoff_code();
+    used_regs.set(cache_state_.cached_instance);
+    DCHECK_EQ(0, register_use_count[liftoff_code]);
+    register_use_count[liftoff_code] = 1;
+  }
   bool valid = memcmp(register_use_count, cache_state_.register_use_count,
                       sizeof(register_use_count)) == 0 &&
                used_regs == cache_state_.used_registers;

src/wasm/baseline/liftoff-assembler.h

@@ -192,6 +192,7 @@ class LiftoffAssembler : public TurboAssembler {
     LiftoffRegList used_registers;
     uint32_t register_use_count[kAfterMaxLiftoffRegCode] = {0};
     LiftoffRegList last_spilled_regs;
+    Register cached_instance = no_reg;
 
     bool has_unused_register(RegClass rc, LiftoffRegList pinned = {}) const {
       if (kNeedI64RegPair && rc == kGpRegPair) {
@@ -241,6 +242,51 @@ class LiftoffAssembler : public TurboAssembler {
       return available_regs.GetFirstRegSet();
     }
 
+    // Volatile registers are registers which are used for caching values that
+    // can easily be reloaded. Those are returned first if we run out of free
+    // registers.
+    // Note: This interface is a bit more generic than currently needed, in
+    // anticipation of more "volatile registers" being added later.
+    bool has_volatile_register(LiftoffRegList candidates) {
+      return cached_instance != no_reg && candidates.has(cached_instance);
+    }
+
+    LiftoffRegister take_volatile_register(LiftoffRegList candidates) {
+      DCHECK(candidates.has(cached_instance));
+      LiftoffRegister ret{cached_instance};
+      DCHECK_EQ(1, register_use_count[ret.liftoff_code()]);
+      register_use_count[ret.liftoff_code()] = 0;
+      used_registers.clear(ret);
+      cached_instance = no_reg;
+      return ret;
+    }
+
+    void SetInstanceCacheRegister(Register reg) {
+      DCHECK_EQ(no_reg, cached_instance);
+      cached_instance = reg;
+      int liftoff_code = LiftoffRegister{reg}.liftoff_code();
+      DCHECK_EQ(0, register_use_count[liftoff_code]);
+      register_use_count[liftoff_code] = 1;
+      used_registers.set(reg);
+    }
+
+    Register TrySetCachedInstanceRegister(LiftoffRegList pinned) {
+      DCHECK_EQ(no_reg, cached_instance);
+      if (!has_unused_register(kGpCacheRegList, pinned)) return no_reg;
+      SetInstanceCacheRegister(unused_register(kGpCacheRegList, pinned).gp());
+      DCHECK_NE(no_reg, cached_instance);
+      return cached_instance;
+    }
+
+    void ClearCachedInstanceRegister() {
+      if (cached_instance == no_reg) return;
+      int liftoff_code = LiftoffRegister{cached_instance}.liftoff_code();
+      DCHECK_EQ(1, register_use_count[liftoff_code]);
+      register_use_count[liftoff_code] = 0;
+      used_registers.clear(cached_instance);
+      cached_instance = no_reg;
+    }
+
     void inc_used(LiftoffRegister reg) {
       if (reg.is_pair()) {
         inc_used(reg.low());
@@ -445,13 +491,18 @@ class LiftoffAssembler : public TurboAssembler {
     if (cache_state_.has_unused_register(candidates, pinned)) {
       return cache_state_.unused_register(candidates, pinned);
     }
+    if (cache_state_.has_volatile_register(candidates)) {
+      LiftoffRegister reg = cache_state_.take_volatile_register(candidates);
+      DCHECK(!pinned.has(reg));
+      return reg;
+    }
     return SpillOneRegister(candidates, pinned);
   }
 
   void MaterializeMergedConstants(uint32_t arity);
 
-  void MergeFullStackWith(const CacheState& target, const CacheState& source);
-  void MergeStackWith(const CacheState& target, uint32_t arity);
+  void MergeFullStackWith(CacheState& target, const CacheState& source);
+  void MergeStackWith(CacheState& target, uint32_t arity);
 
   void Spill(VarState* slot);
   void SpillLocals();
@@ -469,7 +520,12 @@ class LiftoffAssembler : public TurboAssembler {
   template <typename... Regs>
   void SpillRegisters(Regs... regs) {
     for (LiftoffRegister r : {LiftoffRegister(regs)...}) {
-      if (cache_state()->is_used(r)) SpillRegister(r);
+      if (cache_state_.is_free(r)) continue;
+      if (r.is_gp() && cache_state_.cached_instance == r.gp()) {
+        cache_state_.ClearCachedInstanceRegister();
+      } else {
+        SpillRegister(r);
+      }
     }
   }
 
@@ -548,8 +604,11 @@ class LiftoffAssembler : public TurboAssembler {
 
   inline void LoadConstant(LiftoffRegister, WasmValue,
                            RelocInfo::Mode rmode = RelocInfo::NONE);
-  inline void LoadFromInstance(Register dst, int offset, int size);
-  inline void LoadTaggedPointerFromInstance(Register dst, int offset);
+  inline void LoadInstanceFromFrame(Register dst);
+  inline void LoadFromInstance(Register dst, Register instance, int offset,
+                               int size);
+  inline void LoadTaggedPointerFromInstance(Register dst, Register instance,
+                                            int offset);
   inline void SpillInstance(Register instance);
   inline void FillInstanceInto(Register dst);
   inline void LoadTaggedPointer(Register dst, Register src_addr,

src/wasm/baseline/liftoff-compiler.cc

@@ -62,15 +62,16 @@ struct assert_field_size {
 #define WASM_INSTANCE_OBJECT_FIELD_SIZE(name) \
   FIELD_SIZE(WasmInstanceObject::k##name##Offset)
 
-#define LOAD_INSTANCE_FIELD(dst, name, load_size)                              \
-  __ LoadFromInstance(dst, WASM_INSTANCE_OBJECT_FIELD_OFFSET(name),            \
+#define LOAD_INSTANCE_FIELD(dst, name, load_size, pinned)                      \
+  __ LoadFromInstance(dst, LoadInstanceIntoRegister(pinned, dst),              \
+                      WASM_INSTANCE_OBJECT_FIELD_OFFSET(name),                 \
                       assert_field_size<WASM_INSTANCE_OBJECT_FIELD_SIZE(name), \
                                         load_size>::size);
 
-#define LOAD_TAGGED_PTR_INSTANCE_FIELD(dst, name)                         \
-  static_assert(WASM_INSTANCE_OBJECT_FIELD_SIZE(name) == kTaggedSize,     \
-                "field in WasmInstance does not have the expected size"); \
-  __ LoadTaggedPointerFromInstance(dst,                                   \
+#define LOAD_TAGGED_PTR_INSTANCE_FIELD(dst, name, pinned)                      \
+  static_assert(WASM_INSTANCE_OBJECT_FIELD_SIZE(name) == kTaggedSize,          \
+                "field in WasmInstance does not have the expected size");      \
+  __ LoadTaggedPointerFromInstance(dst, LoadInstanceIntoRegister(pinned, dst), \
                                    WASM_INSTANCE_OBJECT_FIELD_OFFSET(name));
 
 #ifdef DEBUG
@@ -663,9 +664,14 @@ class LiftoffCompiler {
         position, regs_to_save, spilled_regs, safepoint_info,
         RegisterOOLDebugSideTableEntry()));
     OutOfLineCode& ool = out_of_line_code_.back();
-    LOAD_INSTANCE_FIELD(limit_address, StackLimitAddress, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(limit_address, StackLimitAddress, kSystemPointerSize,
+                        {});
     __ StackCheck(ool.label.get(), limit_address);
     __ bind(ool.continuation.get());
+    // If the stack check triggers, we lose the cached instance register.
+    // TODO(clemensb): Restore that register in the OOL code so it's always
+    // available at the beginning of the actual function code.
+    __ cache_state()->ClearCachedInstanceRegister();
   }
 
   bool SpillLocalsInitially(FullDecoder* decoder, uint32_t num_params) {
@@ -725,13 +731,14 @@ class LiftoffCompiler {
 
     // Input 0 is the call target, the instance is at 1.
     constexpr int kInstanceParameterIndex = 1;
-    // Check that {kWasmInstanceRegister} matches out call descriptor.
+    // Check that {kWasmInstanceRegister} matches our call descriptor.
     DCHECK_EQ(kWasmInstanceRegister,
               Register::from_code(
                   descriptor_->GetInputLocation(kInstanceParameterIndex)
                       .AsRegister()));
     // Store the instance parameter to a special stack slot.
     __ SpillInstance(kWasmInstanceRegister);
+    __ cache_state()->SetInstanceCacheRegister(kWasmInstanceRegister);
 
     // Process parameters.
     if (num_params) DEBUG_CODE_COMMENT("process parameters");
@@ -797,7 +804,7 @@ class LiftoffCompiler {
       LiftoffRegister array_address =
           pinned.set(__ GetUnusedRegister(kGpReg, pinned));
       LOAD_INSTANCE_FIELD(array_address.gp(), NumLiftoffFunctionCallsArray,
-                          kSystemPointerSize);
+                          kSystemPointerSize, pinned);
 
       // Compute the correct offset in the array.
       uint32_t offset =
@@ -969,13 +976,14 @@ class LiftoffCompiler {
       Register flag = __ GetUnusedRegister(kGpReg, {}).gp();
 
       // Check the "hook on function call" flag. If set, trigger a break.
-      LOAD_INSTANCE_FIELD(flag, HookOnFunctionCallAddress, kSystemPointerSize);
+      LOAD_INSTANCE_FIELD(flag, HookOnFunctionCallAddress, kSystemPointerSize,
+                          {});
       __ Load(LiftoffRegister{flag}, flag, no_reg, 0, LoadType::kI32Load8U, {});
       // Unary "unequal" means "not equals zero".
       __ emit_cond_jump(kUnequal, &do_break, kWasmI32, flag);
 
       // Check if we should stop on "script entry".
-      LOAD_INSTANCE_FIELD(flag, BreakOnEntry, kUInt8Size);
+      LOAD_INSTANCE_FIELD(flag, BreakOnEntry, kUInt8Size, {});
       // Unary "equal" means "equals zero".
       __ emit_cond_jump(kEqual, &no_break, kWasmI32, flag);
 
@@ -1031,6 +1039,8 @@ class LiftoffCompiler {
     // TODO(clemensb): Come up with a better strategy here, involving
     // pre-analysis of the function.
     __ SpillLocals();
+    // Same for the cached instance register.
+    __ cache_state()->ClearCachedInstanceRegister();
 
     __ PrepareLoopArgs(loop->start_merge.arity);
 
@@ -2001,12 +2011,13 @@ class LiftoffCompiler {
                                   LiftoffRegList* pinned, uint32_t* offset) {
     Register addr = pinned->set(__ GetUnusedRegister(kGpReg, {})).gp();
     if (global->mutability && global->imported) {
-      LOAD_INSTANCE_FIELD(addr, ImportedMutableGlobals, kSystemPointerSize);
+      LOAD_INSTANCE_FIELD(addr, ImportedMutableGlobals, kSystemPointerSize,
+                          *pinned);
       __ Load(LiftoffRegister(addr), addr, no_reg,
               global->index * sizeof(Address), kPointerLoadType, *pinned);
       *offset = 0;
     } else {
-      LOAD_INSTANCE_FIELD(addr, GlobalsStart, kSystemPointerSize);
+      LOAD_INSTANCE_FIELD(addr, GlobalsStart, kSystemPointerSize, *pinned);
       *offset = global->offset;
     }
     return addr;
@@ -2018,7 +2029,7 @@ class LiftoffCompiler {
     Register globals_buffer =
         pinned->set(__ GetUnusedRegister(kGpReg, *pinned)).gp();
     LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer,
-                                   ImportedMutableGlobalsBuffers);
+                                   ImportedMutableGlobalsBuffers, *pinned);
     *base = globals_buffer;
     __ LoadTaggedPointer(
         *base, globals_buffer, no_reg,
@@ -2032,7 +2043,7 @@ class LiftoffCompiler {
         pinned->set(__ GetUnusedRegister(kGpReg, *pinned)).gp();
 
     LOAD_INSTANCE_FIELD(imported_mutable_globals, ImportedMutableGlobals,
-                        kSystemPointerSize);
+                        kSystemPointerSize, *pinned);
     *offset = imported_mutable_globals;
     __ Load(LiftoffRegister(*offset), imported_mutable_globals, no_reg,
             global->index * sizeof(Address),
@@ -2065,7 +2076,8 @@ class LiftoffCompiler {
       LiftoffRegList pinned;
       Register globals_buffer =
           pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-      LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer, TaggedGlobalsBuffer);
+      LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer, TaggedGlobalsBuffer,
+                                     pinned);
       Register value = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
       __ LoadTaggedPointer(value, globals_buffer, no_reg,
                            wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
@@ -2106,7 +2118,8 @@ class LiftoffCompiler {
       LiftoffRegList pinned;
       Register globals_buffer =
           pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-      LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer, TaggedGlobalsBuffer);
+      LOAD_TAGGED_PTR_INSTANCE_FIELD(globals_buffer, TaggedGlobalsBuffer,
+                                     pinned);
       LiftoffRegister value = pinned.set(__ PopToRegister(pinned));
       __ StoreTaggedPointer(globals_buffer, no_reg,
                             wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(
@@ -2453,7 +2466,7 @@ class LiftoffCompiler {
     LiftoffRegister end_offset_reg =
         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
     LiftoffRegister mem_size = __ GetUnusedRegister(kGpReg, pinned);
-    LOAD_INSTANCE_FIELD(mem_size.gp(), MemorySize, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(mem_size.gp(), MemorySize, kSystemPointerSize, pinned);
 
     __ LoadConstant(end_offset_reg, WasmValue::ForUintPtr(end_offset));
 
@@ -2574,7 +2587,7 @@ class LiftoffCompiler {
       }
     }
     Register tmp = __ GetUnusedRegister(kGpReg, *pinned).gp();
-    LOAD_INSTANCE_FIELD(tmp, MemoryMask, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(tmp, MemoryMask, kSystemPointerSize, *pinned);
     if (*offset) __ emit_ptrsize_addi(index, index, *offset);
     __ emit_ptrsize_and(index, index, tmp);
     *offset = 0;
@@ -2602,7 +2615,7 @@ class LiftoffCompiler {
     index = AddMemoryMasking(index, &offset, &pinned);
     DEBUG_CODE_COMMENT("load from memory");
     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
     RegClass rc = reg_class_for(value_type);
     LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
     uint32_t protected_load_pc = 0;
@@ -2645,7 +2658,7 @@ class LiftoffCompiler {
     index = AddMemoryMasking(index, &offset, &pinned);
     DEBUG_CODE_COMMENT("load with transformation");
     Register addr = __ GetUnusedRegister(kGpReg, pinned).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
     LiftoffRegister value = __ GetUnusedRegister(reg_class_for(kS128), {});
     uint32_t protected_load_pc = 0;
     __ LoadTransform(value, addr, index, offset, type, transform,
@@ -2687,7 +2700,7 @@ class LiftoffCompiler {
     index = AddMemoryMasking(index, &offset, &pinned);
     DEBUG_CODE_COMMENT("load lane");
     Register addr = __ GetUnusedRegister(kGpReg, pinned).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
     LiftoffRegister result = __ GetUnusedRegister(reg_class_for(kS128), {});
     uint32_t protected_load_pc = 0;
 
@@ -2724,7 +2737,7 @@ class LiftoffCompiler {
     index = AddMemoryMasking(index, &offset, &pinned);
     DEBUG_CODE_COMMENT("store to memory");
     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
     uint32_t protected_store_pc = 0;
     LiftoffRegList outer_pinned;
     if (FLAG_trace_wasm_memory) outer_pinned.set(index);
@@ -2757,7 +2770,7 @@ class LiftoffCompiler {
     index = AddMemoryMasking(index, &offset, &pinned);
     DEBUG_CODE_COMMENT("store lane to memory");
     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
     uint32_t protected_store_pc = 0;
     __ StoreLane(addr, index, offset, value, type, lane, &protected_store_pc);
     if (env_->use_trap_handler) {
@@ -2773,7 +2786,7 @@ class LiftoffCompiler {
 
   void CurrentMemoryPages(FullDecoder* /* decoder */, Value* /* result */) {
     Register mem_size = __ GetUnusedRegister(kGpReg, {}).gp();
-    LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize, {});
     __ emit_ptrsize_shri(mem_size, mem_size, kWasmPageSizeLog2);
     LiftoffRegister result{mem_size};
     if (env_->module->is_memory64 && kNeedI64RegPair) {
@@ -2881,7 +2894,7 @@ class LiftoffCompiler {
     Label cont_false;
     LiftoffRegList pinned;
     LiftoffRegister ref = pinned.set(__ PopToRegister(pinned));
-    Register null = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
+    Register null = __ GetUnusedRegister(kGpReg, pinned).gp();
     LoadNullValue(null, pinned);
     __ emit_cond_jump(kUnequal, &cont_false, ref_object.type, ref.gp(), null);
 
@@ -3558,7 +3571,7 @@ class LiftoffCompiler {
     index = AddMemoryMasking(index, &offset, &pinned);
     DEBUG_CODE_COMMENT("atomic store to memory");
     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
     LiftoffRegList outer_pinned;
     if (FLAG_trace_wasm_memory) outer_pinned.set(index);
     __ AtomicStore(addr, index, offset, value, type, outer_pinned);
@@ -3582,7 +3595,7 @@ class LiftoffCompiler {
     index = AddMemoryMasking(index, &offset, &pinned);
     DEBUG_CODE_COMMENT("atomic load from memory");
     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
     RegClass rc = reg_class_for(value_type);
     LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
     __ AtomicLoad(value, addr, index, offset, type, pinned);
@@ -3630,7 +3643,7 @@ class LiftoffCompiler {
     uintptr_t offset = imm.offset;
     index = AddMemoryMasking(index, &offset, &pinned);
     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
 
     (asm_.*emit_fn)(addr, index, offset, value, result, type);
     __ PushRegister(result_type, result);
@@ -3654,7 +3667,7 @@ class LiftoffCompiler {
     uintptr_t offset = imm.offset;
     index = AddMemoryMasking(index, &offset, &pinned);
     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
     __ emit_i32_add(addr, addr, index);
     pinned.clear(LiftoffRegister(index));
     LiftoffRegister new_value = pinned.set(__ PopToRegister(pinned));
@@ -3686,7 +3699,7 @@ class LiftoffCompiler {
     uintptr_t offset = imm.offset;
     index = AddMemoryMasking(index, &offset, &pinned);
     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned);
     LiftoffRegister result =
         pinned.set(__ GetUnusedRegister(reg_class_for(result_type), pinned));
 
@@ -3989,7 +4002,8 @@ class LiftoffCompiler {
 
     Register seg_size_array =
         pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
-    LOAD_INSTANCE_FIELD(seg_size_array, DataSegmentSizes, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(seg_size_array, DataSegmentSizes, kSystemPointerSize,
+                        pinned);
 
     LiftoffRegister seg_index =
         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
@@ -4101,7 +4115,7 @@ class LiftoffCompiler {
     Register dropped_elem_segments =
         pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
     LOAD_INSTANCE_FIELD(dropped_elem_segments, DroppedElemSegments,
-                        kSystemPointerSize);
+                        kSystemPointerSize, pinned);
 
     LiftoffRegister seg_index =
         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
@@ -4230,7 +4244,7 @@ class LiftoffCompiler {
     LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
     MaybeEmitNullCheck(decoder, obj.gp(), pinned, struct_obj.type);
     LiftoffRegister value =
-        pinned.set(__ GetUnusedRegister(reg_class_for(field_type), pinned));
+        __ GetUnusedRegister(reg_class_for(field_type), pinned);
     LoadObjectField(value, obj.gp(), no_reg, offset, field_type, is_signed,
                     pinned);
     __ PushRegister(field_type.Unpacked(), value);
@@ -4378,7 +4392,7 @@ class LiftoffCompiler {
     LiftoffRegList pinned;
     LiftoffRegister obj = pinned.set(__ PopToRegister(pinned));
     MaybeEmitNullCheck(decoder, obj.gp(), pinned, array_obj.type);
-    LiftoffRegister len = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
+    LiftoffRegister len = __ GetUnusedRegister(kGpReg, pinned);
     int kLengthOffset = wasm::ObjectAccess::ToTagged(WasmArray::kLengthOffset);
     LoadObjectField(len, obj.gp(), no_reg, kLengthOffset, kWasmI32, false,
                     pinned);
@@ -4427,7 +4441,7 @@ class LiftoffCompiler {
 
   void RttCanon(FullDecoder* decoder, uint32_t type_index, Value* result) {
     LiftoffRegister rtt = __ GetUnusedRegister(kGpReg, {});
-    LOAD_TAGGED_PTR_INSTANCE_FIELD(rtt.gp(), ManagedObjectMaps);
+    LOAD_TAGGED_PTR_INSTANCE_FIELD(rtt.gp(), ManagedObjectMaps, {});
     __ LoadTaggedPointer(
         rtt.gp(), rtt.gp(), no_reg,
         wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(type_index), {});
@@ -4808,13 +4822,13 @@ class LiftoffCompiler {
 
       Register imported_targets = tmp;
       LOAD_INSTANCE_FIELD(imported_targets, ImportedFunctionTargets,
-                          kSystemPointerSize);
+                          kSystemPointerSize, pinned);
       __ Load(LiftoffRegister(target), imported_targets, no_reg,
               imm.index * sizeof(Address), kPointerLoadType, pinned);
 
       Register imported_function_refs = tmp;
       LOAD_TAGGED_PTR_INSTANCE_FIELD(imported_function_refs,
-                                     ImportedFunctionRefs);
+                                     ImportedFunctionRefs, pinned);
       Register imported_function_ref = tmp;
       __ LoadTaggedPointer(
           imported_function_ref, imported_function_refs, no_reg,
@@ -4888,7 +4902,8 @@ class LiftoffCompiler {
 
     // Compare against table size stored in
     // {instance->indirect_function_table_size}.
-    LOAD_INSTANCE_FIELD(tmp_const, IndirectFunctionTableSize, kUInt32Size);
+    LOAD_INSTANCE_FIELD(tmp_const, IndirectFunctionTableSize, kUInt32Size,
+                        pinned);
     __ emit_cond_jump(kUnsignedGreaterEqual, invalid_func_label, kWasmI32,
                       index, tmp_const);
 
@@ -4916,7 +4931,8 @@ class LiftoffCompiler {
 
     DEBUG_CODE_COMMENT("Check indirect call signature");
     // Load the signature from {instance->ift_sig_ids[key]}
-    LOAD_INSTANCE_FIELD(table, IndirectFunctionTableSigIds, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(table, IndirectFunctionTableSigIds, kSystemPointerSize,
+                        pinned);
     // Shift {index} by 2 (multiply by 4) to represent kInt32Size items.
     STATIC_ASSERT((1 << 2) == kInt32Size);
     __ emit_i32_shli(index, index, 2);
@@ -4941,7 +4957,7 @@ class LiftoffCompiler {
     // At this point {index} has already been multiplied by kTaggedSize.
 
     // Load the instance from {instance->ift_instances[key]}
-    LOAD_TAGGED_PTR_INSTANCE_FIELD(table, IndirectFunctionTableRefs);
+    LOAD_TAGGED_PTR_INSTANCE_FIELD(table, IndirectFunctionTableRefs, pinned);
     __ LoadTaggedPointer(tmp_const, table, index,
                          ObjectAccess::ElementOffsetInTaggedFixedArray(0),
                          pinned);
@@ -4956,8 +4972,8 @@ class LiftoffCompiler {
     Register* explicit_instance = &tmp_const;
 
     // Load the target from {instance->ift_targets[key]}
-    LOAD_INSTANCE_FIELD(table, IndirectFunctionTableTargets,
-                        kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(table, IndirectFunctionTableTargets, kSystemPointerSize,
+                        pinned);
     __ Load(LiftoffRegister(scratch), table, index, 0, kPointerLoadType,
             pinned);
 
@@ -5194,7 +5210,7 @@ class LiftoffCompiler {
   }
 
   void LoadNullValue(Register null, LiftoffRegList pinned) {
-    LOAD_INSTANCE_FIELD(null, IsolateRoot, kSystemPointerSize);
+    LOAD_INSTANCE_FIELD(null, IsolateRoot, kSystemPointerSize, pinned);
     __ LoadTaggedPointer(null, null, no_reg,
                          IsolateData::root_slot_offset(RootIndex::kNullValue),
                          pinned);
@@ -5372,6 +5388,17 @@ class LiftoffCompiler {
     __ cache_state()->DefineSafepoint(safepoint);
   }
 
+  Register LoadInstanceIntoRegister(LiftoffRegList pinned, Register fallback) {
+    Register instance = __ cache_state()->cached_instance;
+    if (instance == no_reg) {
+      instance = __ cache_state()->TrySetCachedInstanceRegister(
+          pinned | LiftoffRegList::ForRegs(fallback));
+      if (instance == no_reg) instance = fallback;
+      __ LoadInstanceFromFrame(instance);
+    }
+    return instance;
+  }
+
   DISALLOW_IMPLICIT_CONSTRUCTORS(LiftoffCompiler);
 };
 

src/wasm/baseline/liftoff-register.h

@@ -377,6 +377,10 @@ class LiftoffRegList {
     }
     return reg;
   }
+  Register clear(Register reg) { return clear(LiftoffRegister{reg}).gp(); }
+  DoubleRegister clear(DoubleRegister reg) {
+    return clear(LiftoffRegister{reg}).fp();
+  }
 
   bool has(LiftoffRegister reg) const {
     if (reg.is_pair()) {
@@ -385,8 +389,8 @@ class LiftoffRegList {
     }
     return (regs_ & (storage_t{1} << reg.liftoff_code())) != 0;
   }
-  bool has(Register reg) const { return has(LiftoffRegister(reg)); }
-  bool has(DoubleRegister reg) const { return has(LiftoffRegister(reg)); }
+  bool has(Register reg) const { return has(LiftoffRegister{reg}); }
+  bool has(DoubleRegister reg) const { return has(LiftoffRegister{reg}); }
 
   constexpr bool is_empty() const { return regs_ == 0; }
 

src/wasm/baseline/x64/liftoff-assembler-x64.h

@@ -280,10 +280,14 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
   }
 }
 
-void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) {
-  DCHECK_LE(0, offset);
+void LiftoffAssembler::LoadInstanceFromFrame(Register dst) {
   movq(dst, liftoff::GetInstanceOperand());
-  Operand src{dst, offset};
+}
+
+void LiftoffAssembler::LoadFromInstance(Register dst, Register instance,
+                                        int offset, int size) {
+  DCHECK_LE(0, offset);
+  Operand src{instance, offset};
   switch (size) {
     case 1:
       movzxbl(dst, src);
@@ -299,10 +303,11 @@ void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) {
   }
 }
 
-void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) {
+void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst,
+                                                     Register instance,
+                                                     int offset) {
   DCHECK_LE(0, offset);
-  movq(dst, liftoff::GetInstanceOperand());
-  LoadTaggedPointerField(dst, Operand(dst, offset));
+  LoadTaggedPointerField(dst, Operand(instance, offset));
 }
 
 void LiftoffAssembler::SpillInstance(Register instance) {