MIPS[64]: Add optimizations to memory load/store helper.

The CL replaces several helper functions for memory load/store using
base register and offset with one helper function that contains several
optimizations.

BUG=

Change-Id: I187e7e882131552abd9a0b3a0070d78adefd25b6
Reviewed-on: https://chromium-review.googlesource.com/552119
Commit-Queue: Miran Karić <Miran.Karic@imgtec.com>
Reviewed-by: Ivica Bogosavljevic <ivica.bogosavljevic@imgtec.com>
Cr-Commit-Position: refs/heads/master@{#46420}

src/mips/assembler-mips.cc

@@ -1930,107 +1930,151 @@ void Assembler::lsa(Register rd, Register rt, Register rs, uint8_t sa) {
 
 // ------------Memory-instructions-------------
 
-// Helper for base-reg + offset, when offset is larger than int16.
-void Assembler::LoadRegPlusOffsetToAt(const MemOperand& src) {
-  DCHECK(!src.rm().is(at));
-  if (IsMipsArchVariant(kMips32r6)) {
-    int32_t hi = (src.offset_ >> kLuiShift) & kImm16Mask;
-    if (src.offset_ & kNegOffset) {
-      hi += 1;
-    }
-    aui(at, src.rm(), hi);
-    addiu(at, at, src.offset_ & kImm16Mask);
-  } else {
-    lui(at, (src.offset_ >> kLuiShift) & kImm16Mask);
-    ori(at, at, src.offset_ & kImm16Mask);  // Load 32-bit offset.
-    addu(at, at, src.rm());                 // Add base register.
+void Assembler::AdjustBaseAndOffset(MemOperand& src,
+                                    OffsetAccessType access_type,
+                                    int second_access_add_to_offset) {
+  // This method is used to adjust the base register and offset pair
+  // for a load/store when the offset doesn't fit into int16_t.
+  // It is assumed that 'base + offset' is sufficiently aligned for memory
+  // operands that are machine word in size or smaller. For doubleword-sized
+  // operands it's assumed that 'base' is a multiple of 8, while 'offset'
+  // may be a multiple of 4 (e.g. 4-byte-aligned long and double arguments
+  // and spilled variables on the stack accessed relative to the stack
+  // pointer register).
+  // We preserve the "alignment" of 'offset' by adjusting it by a multiple of 8.
+
+  bool doubleword_aligned = (src.offset() & (kDoubleSize - 1)) == 0;
+  bool two_accesses = static_cast<bool>(access_type) || !doubleword_aligned;
+  DCHECK(second_access_add_to_offset <= 7);  // Must be <= 7.
+
+  // is_int16 must be passed a signed value, hence the static cast below.
+  if (is_int16(src.offset()) &&
+      (!two_accesses || is_int16(static_cast<int32_t>(
+                            src.offset() + second_access_add_to_offset)))) {
+    // Nothing to do: 'offset' (and, if needed, 'offset + 4', or other specified
+    // value) fits into int16_t.
+    return;
   }
-}
 
-// Helper for base-reg + upper part of offset, when offset is larger than int16.
-// Loads higher part of the offset to AT register.
-// Returns lower part of the offset to be used as offset
-// in Load/Store instructions
-int32_t Assembler::LoadRegPlusUpperOffsetPartToAt(const MemOperand& src) {
-  DCHECK(!src.rm().is(at));
-  int32_t hi = (src.offset_ >> kLuiShift) & kImm16Mask;
-  // If the highest bit of the lower part of the offset is 1, this would make
-  // the offset in the load/store instruction negative. We need to compensate
-  // for this by adding 1 to the upper part of the offset.
-  if (src.offset_ & kNegOffset) {
-    hi += 1;
-  }
+  DCHECK(!src.rm().is(
+      at));  // Must not overwrite the register 'base' while loading 'offset'.
 
-  if (IsMipsArchVariant(kMips32r6)) {
-    aui(at, src.rm(), hi);
+#ifdef DEBUG
+  // Remember the "(mis)alignment" of 'offset', it will be checked at the end.
+  uint32_t misalignment = src.offset() & (kDoubleSize - 1);
+#endif
+
+  // Do not load the whole 32-bit 'offset' if it can be represented as
+  // a sum of two 16-bit signed offsets. This can save an instruction or two.
+  // To simplify matters, only do this for a symmetric range of offsets from
+  // about -64KB to about +64KB, allowing further addition of 4 when accessing
+  // 64-bit variables with two 32-bit accesses.
+  constexpr int32_t kMinOffsetForSimpleAdjustment =
+      0x7ff8;  // Max int16_t that's a multiple of 8.
+  constexpr int32_t kMaxOffsetForSimpleAdjustment =
+      2 * kMinOffsetForSimpleAdjustment;
+  if (0 <= src.offset() && src.offset() <= kMaxOffsetForSimpleAdjustment) {
+    addiu(at, src.rm(), kMinOffsetForSimpleAdjustment);
+    src.offset_ -= kMinOffsetForSimpleAdjustment;
+  } else if (-kMaxOffsetForSimpleAdjustment <= src.offset() &&
+             src.offset() < 0) {
+    addiu(at, src.rm(), -kMinOffsetForSimpleAdjustment);
+    src.offset_ += kMinOffsetForSimpleAdjustment;
+  } else if (IsMipsArchVariant(kMips32r6)) {
+    // On r6 take advantage of the aui instruction, e.g.:
+    //   aui   at, base, offset_high
+    //   lw    reg_lo, offset_low(at)
+    //   lw    reg_hi, (offset_low+4)(at)
+    // or when offset_low+4 overflows int16_t:
+    //   aui   at, base, offset_high
+    //   addiu at, at, 8
+    //   lw    reg_lo, (offset_low-8)(at)
+    //   lw    reg_hi, (offset_low-4)(at)
+    int16_t offset_high = static_cast<uint16_t>(src.offset() >> 16);
+    int16_t offset_low = static_cast<uint16_t>(src.offset());
+    offset_high += (offset_low < 0)
+                       ? 1
+                       : 0;  // Account for offset sign extension in load/store.
+    aui(at, src.rm(), offset_high);
+    if (two_accesses && !is_int16(static_cast<int32_t>(
+                            offset_low + second_access_add_to_offset))) {
+      // Avoid overflow in the 16-bit offset of the load/store instruction when
+      // adding 4.
+      addiu(at, at, kDoubleSize);
+      offset_low -= kDoubleSize;
+    }
+    src.offset_ = offset_low;
   } else {
-    lui(at, hi);
-    addu(at, at, src.rm());
+    // Do not load the whole 32-bit 'offset' if it can be represented as
+    // a sum of three 16-bit signed offsets. This can save an instruction.
+    // To simplify matters, only do this for a symmetric range of offsets from
+    // about -96KB to about +96KB, allowing further addition of 4 when accessing
+    // 64-bit variables with two 32-bit accesses.
+    constexpr int32_t kMinOffsetForMediumAdjustment =
+        2 * kMinOffsetForSimpleAdjustment;
+    constexpr int32_t kMaxOffsetForMediumAdjustment =
+        3 * kMinOffsetForSimpleAdjustment;
+    if (0 <= src.offset() && src.offset() <= kMaxOffsetForMediumAdjustment) {
+      addiu(at, src.rm(), kMinOffsetForMediumAdjustment / 2);
+      addiu(at, at, kMinOffsetForMediumAdjustment / 2);
+      src.offset_ -= kMinOffsetForMediumAdjustment;
+    } else if (-kMaxOffsetForMediumAdjustment <= src.offset() &&
+               src.offset() < 0) {
+      addiu(at, src.rm(), -kMinOffsetForMediumAdjustment / 2);
+      addiu(at, at, -kMinOffsetForMediumAdjustment / 2);
+      src.offset_ += kMinOffsetForMediumAdjustment;
+    } else {
+      // Now that all shorter options have been exhausted, load the full 32-bit
+      // offset.
+      int32_t loaded_offset = RoundDown(src.offset(), kDoubleSize);
+      lui(at, (loaded_offset >> kLuiShift) & kImm16Mask);
+      ori(at, at, loaded_offset & kImm16Mask);  // Load 32-bit offset.
+      addu(at, at, src.rm());
+      src.offset_ -= loaded_offset;
+    }
   }
-  return (src.offset_ & kImm16Mask);
-}
+  src.rm_ = at;
 
-// Helper for loading base-reg + upper offset's part to AT reg when we are using
-// two 32-bit loads/stores instead of one 64-bit
-int32_t Assembler::LoadUpperOffsetForTwoMemoryAccesses(const MemOperand& src) {
-  DCHECK(!src.rm().is(at));
-  if (is_int16((src.offset_ & kImm16Mask) + kIntSize)) {
-    // Only if lower part of offset + kIntSize fits in 16bits
-    return LoadRegPlusUpperOffsetPartToAt(src);
+  DCHECK(is_int16(src.offset()));
+  if (two_accesses) {
+    DCHECK(is_int16(
+        static_cast<int32_t>(src.offset() + second_access_add_to_offset)));
   }
-  // In case offset's lower part + kIntSize doesn't fit in 16bits,
-  // load reg + hole offset to AT
-  LoadRegPlusOffsetToAt(src);
-  return 0;
+  DCHECK(misalignment == (src.offset() & (kDoubleSize - 1)));
 }
 
 void Assembler::lb(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset_)) {
-    GenInstrImmediate(LB, rs.rm(), rd, rs.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    GenInstrImmediate(LB, at, rd, off16);
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  GenInstrImmediate(LB, source.rm(), rd, source.offset());
 }
 
 
 void Assembler::lbu(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset_)) {
-    GenInstrImmediate(LBU, rs.rm(), rd, rs.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    GenInstrImmediate(LBU, at, rd, off16);
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  GenInstrImmediate(LBU, source.rm(), rd, source.offset());
 }
 
 
 void Assembler::lh(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset_)) {
-    GenInstrImmediate(LH, rs.rm(), rd, rs.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    GenInstrImmediate(LH, at, rd, off16);
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  GenInstrImmediate(LH, source.rm(), rd, source.offset());
 }
 
 
 void Assembler::lhu(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset_)) {
-    GenInstrImmediate(LHU, rs.rm(), rd, rs.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    GenInstrImmediate(LHU, at, rd, off16);
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  GenInstrImmediate(LHU, source.rm(), rd, source.offset());
 }
 
 
 void Assembler::lw(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset_)) {
-    GenInstrImmediate(LW, rs.rm(), rd, rs.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    GenInstrImmediate(LW, at, rd, off16);
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  GenInstrImmediate(LW, source.rm(), rd, source.offset());
 }
 
 
@@ -2051,32 +2095,23 @@ void Assembler::lwr(Register rd, const MemOperand& rs) {
 
 
 void Assembler::sb(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset_)) {
-    GenInstrImmediate(SB, rs.rm(), rd, rs.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to store.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    GenInstrImmediate(SB, at, rd, off16);
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  GenInstrImmediate(SB, source.rm(), rd, source.offset());
 }
 
 
 void Assembler::sh(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset_)) {
-    GenInstrImmediate(SH, rs.rm(), rd, rs.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to store.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    GenInstrImmediate(SH, at, rd, off16);
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  GenInstrImmediate(SH, source.rm(), rd, source.offset());
 }
 
 
 void Assembler::sw(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset_)) {
-    GenInstrImmediate(SW, rs.rm(), rd, rs.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to store.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    GenInstrImmediate(SW, at, rd, off16);
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  GenInstrImmediate(SW, source.rm(), rd, source.offset());
 }
 
 
@@ -2361,22 +2396,16 @@ void Assembler::seb(Register rd, Register rt) {
 
 // Load, store, move.
 void Assembler::lwc1(FPURegister fd, const MemOperand& src) {
-  if (is_int16(src.offset_)) {
-    GenInstrImmediate(LWC1, src.rm(), fd, src.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src);
-    GenInstrImmediate(LWC1, at, fd, off16);
-  }
+  MemOperand tmp = src;
+  AdjustBaseAndOffset(tmp);
+  GenInstrImmediate(LWC1, tmp.rm(), fd, tmp.offset());
 }
 
 
 void Assembler::swc1(FPURegister fd, const MemOperand& src) {
-  if (is_int16(src.offset_)) {
-    GenInstrImmediate(SWC1, src.rm(), fd, src.offset_);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src);
-    GenInstrImmediate(SWC1, at, fd, off16);
-  }
+  MemOperand tmp = src;
+  AdjustBaseAndOffset(tmp);
+  GenInstrImmediate(SWC1, tmp.rm(), fd, tmp.offset());
 }
 
 
@@ -3052,14 +3081,17 @@ MSA_BRANCH_LIST(MSA_BRANCH)
   V(st_w, ST_W)           \
   V(st_d, ST_D)
 
-#define MSA_LD_ST(name, opcode)                                \
-  void Assembler::name(MSARegister wd, const MemOperand& rs) { \
-    if (is_int10(rs.offset())) {                               \
-      GenInstrMsaMI10(opcode, rs.offset(), rs.rm(), wd);       \
-    } else {                                                   \
-      LoadRegPlusOffsetToAt(rs);                               \
-      GenInstrMsaMI10(opcode, 0, at, wd);                      \
-    }                                                          \
+#define MSA_LD_ST(name, opcode)                                  \
+  void Assembler::name(MSARegister wd, const MemOperand& rs) {   \
+    MemOperand source = rs;                                      \
+    AdjustBaseAndOffset(source);                                 \
+    if (is_int10(source.offset())) {                             \
+      GenInstrMsaMI10(opcode, source.offset(), source.rm(), wd); \
+    } else {                                                     \
+      DCHECK(!rs.rm().is(at));                                   \
+      addiu(at, source.rm(), source.offset());                   \
+      GenInstrMsaMI10(opcode, 0, at, wd);                        \
+    }                                                            \
   }
 
 MSA_LD_ST_LIST(MSA_LD_ST)

src/mips/assembler-mips.h

@@ -1866,10 +1866,18 @@ class Assembler : public AssemblerBase {
   // Load Scaled Address instruction.
   void lsa(Register rd, Register rt, Register rs, uint8_t sa);
 
-  // Helpers.
-  void LoadRegPlusOffsetToAt(const MemOperand& src);
-  int32_t LoadRegPlusUpperOffsetPartToAt(const MemOperand& src);
-  int32_t LoadUpperOffsetForTwoMemoryAccesses(const MemOperand& src);
+  // Readable constants for base and offset adjustment helper, these indicate if
+  // aside from offset, another value like offset + 4 should fit into int16.
+  enum class OffsetAccessType : bool {
+    SINGLE_ACCESS = false,
+    TWO_ACCESSES = true
+  };
+
+  // Helper function for memory load/store using base register and offset.
+  void AdjustBaseAndOffset(
+      MemOperand& src,
+      OffsetAccessType access_type = OffsetAccessType::SINGLE_ACCESS,
+      int second_access_add_to_offset = 4);
 
   int32_t buffer_space() const { return reloc_info_writer.pos() - pc_; }
 

src/mips/macro-assembler-mips.cc

@@ -1133,20 +1133,17 @@ void MacroAssembler::Ulw(Register rd, const MemOperand& rs) {
   } else {
     DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
            IsMipsArchVariant(kLoongson));
-    if (is_int16(rs.offset() + kMipsLwrOffset) &&
-        is_int16(rs.offset() + kMipsLwlOffset)) {
-      if (!rd.is(rs.rm())) {
-        lwr(rd, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
-        lwl(rd, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
-      } else {
-        lwr(at, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
-        lwl(at, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
-        mov(rd, at);
-      }
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      LoadRegPlusOffsetToAt(rs);
-      lwr(rd, MemOperand(at, kMipsLwrOffset));
-      lwl(rd, MemOperand(at, kMipsLwlOffset));
+    DCHECK(kMipsLwrOffset <= 3 && kMipsLwlOffset <= 3);
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3);
+    if (!rd.is(source.rm())) {
+      lwr(rd, MemOperand(source.rm(), source.offset() + kMipsLwrOffset));
+      lwl(rd, MemOperand(source.rm(), source.offset() + kMipsLwlOffset));
+    } else {
+      lwr(at, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
+      lwl(at, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
+      mov(rd, at);
     }
   }
 }
@@ -1155,20 +1152,18 @@ void MacroAssembler::Ulw(Register rd, const MemOperand& rs) {
 void MacroAssembler::Usw(Register rd, const MemOperand& rs) {
   DCHECK(!rd.is(at));
   DCHECK(!rs.rm().is(at));
+  DCHECK(!rd.is(rs.rm()));
   if (IsMipsArchVariant(kMips32r6)) {
     sw(rd, rs);
   } else {
     DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
            IsMipsArchVariant(kLoongson));
-    if (is_int16(rs.offset() + kMipsSwrOffset) &&
-        is_int16(rs.offset() + kMipsSwlOffset)) {
-      swr(rd, MemOperand(rs.rm(), rs.offset() + kMipsSwrOffset));
-      swl(rd, MemOperand(rs.rm(), rs.offset() + kMipsSwlOffset));
-    } else {
-      LoadRegPlusOffsetToAt(rs);
-      swr(rd, MemOperand(at, kMipsSwrOffset));
-      swl(rd, MemOperand(at, kMipsSwlOffset));
-    }
+    DCHECK(kMipsSwrOffset <= 3 && kMipsSwlOffset <= 3);
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3);
+    swr(rd, MemOperand(source.rm(), source.offset() + kMipsSwrOffset));
+    swl(rd, MemOperand(source.rm(), source.offset() + kMipsSwlOffset));
   }
 }
 
@@ -1180,22 +1175,24 @@ void MacroAssembler::Ulh(Register rd, const MemOperand& rs) {
   } else {
     DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
            IsMipsArchVariant(kLoongson));
-    if (is_int16(rs.offset()) && is_int16(rs.offset() + 1)) {
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1);
+    if (source.rm().is(at)) {
 #if defined(V8_TARGET_LITTLE_ENDIAN)
-      lbu(at, rs);
-      lb(rd, MemOperand(rs.rm(), rs.offset() + 1));
+      lb(rd, MemOperand(source.rm(), source.offset() + 1));
+      lbu(at, source);
 #elif defined(V8_TARGET_BIG_ENDIAN)
-      lbu(at, MemOperand(rs.rm(), rs.offset() + 1));
-      lb(rd, rs);
+      lb(rd, source);
+      lbu(at, MemOperand(source.rm(), source.offset() + 1));
 #endif
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      LoadRegPlusOffsetToAt(rs);
+    } else {
 #if defined(V8_TARGET_LITTLE_ENDIAN)
-      lb(rd, MemOperand(at, 1));
-      lbu(at, MemOperand(at, 0));
+      lbu(at, source);
+      lb(rd, MemOperand(source.rm(), source.offset() + 1));
 #elif defined(V8_TARGET_BIG_ENDIAN)
-      lb(rd, MemOperand(at, 0));
-      lbu(at, MemOperand(at, 1));
+      lbu(at, MemOperand(source.rm(), source.offset() + 1));
+      lb(rd, source);
 #endif
     }
     sll(rd, rd, 8);
@@ -1211,22 +1208,24 @@ void MacroAssembler::Ulhu(Register rd, const MemOperand& rs) {
   } else {
     DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
            IsMipsArchVariant(kLoongson));
-    if (is_int16(rs.offset()) && is_int16(rs.offset() + 1)) {
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1);
+    if (source.rm().is(at)) {
 #if defined(V8_TARGET_LITTLE_ENDIAN)
-      lbu(at, rs);
-      lbu(rd, MemOperand(rs.rm(), rs.offset() + 1));
+      lbu(rd, MemOperand(source.rm(), source.offset() + 1));
+      lbu(at, source);
 #elif defined(V8_TARGET_BIG_ENDIAN)
-      lbu(at, MemOperand(rs.rm(), rs.offset() + 1));
-      lbu(rd, rs);
+      lbu(rd, source);
+      lbu(at, MemOperand(source.rm(), source.offset() + 1));
 #endif
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      LoadRegPlusOffsetToAt(rs);
+    } else {
 #if defined(V8_TARGET_LITTLE_ENDIAN)
-      lbu(rd, MemOperand(at, 1));
-      lbu(at, MemOperand(at, 0));
+      lbu(at, source);
+      lbu(rd, MemOperand(source.rm(), source.offset() + 1));
 #elif defined(V8_TARGET_BIG_ENDIAN)
-      lbu(rd, MemOperand(at, 0));
-      lbu(at, MemOperand(at, 1));
+      lbu(at, MemOperand(source.rm(), source.offset() + 1));
+      lbu(rd, source);
 #endif
     }
     sll(rd, rd, 8);
@@ -1245,11 +1244,8 @@ void MacroAssembler::Ush(Register rd, const MemOperand& rs, Register scratch) {
     DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r1) ||
            IsMipsArchVariant(kLoongson));
     MemOperand source = rs;
-    // If offset > 16 bits, load address to at with offset 0.
-    if (!is_int16(rs.offset()) || !is_int16(rs.offset() + 1)) {
-      LoadRegPlusOffsetToAt(rs);
-      source = MemOperand(at, 0);
-    }
+    // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1);
 
     if (!scratch.is(rd)) {
       mov(scratch, rd);
@@ -1324,70 +1320,44 @@ void MacroAssembler::Usdc1(FPURegister fd, const MemOperand& rs,
 void MacroAssembler::Ldc1(FPURegister fd, const MemOperand& src) {
   // Workaround for non-8-byte alignment of HeapNumber, convert 64-bit
   // load to two 32-bit loads.
+  DCHECK(Register::kMantissaOffset <= 4 && Register::kExponentOffset <= 4);
+  MemOperand tmp = src;
+  AdjustBaseAndOffset(tmp, OffsetAccessType::TWO_ACCESSES);
+  lwc1(fd, MemOperand(tmp.rm(), tmp.offset() + Register::kMantissaOffset));
   if (IsFp32Mode()) {  // fp32 mode.
-    if (is_int16(src.offset()) && is_int16(src.offset() + kIntSize)) {
-      lwc1(fd, MemOperand(src.rm(), src.offset() + Register::kMantissaOffset));
-      FPURegister nextfpreg;
-      nextfpreg.setcode(fd.code() + 1);
-      lwc1(nextfpreg,
-           MemOperand(src.rm(), src.offset() + Register::kExponentOffset));
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      int32_t off16 = LoadUpperOffsetForTwoMemoryAccesses(src);
-      lwc1(fd, MemOperand(at, off16 + Register::kMantissaOffset));
-      FPURegister nextfpreg;
-      nextfpreg.setcode(fd.code() + 1);
-      lwc1(nextfpreg, MemOperand(at, off16 + Register::kExponentOffset));
-    }
+    FPURegister nextfpreg;
+    nextfpreg.setcode(fd.code() + 1);
+    lwc1(nextfpreg,
+         MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset));
   } else {
     DCHECK(IsFp64Mode() || IsFpxxMode());
     // Currently we support FPXX and FP64 on Mips32r2 and Mips32r6
     DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
-    if (is_int16(src.offset()) && is_int16(src.offset() + kIntSize)) {
-      lwc1(fd, MemOperand(src.rm(), src.offset() + Register::kMantissaOffset));
-      lw(at, MemOperand(src.rm(), src.offset() + Register::kExponentOffset));
-      mthc1(at, fd);
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      int32_t off16 = LoadUpperOffsetForTwoMemoryAccesses(src);
-      lwc1(fd, MemOperand(at, off16 + Register::kMantissaOffset));
-      lw(at, MemOperand(at, off16 + Register::kExponentOffset));
-      mthc1(at, fd);
-    }
+    DCHECK(!src.rm().is(at));
+    lw(at, MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset));
+    mthc1(at, fd);
   }
 }
 
 void MacroAssembler::Sdc1(FPURegister fd, const MemOperand& src) {
   // Workaround for non-8-byte alignment of HeapNumber, convert 64-bit
   // store to two 32-bit stores.
-  DCHECK(!src.rm().is(at));
-  DCHECK(!src.rm().is(t8));
+  DCHECK(Register::kMantissaOffset <= 4 && Register::kExponentOffset <= 4);
+  MemOperand tmp = src;
+  AdjustBaseAndOffset(tmp, OffsetAccessType::TWO_ACCESSES);
+  swc1(fd, MemOperand(tmp.rm(), tmp.offset() + Register::kMantissaOffset));
   if (IsFp32Mode()) {  // fp32 mode.
-    if (is_int16(src.offset()) && is_int16(src.offset() + kIntSize)) {
-      swc1(fd, MemOperand(src.rm(), src.offset() + Register::kMantissaOffset));
-      FPURegister nextfpreg;
-      nextfpreg.setcode(fd.code() + 1);
-      swc1(nextfpreg,
-           MemOperand(src.rm(), src.offset() + Register::kExponentOffset));
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      int32_t off16 = LoadUpperOffsetForTwoMemoryAccesses(src);
-      swc1(fd, MemOperand(at, off16 + Register::kMantissaOffset));
-      FPURegister nextfpreg;
-      nextfpreg.setcode(fd.code() + 1);
-      swc1(nextfpreg, MemOperand(at, off16 + Register::kExponentOffset));
-    }
+    FPURegister nextfpreg;
+    nextfpreg.setcode(fd.code() + 1);
+    swc1(nextfpreg,
+         MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset));
   } else {
     DCHECK(IsFp64Mode() || IsFpxxMode());
     // Currently we support FPXX and FP64 on Mips32r2 and Mips32r6
     DCHECK(IsMipsArchVariant(kMips32r2) || IsMipsArchVariant(kMips32r6));
-    if (is_int16(src.offset()) && is_int16(src.offset() + kIntSize)) {
-      swc1(fd, MemOperand(src.rm(), src.offset() + Register::kMantissaOffset));
-      mfhc1(at, fd);
-      sw(at, MemOperand(src.rm(), src.offset() + Register::kExponentOffset));
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      int32_t off16 = LoadUpperOffsetForTwoMemoryAccesses(src);
-      swc1(fd, MemOperand(at, off16 + Register::kMantissaOffset));
-      mfhc1(t8, fd);
-      sw(t8, MemOperand(at, off16 + Register::kExponentOffset));
-    }
+    DCHECK(!src.rm().is(at));
+    mfhc1(at, fd);
+    sw(at, MemOperand(tmp.rm(), tmp.offset() + Register::kExponentOffset));
   }
 }
 

src/mips64/assembler-mips64.cc

@@ -2090,31 +2090,130 @@ void Assembler::dlsa(Register rd, Register rt, Register rs, uint8_t sa) {
 
 // ------------Memory-instructions-------------
 
-// Helper for base-reg + offset, when offset is larger than int16.
-void Assembler::LoadRegPlusOffsetToAt(const MemOperand& src) {
-  DCHECK(!src.rm().is(at));
-  DCHECK(is_int32(src.offset_));
+void Assembler::AdjustBaseAndOffset(MemOperand& src,
+                                    OffsetAccessType access_type,
+                                    int second_access_add_to_offset) {
+  // This method is used to adjust the base register and offset pair
+  // for a load/store when the offset doesn't fit into int16_t.
+  // It is assumed that 'base + offset' is sufficiently aligned for memory
+  // operands that are machine word in size or smaller. For doubleword-sized
+  // operands it's assumed that 'base' is a multiple of 8, while 'offset'
+  // may be a multiple of 4 (e.g. 4-byte-aligned long and double arguments
+  // and spilled variables on the stack accessed relative to the stack
+  // pointer register).
+  // We preserve the "alignment" of 'offset' by adjusting it by a multiple of 8.
+
+  bool doubleword_aligned = (src.offset() & (kDoubleSize - 1)) == 0;
+  bool two_accesses = static_cast<bool>(access_type) || !doubleword_aligned;
+  DCHECK(second_access_add_to_offset <= 7);  // Must be <= 7.
+
+  // is_int16 must be passed a signed value, hence the static cast below.
+  if (is_int16(src.offset()) &&
+      (!two_accesses || is_int16(static_cast<int32_t>(
+                            src.offset() + second_access_add_to_offset)))) {
+    // Nothing to do: 'offset' (and, if needed, 'offset + 4', or other specified
+    // value) fits into int16_t.
+    return;
+  }
 
-  if (kArchVariant == kMips64r6) {
-    int32_t hi = (src.offset_ >> kLuiShift) & kImm16Mask;
-    if (src.offset_ & kNegOffset) {
-      if ((hi & kNegOffset) != ((hi + 1) & kNegOffset)) {
-        lui(at, (src.offset_ >> kLuiShift) & kImm16Mask);
-        ori(at, at, src.offset_ & kImm16Mask);  // Load 32-bit offset.
-        daddu(at, at, src.rm());                // Add base register.
-        return;
-      }
+  DCHECK(!src.rm().is(
+      at));  // Must not overwrite the register 'base' while loading 'offset'.
 
-      hi += 1;
+#ifdef DEBUG
+  // Remember the "(mis)alignment" of 'offset', it will be checked at the end.
+  uint32_t misalignment = src.offset() & (kDoubleSize - 1);
+#endif
+
+  // Do not load the whole 32-bit 'offset' if it can be represented as
+  // a sum of two 16-bit signed offsets. This can save an instruction or two.
+  // To simplify matters, only do this for a symmetric range of offsets from
+  // about -64KB to about +64KB, allowing further addition of 4 when accessing
+  // 64-bit variables with two 32-bit accesses.
+  constexpr int32_t kMinOffsetForSimpleAdjustment =
+      0x7ff8;  // Max int16_t that's a multiple of 8.
+  constexpr int32_t kMaxOffsetForSimpleAdjustment =
+      2 * kMinOffsetForSimpleAdjustment;
+
+  if (0 <= src.offset() && src.offset() <= kMaxOffsetForSimpleAdjustment) {
+    daddiu(at, src.rm(), kMinOffsetForSimpleAdjustment);
+    src.offset_ -= kMinOffsetForSimpleAdjustment;
+  } else if (-kMaxOffsetForSimpleAdjustment <= src.offset() &&
+             src.offset() < 0) {
+    daddiu(at, src.rm(), -kMinOffsetForSimpleAdjustment);
+    src.offset_ += kMinOffsetForSimpleAdjustment;
+  } else if (kArchVariant == kMips64r6) {
+    // On r6 take advantage of the daui instruction, e.g.:
+    //    daui   AT, base, offset_high
+    //   [dahi   AT, 1]                       // When `offset` is close to +2GB.
+    //    lw     reg_lo, offset_low(AT)
+    //   [lw     reg_hi, (offset_low+4)(AT)]  // If misaligned 64-bit load.
+    // or when offset_low+4 overflows int16_t:
+    //    daui   AT, base, offset_high
+    //    daddiu AT, AT, 8
+    //    lw     reg_lo, (offset_low-8)(AT)
+    //    lw     reg_hi, (offset_low-4)(AT)
+    int16_t offset_low = static_cast<uint16_t>(src.offset());
+    int32_t offset_low32 = offset_low;
+    int16_t offset_high = static_cast<uint16_t>(src.offset() >> 16);
+    bool increment_hi16 = offset_low < 0;
+    bool overflow_hi16 = false;
+
+    if (increment_hi16) {
+      offset_high++;
+      overflow_hi16 = (offset_high == -32768);
     }
+    daui(at, src.rm(), offset_high);
 
-    daui(at, src.rm(), hi);
-    daddiu(at, at, src.offset_ & kImm16Mask);
+    if (overflow_hi16) {
+      dahi(at, 1);
+    }
+
+    if (two_accesses && !is_int16(static_cast<int32_t>(
+                            offset_low32 + second_access_add_to_offset))) {
+      // Avoid overflow in the 16-bit offset of the load/store instruction when
+      // adding 4.
+      daddiu(at, at, kDoubleSize);
+      offset_low32 -= kDoubleSize;
+    }
+
+    src.offset_ = offset_low32;
   } else {
-    lui(at, (src.offset_ >> kLuiShift) & kImm16Mask);
-    ori(at, at, src.offset_ & kImm16Mask);  // Load 32-bit offset.
-    daddu(at, at, src.rm());                // Add base register.
+    // Do not load the whole 32-bit 'offset' if it can be represented as
+    // a sum of three 16-bit signed offsets. This can save an instruction.
+    // To simplify matters, only do this for a symmetric range of offsets from
+    // about -96KB to about +96KB, allowing further addition of 4 when accessing
+    // 64-bit variables with two 32-bit accesses.
+    constexpr int32_t kMinOffsetForMediumAdjustment =
+        2 * kMinOffsetForSimpleAdjustment;
+    constexpr int32_t kMaxOffsetForMediumAdjustment =
+        3 * kMinOffsetForSimpleAdjustment;
+    if (0 <= src.offset() && src.offset() <= kMaxOffsetForMediumAdjustment) {
+      daddiu(at, src.rm(), kMinOffsetForMediumAdjustment / 2);
+      daddiu(at, at, kMinOffsetForMediumAdjustment / 2);
+      src.offset_ -= kMinOffsetForMediumAdjustment;
+    } else if (-kMaxOffsetForMediumAdjustment <= src.offset() &&
+               src.offset() < 0) {
+      daddiu(at, src.rm(), -kMinOffsetForMediumAdjustment / 2);
+      daddiu(at, at, -kMinOffsetForMediumAdjustment / 2);
+      src.offset_ += kMinOffsetForMediumAdjustment;
+    } else {
+      // Now that all shorter options have been exhausted, load the full 32-bit
+      // offset.
+      int32_t loaded_offset = RoundDown(src.offset(), kDoubleSize);
+      lui(at, (loaded_offset >> kLuiShift) & kImm16Mask);
+      ori(at, at, loaded_offset & kImm16Mask);  // Load 32-bit offset.
+      daddu(at, at, src.rm());
+      src.offset_ -= loaded_offset;
+    }
+  }
+  src.rm_ = at;
+
+  DCHECK(is_int16(src.offset()));
+  if (two_accesses) {
+    DCHECK(is_int16(
+        static_cast<int32_t>(src.offset() + second_access_add_to_offset)));
   }
+  DCHECK(misalignment == (src.offset() & (kDoubleSize - 1)));
 }
 
 void Assembler::lb(Register rd, const MemOperand& rs) {
@@ -3285,14 +3384,17 @@ MSA_BRANCH_LIST(MSA_BRANCH)
   V(st_w, ST_W)           \
   V(st_d, ST_D)
 
-#define MSA_LD_ST(name, opcode)                                \
-  void Assembler::name(MSARegister wd, const MemOperand& rs) { \
-    if (is_int10(rs.offset())) {                               \
-      GenInstrMsaMI10(opcode, rs.offset(), rs.rm(), wd);       \
-    } else {                                                   \
-      LoadRegPlusOffsetToAt(rs);                               \
-      GenInstrMsaMI10(opcode, 0, at, wd);                      \
-    }                                                          \
+#define MSA_LD_ST(name, opcode)                                  \
+  void Assembler::name(MSARegister wd, const MemOperand& rs) {   \
+    MemOperand source = rs;                                      \
+    AdjustBaseAndOffset(source);                                 \
+    if (is_int10(source.offset())) {                             \
+      GenInstrMsaMI10(opcode, source.offset(), source.rm(), wd); \
+    } else {                                                     \
+      DCHECK(!rs.rm().is(at));                                   \
+      daddiu(at, source.rm(), source.offset());                  \
+      GenInstrMsaMI10(opcode, 0, at, wd);                        \
+    }                                                            \
   }
 
 MSA_LD_ST_LIST(MSA_LD_ST)

src/mips64/assembler-mips64.h

@@ -1918,8 +1918,18 @@ class Assembler : public AssemblerBase {
   void lsa(Register rd, Register rt, Register rs, uint8_t sa);
   void dlsa(Register rd, Register rt, Register rs, uint8_t sa);
 
-  // Helpers.
-  void LoadRegPlusOffsetToAt(const MemOperand& src);
+  // Readable constants for base and offset adjustment helper, these indicate if
+  // aside from offset, another value like offset + 4 should fit into int16.
+  enum class OffsetAccessType : bool {
+    SINGLE_ACCESS = false,
+    TWO_ACCESSES = true
+  };
+
+  // Helper function for memory load/store using base register and offset.
+  void AdjustBaseAndOffset(
+      MemOperand& src,
+      OffsetAccessType access_type = OffsetAccessType::SINGLE_ACCESS,
+      int second_access_add_to_offset = 4);
 
   inline static void set_target_internal_reference_encoded_at(Address pc,
                                                               Address target);

src/mips64/macro-assembler-mips64.cc

@@ -194,62 +194,6 @@ MemOperand MacroAssembler::SafepointRegistersAndDoublesSlot(Register reg) {
   return MemOperand(sp, doubles_size + register_offset);
 }
 
-// Helper for base-reg + offset, when offset is larger than int16.
-void MacroAssembler::LoadRegPlusOffsetToAt(const MemOperand& src) {
-  DCHECK(!src.rm().is(at));
-  DCHECK(is_int32(src.offset()));
-
-  if (kArchVariant == kMips64r6) {
-    int32_t hi = (src.offset() >> kLuiShift) & kImm16Mask;
-    if (src.offset() & kNegOffset) {
-      if ((hi & kNegOffset) != ((hi + 1) & kNegOffset)) {
-        lui(at, (src.offset() >> kLuiShift) & kImm16Mask);
-        ori(at, at, src.offset() & kImm16Mask);  // Load 32-bit offset.
-        daddu(at, at, src.rm());                 // Add base register.
-        return;
-      }
-
-      hi += 1;
-    }
-
-    daui(at, src.rm(), hi);
-    daddiu(at, at, src.offset() & kImm16Mask);
-  } else {
-    lui(at, (src.offset() >> kLuiShift) & kImm16Mask);
-    ori(at, at, src.offset() & kImm16Mask);  // Load 32-bit offset.
-    daddu(at, at, src.rm());                 // Add base register.
-  }
-}
-
-// Helper for base-reg + upper part of offset, when offset is larger than int16.
-// Loads higher part of the offset to AT register.
-// Returns lower part of the offset to be used as offset
-// in Load/Store instructions
-int32_t MacroAssembler::LoadRegPlusUpperOffsetPartToAt(const MemOperand& src) {
-  DCHECK(!src.rm().is(at));
-  DCHECK(is_int32(src.offset()));
-  int32_t hi = (src.offset() >> kLuiShift) & kImm16Mask;
-  // If the highest bit of the lower part of the offset is 1, this would make
-  // the offset in the load/store instruction negative. We need to compensate
-  // for this by adding 1 to the upper part of the offset.
-  if (src.offset() & kNegOffset) {
-    if ((hi & kNegOffset) != ((hi + 1) & kNegOffset)) {
-      LoadRegPlusOffsetToAt(src);
-      return 0;
-    }
-
-    hi += 1;
-  }
-
-  if (kArchVariant == kMips64r6) {
-    daui(at, src.rm(), hi);
-  } else {
-    lui(at, hi);
-    daddu(at, at, src.rm());
-  }
-  return (src.offset() & kImm16Mask);
-}
-
 void MacroAssembler::InNewSpace(Register object,
                                 Register scratch,
                                 Condition cc,
@@ -1321,20 +1265,17 @@ void MacroAssembler::Ulw(Register rd, const MemOperand& rs) {
     Lw(rd, rs);
   } else {
     DCHECK(kArchVariant == kMips64r2);
-    if (is_int16(rs.offset() + kMipsLwrOffset) &&
-        is_int16(rs.offset() + kMipsLwlOffset)) {
-      if (!rd.is(rs.rm())) {
-        lwr(rd, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
-        lwl(rd, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
-      } else {
-        lwr(at, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
-        lwl(at, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
-        mov(rd, at);
-      }
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      LoadRegPlusOffsetToAt(rs);
-      lwr(rd, MemOperand(at, kMipsLwrOffset));
-      lwl(rd, MemOperand(at, kMipsLwlOffset));
+    DCHECK(kMipsLwrOffset <= 3 && kMipsLwlOffset <= 3);
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3);
+    if (!rd.is(source.rm())) {
+      lwr(rd, MemOperand(source.rm(), source.offset() + kMipsLwrOffset));
+      lwl(rd, MemOperand(source.rm(), source.offset() + kMipsLwlOffset));
+    } else {
+      lwr(at, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
+      lwl(at, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
+      mov(rd, at);
     }
   }
 }
@@ -1353,19 +1294,17 @@ void MacroAssembler::Ulwu(Register rd, const MemOperand& rs) {
 void MacroAssembler::Usw(Register rd, const MemOperand& rs) {
   DCHECK(!rd.is(at));
   DCHECK(!rs.rm().is(at));
+  DCHECK(!rd.is(rs.rm()));
   if (kArchVariant == kMips64r6) {
     Sw(rd, rs);
   } else {
     DCHECK(kArchVariant == kMips64r2);
-    if (is_int16(rs.offset() + kMipsSwrOffset) &&
-        is_int16(rs.offset() + kMipsSwlOffset)) {
-      swr(rd, MemOperand(rs.rm(), rs.offset() + kMipsSwrOffset));
-      swl(rd, MemOperand(rs.rm(), rs.offset() + kMipsSwlOffset));
-    } else {
-      LoadRegPlusOffsetToAt(rs);
-      swr(rd, MemOperand(at, kMipsSwrOffset));
-      swl(rd, MemOperand(at, kMipsSwlOffset));
-    }
+    DCHECK(kMipsSwrOffset <= 3 && kMipsSwlOffset <= 3);
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3);
+    swr(rd, MemOperand(source.rm(), source.offset() + kMipsSwrOffset));
+    swl(rd, MemOperand(source.rm(), source.offset() + kMipsSwlOffset));
   }
 }
 
@@ -1376,22 +1315,24 @@ void MacroAssembler::Ulh(Register rd, const MemOperand& rs) {
     Lh(rd, rs);
   } else {
     DCHECK(kArchVariant == kMips64r2);
-    if (is_int16(rs.offset()) && is_int16(rs.offset() + 1)) {
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1);
+    if (source.rm().is(at)) {
 #if defined(V8_TARGET_LITTLE_ENDIAN)
-      Lbu(at, rs);
-      Lb(rd, MemOperand(rs.rm(), rs.offset() + 1));
+      Lb(rd, MemOperand(source.rm(), source.offset() + 1));
+      Lbu(at, source);
 #elif defined(V8_TARGET_BIG_ENDIAN)
-      Lbu(at, MemOperand(rs.rm(), rs.offset() + 1));
-      Lb(rd, rs);
+      Lb(rd, source);
+      Lbu(at, MemOperand(source.rm(), source.offset() + 1));
 #endif
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      LoadRegPlusOffsetToAt(rs);
+    } else {
 #if defined(V8_TARGET_LITTLE_ENDIAN)
-      Lb(rd, MemOperand(at, 1));
-      Lbu(at, MemOperand(at, 0));
+      Lbu(at, source);
+      Lb(rd, MemOperand(source.rm(), source.offset() + 1));
 #elif defined(V8_TARGET_BIG_ENDIAN)
-      Lb(rd, MemOperand(at, 0));
-      Lbu(at, MemOperand(at, 1));
+      Lbu(at, MemOperand(source.rm(), source.offset() + 1));
+      Lb(rd, source);
 #endif
     }
     dsll(rd, rd, 8);
@@ -1406,22 +1347,24 @@ void MacroAssembler::Ulhu(Register rd, const MemOperand& rs) {
     Lhu(rd, rs);
   } else {
     DCHECK(kArchVariant == kMips64r2);
-    if (is_int16(rs.offset()) && is_int16(rs.offset() + 1)) {
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1);
+    if (source.rm().is(at)) {
 #if defined(V8_TARGET_LITTLE_ENDIAN)
-      Lbu(at, rs);
-      Lbu(rd, MemOperand(rs.rm(), rs.offset() + 1));
+      Lbu(rd, MemOperand(source.rm(), source.offset() + 1));
+      Lbu(at, source);
 #elif defined(V8_TARGET_BIG_ENDIAN)
-      Lbu(at, MemOperand(rs.rm(), rs.offset() + 1));
-      Lbu(rd, rs);
+      Lbu(rd, source);
+      Lbu(at, MemOperand(source.rm(), source.offset() + 1));
 #endif
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      LoadRegPlusOffsetToAt(rs);
+    } else {
 #if defined(V8_TARGET_LITTLE_ENDIAN)
-      Lbu(rd, MemOperand(at, 1));
-      Lbu(at, MemOperand(at, 0));
+      Lbu(at, source);
+      Lbu(rd, MemOperand(source.rm(), source.offset() + 1));
 #elif defined(V8_TARGET_BIG_ENDIAN)
-      Lbu(rd, MemOperand(at, 0));
-      Lbu(at, MemOperand(at, 1));
+      Lbu(at, MemOperand(source.rm(), source.offset() + 1));
+      Lbu(rd, source);
 #endif
     }
     dsll(rd, rd, 8);
@@ -1439,11 +1382,8 @@ void MacroAssembler::Ush(Register rd, const MemOperand& rs, Register scratch) {
   } else {
     DCHECK(kArchVariant == kMips64r2);
     MemOperand source = rs;
-    // If offset > 16 bits, load address to at with offset 0.
-    if (!is_int16(rs.offset()) || !is_int16(rs.offset() + 1)) {
-      LoadRegPlusOffsetToAt(rs);
-      source = MemOperand(at, 0);
-    }
+    // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1);
 
     if (!scratch.is(rd)) {
       mov(scratch, rd);
@@ -1468,20 +1408,17 @@ void MacroAssembler::Uld(Register rd, const MemOperand& rs) {
     Ld(rd, rs);
   } else {
     DCHECK(kArchVariant == kMips64r2);
-    if (is_int16(rs.offset() + kMipsLdrOffset) &&
-        is_int16(rs.offset() + kMipsLdlOffset)) {
-      if (!rd.is(rs.rm())) {
-        ldr(rd, MemOperand(rs.rm(), rs.offset() + kMipsLdrOffset));
-        ldl(rd, MemOperand(rs.rm(), rs.offset() + kMipsLdlOffset));
-      } else {
-        ldr(at, MemOperand(rs.rm(), rs.offset() + kMipsLdrOffset));
-        ldl(at, MemOperand(rs.rm(), rs.offset() + kMipsLdlOffset));
-        mov(rd, at);
-      }
-    } else {  // Offset > 16 bits, use multiple instructions to load.
-      LoadRegPlusOffsetToAt(rs);
-      ldr(rd, MemOperand(at, kMipsLdrOffset));
-      ldl(rd, MemOperand(at, kMipsLdlOffset));
+    DCHECK(kMipsLdrOffset <= 7 && kMipsLdlOffset <= 7);
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 7 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 7);
+    if (!rd.is(source.rm())) {
+      ldr(rd, MemOperand(source.rm(), source.offset() + kMipsLdrOffset));
+      ldl(rd, MemOperand(source.rm(), source.offset() + kMipsLdlOffset));
+    } else {
+      ldr(at, MemOperand(rs.rm(), rs.offset() + kMipsLdrOffset));
+      ldl(at, MemOperand(rs.rm(), rs.offset() + kMipsLdlOffset));
+      mov(rd, at);
     }
   }
 }
@@ -1505,15 +1442,12 @@ void MacroAssembler::Usd(Register rd, const MemOperand& rs) {
     Sd(rd, rs);
   } else {
     DCHECK(kArchVariant == kMips64r2);
-    if (is_int16(rs.offset() + kMipsSdrOffset) &&
-        is_int16(rs.offset() + kMipsSdlOffset)) {
-      sdr(rd, MemOperand(rs.rm(), rs.offset() + kMipsSdrOffset));
-      sdl(rd, MemOperand(rs.rm(), rs.offset() + kMipsSdlOffset));
-    } else {
-      LoadRegPlusOffsetToAt(rs);
-      sdr(rd, MemOperand(at, kMipsSdrOffset));
-      sdl(rd, MemOperand(at, kMipsSdlOffset));
-    }
+    DCHECK(kMipsSdrOffset <= 7 && kMipsSdlOffset <= 7);
+    MemOperand source = rs;
+    // Adjust offset for two accesses and check if offset + 7 fits into int16_t.
+    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 7);
+    sdr(rd, MemOperand(source.rm(), source.offset() + kMipsSdrOffset));
+    sdl(rd, MemOperand(source.rm(), source.offset() + kMipsSdlOffset));
   }
 }
 
@@ -1573,139 +1507,93 @@ void MacroAssembler::Usdc1(FPURegister fd, const MemOperand& rs,
 }
 
 void MacroAssembler::Lb(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    lb(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    lb(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  lb(rd, source);
 }
 
 void MacroAssembler::Lbu(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    lbu(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    lbu(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  lbu(rd, source);
 }
 
 void MacroAssembler::Sb(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    sb(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to store.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    sb(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  sb(rd, source);
 }
 
 void MacroAssembler::Lh(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    lh(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    lh(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  lh(rd, source);
 }
 
 void MacroAssembler::Lhu(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    lhu(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    lhu(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  lhu(rd, source);
 }
 
 void MacroAssembler::Sh(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    sh(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to store.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    sh(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  sh(rd, source);
 }
 
 void MacroAssembler::Lw(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    lw(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    lw(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  lw(rd, source);
 }
 
 void MacroAssembler::Lwu(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    lwu(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    lwu(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  lwu(rd, source);
 }
 
 void MacroAssembler::Sw(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    sw(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to store.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    sw(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  sw(rd, source);
 }
 
 void MacroAssembler::Ld(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    ld(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    ld(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  ld(rd, source);
 }
 
 void MacroAssembler::Sd(Register rd, const MemOperand& rs) {
-  if (is_int16(rs.offset())) {
-    sd(rd, rs);
-  } else {  // Offset > 16 bits, use multiple instructions to store.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(rs);
-    sd(rd, MemOperand(at, off16));
-  }
+  MemOperand source = rs;
+  AdjustBaseAndOffset(source);
+  sd(rd, source);
 }
 
 void MacroAssembler::Lwc1(FPURegister fd, const MemOperand& src) {
-  if (is_int16(src.offset())) {
-    lwc1(fd, src);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src);
-    lwc1(fd, MemOperand(at, off16));
-  }
+  MemOperand tmp = src;
+  AdjustBaseAndOffset(tmp);
+  lwc1(fd, tmp);
 }
 
 void MacroAssembler::Swc1(FPURegister fs, const MemOperand& src) {
-  if (is_int16(src.offset())) {
-    swc1(fs, src);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src);
-    swc1(fs, MemOperand(at, off16));
-  }
+  MemOperand tmp = src;
+  AdjustBaseAndOffset(tmp);
+  swc1(fs, tmp);
 }
 
 void MacroAssembler::Ldc1(FPURegister fd, const MemOperand& src) {
-  if (is_int16(src.offset())) {
-    ldc1(fd, src);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src);
-    ldc1(fd, MemOperand(at, off16));
-  }
+  MemOperand tmp = src;
+  AdjustBaseAndOffset(tmp);
+  ldc1(fd, tmp);
 }
 
 void MacroAssembler::Sdc1(FPURegister fs, const MemOperand& src) {
-  DCHECK(!src.rm().is(at));
-  if (is_int16(src.offset())) {
-    sdc1(fs, src);
-  } else {  // Offset > 16 bits, use multiple instructions to load.
-    int32_t off16 = LoadRegPlusUpperOffsetPartToAt(src);
-    sdc1(fs, MemOperand(at, off16));
-  }
+  MemOperand tmp = src;
+  AdjustBaseAndOffset(tmp);
+  sdc1(fs, tmp);
 }
 
 void MacroAssembler::li(Register dst, Handle<Object> value, LiFlags mode) {

src/mips64/macro-assembler-mips64.h

@@ -1873,10 +1873,6 @@ const Operand& rt = Operand(zero_reg), BranchDelaySlot bd = PROTECT
   MemOperand SafepointRegisterSlot(Register reg);
   MemOperand SafepointRegistersAndDoublesSlot(Register reg);
 
-  // Helpers.
-  void LoadRegPlusOffsetToAt(const MemOperand& src);
-  int32_t LoadRegPlusUpperOffsetPartToAt(const MemOperand& src);
-
   bool has_frame_;
   bool has_double_zero_reg_set_;
   Isolate* isolate_;