[nojit] Convert generated memcpy functions into builtins

On ia32, arm and mips we generate miscellaneous memcpy-related functions
at runtime:

arm: memcpy for uint8-uint8 and uint16-uint8 {dest-source} pairs.
ia32: memmove
mips: memcpy uint8-uint8

In jitless mode, runtime codegen is disallowed, so these must be
converted into builtins.

As far as I can tell, the mips64 files were dead code (#ifdef'd to
V8_HOST_ARCH_MIPS instead of MIPS64).

Note also the slightly changed implementation of ia32's MemMove's
jump tables.

Bug: v8:8675
Change-Id: I5dc2a50fcbad332ce9f78228425b987b0d9acdf3
Reviewed-on: https://chromium-review.googlesource.com/c/1407067Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#58839}

BUILD.gn

@@ -2685,7 +2685,6 @@ v8_source_set("v8_base") {
       "src/ia32/assembler-ia32-inl.h",
       "src/ia32/assembler-ia32.cc",
       "src/ia32/assembler-ia32.h",
-      "src/ia32/codegen-ia32.cc",
       "src/ia32/constants-ia32.h",
       "src/ia32/cpu-ia32.cc",
       "src/ia32/deoptimizer-ia32.cc",
@@ -2749,7 +2748,6 @@ v8_source_set("v8_base") {
       "src/arm/assembler-arm-inl.h",
       "src/arm/assembler-arm.cc",
       "src/arm/assembler-arm.h",
-      "src/arm/codegen-arm.cc",
       "src/arm/constants-arm.cc",
       "src/arm/constants-arm.h",
       "src/arm/cpu-arm.cc",
@@ -2835,7 +2833,6 @@ v8_source_set("v8_base") {
       "src/mips/assembler-mips-inl.h",
       "src/mips/assembler-mips.cc",
       "src/mips/assembler-mips.h",
-      "src/mips/codegen-mips.cc",
       "src/mips/constants-mips.cc",
       "src/mips/constants-mips.h",
       "src/mips/cpu-mips.cc",
@@ -2863,7 +2860,6 @@ v8_source_set("v8_base") {
       "src/mips64/assembler-mips64-inl.h",
       "src/mips64/assembler-mips64.cc",
       "src/mips64/assembler-mips64.h",
-      "src/mips64/codegen-mips64.cc",
       "src/mips64/constants-mips64.cc",
       "src/mips64/constants-mips64.h",
       "src/mips64/cpu-mips64.cc",

src/arm/codegen-arm.cc

-// Copyright 2012 the V8 project authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#if V8_TARGET_ARCH_ARM
-
-#include <memory>
-
-#include "src/arm/assembler-arm-inl.h"
-#include "src/arm/simulator-arm.h"
-#include "src/macro-assembler.h"
-
-namespace v8 {
-namespace internal {
-
-#define __ masm.
-
-#if defined(V8_HOST_ARCH_ARM)
-
-MemCopyUint8Function CreateMemCopyUint8Function(MemCopyUint8Function stub) {
-#if defined(USE_SIMULATOR)
-  return stub;
-#else
-  v8::PageAllocator* page_allocator = GetPlatformPageAllocator();
-  size_t allocated = 0;
-  byte* buffer = AllocatePage(page_allocator,
-                              page_allocator->GetRandomMmapAddr(), &allocated);
-  if (buffer == nullptr) return stub;
-
-  MacroAssembler masm(AssemblerOptions{}, buffer, static_cast<int>(allocated));
-
-  Register dest = r0;
-  Register src = r1;
-  Register chars = r2;
-  Register temp1 = r3;
-  Label less_4;
-
-  {
-    UseScratchRegisterScope temps(&masm);
-    Register temp2 = temps.Acquire();
-    Label loop;
-
-    __ bic(temp2, chars, Operand(0x3), SetCC);
-    __ b(&less_4, eq);
-    __ add(temp2, dest, temp2);
-
-    __ bind(&loop);
-    __ ldr(temp1, MemOperand(src, 4, PostIndex));
-    __ str(temp1, MemOperand(dest, 4, PostIndex));
-    __ cmp(dest, temp2);
-    __ b(&loop, ne);
-  }
-
-  __ bind(&less_4);
-  __ mov(chars, Operand(chars, LSL, 31), SetCC);
-  // bit0 => Z (ne), bit1 => C (cs)
-  __ ldrh(temp1, MemOperand(src, 2, PostIndex), cs);
-  __ strh(temp1, MemOperand(dest, 2, PostIndex), cs);
-  __ ldrb(temp1, MemOperand(src), ne);
-  __ strb(temp1, MemOperand(dest), ne);
-  __ Ret();
-
-  CodeDesc desc;
-  masm.GetCode(nullptr, &desc);
-  DCHECK(!RelocInfo::RequiresRelocationAfterCodegen(desc));
-
-  Assembler::FlushICache(buffer, allocated);
-  CHECK(SetPermissions(page_allocator, buffer, allocated,
-                       PageAllocator::kReadExecute));
-  return FUNCTION_CAST<MemCopyUint8Function>(buffer);
-#endif
-}
-
-
-// Convert 8 to 16. The number of character to copy must be at least 8.
-MemCopyUint16Uint8Function CreateMemCopyUint16Uint8Function(
-    MemCopyUint16Uint8Function stub) {
-#if defined(USE_SIMULATOR)
-  return stub;
-#else
-  v8::PageAllocator* page_allocator = GetPlatformPageAllocator();
-  size_t allocated = 0;
-  byte* buffer = AllocatePage(page_allocator,
-                              page_allocator->GetRandomMmapAddr(), &allocated);
-  if (buffer == nullptr) return stub;
-
-  MacroAssembler masm(AssemblerOptions{}, buffer, static_cast<int>(allocated));
-
-  Register dest = r0;
-  Register src = r1;
-  Register chars = r2;
-
-  {
-    UseScratchRegisterScope temps(&masm);
-
-    Register temp1 = r3;
-    Register temp2 = temps.Acquire();
-    Register temp3 = lr;
-    Register temp4 = r4;
-    Label loop;
-    Label not_two;
-
-    __ Push(lr, r4);
-    __ bic(temp2, chars, Operand(0x3));
-    __ add(temp2, dest, Operand(temp2, LSL, 1));
-
-    __ bind(&loop);
-    __ ldr(temp1, MemOperand(src, 4, PostIndex));
-    __ uxtb16(temp3, temp1);
-    __ uxtb16(temp4, temp1, 8);
-    __ pkhbt(temp1, temp3, Operand(temp4, LSL, 16));
-    __ str(temp1, MemOperand(dest));
-    __ pkhtb(temp1, temp4, Operand(temp3, ASR, 16));
-    __ str(temp1, MemOperand(dest, 4));
-    __ add(dest, dest, Operand(8));
-    __ cmp(dest, temp2);
-    __ b(&loop, ne);
-
-    __ mov(chars, Operand(chars, LSL, 31), SetCC);  // bit0 => ne, bit1 => cs
-    __ b(&not_two, cc);
-    __ ldrh(temp1, MemOperand(src, 2, PostIndex));
-    __ uxtb(temp3, temp1, 8);
-    __ mov(temp3, Operand(temp3, LSL, 16));
-    __ uxtab(temp3, temp3, temp1);
-    __ str(temp3, MemOperand(dest, 4, PostIndex));
-    __ bind(&not_two);
-    __ ldrb(temp1, MemOperand(src), ne);
-    __ strh(temp1, MemOperand(dest), ne);
-    __ Pop(pc, r4);
-  }
-
-  CodeDesc desc;
-  masm.GetCode(nullptr, &desc);
-
-  Assembler::FlushICache(buffer, allocated);
-  CHECK(SetPermissions(page_allocator, buffer, allocated,
-                       PageAllocator::kReadExecute));
-  return FUNCTION_CAST<MemCopyUint16Uint8Function>(buffer);
-#endif
-}
-#endif
-
-#undef __
-
-}  // namespace internal
-}  // namespace v8
-
-#endif  // V8_TARGET_ARCH_ARM

src/builtins/arm/builtins-arm.cc

@@ -3141,6 +3141,84 @@ void Builtins::Generate_DirectCEntry(MacroAssembler* masm) {
   __ ldr(pc, MemOperand(sp, 0));  // Return to calling code.
 }
 
+void Builtins::Generate_MemCopyUint8Uint8(MacroAssembler* masm) {
+  Register dest = r0;
+  Register src = r1;
+  Register chars = r2;
+  Register temp1 = r3;
+  Label less_4;
+
+  {
+    UseScratchRegisterScope temps(masm);
+    Register temp2 = temps.Acquire();
+    Label loop;
+
+    __ bic(temp2, chars, Operand(0x3), SetCC);
+    __ b(&less_4, eq);
+    __ add(temp2, dest, temp2);
+
+    __ bind(&loop);
+    __ ldr(temp1, MemOperand(src, 4, PostIndex));
+    __ str(temp1, MemOperand(dest, 4, PostIndex));
+    __ cmp(dest, temp2);
+    __ b(&loop, ne);
+  }
+
+  __ bind(&less_4);
+  __ mov(chars, Operand(chars, LSL, 31), SetCC);
+  // bit0 => Z (ne), bit1 => C (cs)
+  __ ldrh(temp1, MemOperand(src, 2, PostIndex), cs);
+  __ strh(temp1, MemOperand(dest, 2, PostIndex), cs);
+  __ ldrb(temp1, MemOperand(src), ne);
+  __ strb(temp1, MemOperand(dest), ne);
+  __ Ret();
+}
+
+void Builtins::Generate_MemCopyUint16Uint8(MacroAssembler* masm) {
+  Register dest = r0;
+  Register src = r1;
+  Register chars = r2;
+
+  {
+    UseScratchRegisterScope temps(masm);
+
+    Register temp1 = r3;
+    Register temp2 = temps.Acquire();
+    Register temp3 = lr;
+    Register temp4 = r4;
+    Label loop;
+    Label not_two;
+
+    __ Push(lr, r4);
+    __ bic(temp2, chars, Operand(0x3));
+    __ add(temp2, dest, Operand(temp2, LSL, 1));
+
+    __ bind(&loop);
+    __ ldr(temp1, MemOperand(src, 4, PostIndex));
+    __ uxtb16(temp3, temp1);
+    __ uxtb16(temp4, temp1, 8);
+    __ pkhbt(temp1, temp3, Operand(temp4, LSL, 16));
+    __ str(temp1, MemOperand(dest));
+    __ pkhtb(temp1, temp4, Operand(temp3, ASR, 16));
+    __ str(temp1, MemOperand(dest, 4));
+    __ add(dest, dest, Operand(8));
+    __ cmp(dest, temp2);
+    __ b(&loop, ne);
+
+    __ mov(chars, Operand(chars, LSL, 31), SetCC);  // bit0 => ne, bit1 => cs
+    __ b(&not_two, cc);
+    __ ldrh(temp1, MemOperand(src, 2, PostIndex));
+    __ uxtb(temp3, temp1, 8);
+    __ mov(temp3, Operand(temp3, LSL, 16));
+    __ uxtab(temp3, temp3, temp1);
+    __ str(temp3, MemOperand(dest, 4, PostIndex));
+    __ bind(&not_two);
+    __ ldrb(temp1, MemOperand(src), ne);
+    __ strh(temp1, MemOperand(dest), ne);
+    __ Pop(pc, r4);
+  }
+}
+
 #undef __
 
 }  // namespace internal

src/builtins/builtins-definitions.h

@@ -1354,6 +1354,9 @@ namespace internal {
   TFS(SetProperty, kReceiver, kKey, kValue)                                    \
   TFS(SetPropertyInLiteral, kReceiver, kKey, kValue)                           \
   ASM(MathPowInternal, Dummy)                                                  \
+  ASM(MemCopyUint8Uint8, CCall)                                                \
+  ASM(MemCopyUint16Uint8, CCall)                                               \
+  ASM(MemMove, CCall)                                                          \
                                                                                \
   /* Trace */                                                                  \
   CPP(IsTraceCategoryEnabled)                                                  \

src/builtins/builtins-internal-gen.cc

@@ -758,6 +758,24 @@ void Builtins::Generate_CEntry_Return2_SaveFPRegs_ArgvOnStack_BuiltinExit(
   Generate_CEntry(masm, 2, kSaveFPRegs, kArgvOnStack, true);
 }
 
+#if !defined(V8_TARGET_ARCH_ARM) && !defined(V8_TARGET_ARCH_MIPS)
+void Builtins::Generate_MemCopyUint8Uint8(MacroAssembler* masm) {
+  masm->Call(BUILTIN_CODE(masm->isolate(), Illegal), RelocInfo::CODE_TARGET);
+}
+#endif  // !defined(V8_TARGET_ARCH_ARM) && !defined(V8_TARGET_ARCH_MIPS)
+
+#ifndef V8_TARGET_ARCH_ARM
+void Builtins::Generate_MemCopyUint16Uint8(MacroAssembler* masm) {
+  masm->Call(BUILTIN_CODE(masm->isolate(), Illegal), RelocInfo::CODE_TARGET);
+}
+#endif  // V8_TARGET_ARCH_ARM
+
+#ifndef V8_TARGET_ARCH_IA32
+void Builtins::Generate_MemMove(MacroAssembler* masm) {
+  masm->Call(BUILTIN_CODE(masm->isolate(), Illegal), RelocInfo::CODE_TARGET);
+}
+#endif  // V8_TARGET_ARCH_IA32
+
 // ES6 [[Get]] operation.
 TF_BUILTIN(GetProperty, CodeStubAssembler) {
   Node* object = Parameter(Descriptor::kObject);

src/interface-descriptors.h

@@ -463,6 +463,9 @@ class V8_EXPORT_PRIVATE VoidDescriptor : public CallInterfaceDescriptor {
 // descriptor associated.
 typedef VoidDescriptor DummyDescriptor;
 
+// Dummy descriptor that marks builtins with C calling convention.
+typedef VoidDescriptor CCallDescriptor;
+
 class AllocateDescriptor : public CallInterfaceDescriptor {
  public:
   DEFINE_PARAMETERS_NO_CONTEXT(kRequestedSize)

src/isolate.cc

@@ -183,6 +183,22 @@ void FreeCurrentEmbeddedBlob() {
   sticky_embedded_blob_size_ = 0;
 }
 
+// static
+bool Isolate::CurrentEmbeddedBlobIsBinaryEmbedded() {
+  // In some situations, we must be able to rely on the embedded blob being
+  // immortal immovable. This is the case if the blob is binary-embedded.
+  // See blob lifecycle controls above for descriptions of when the current
+  // embedded blob may change (e.g. in tests or mksnapshot). If the blob is
+  // binary-embedded, it is immortal immovable.
+  const uint8_t* blob =
+      current_embedded_blob_.load(std::memory_order::memory_order_relaxed);
+  if (blob == nullptr) return false;
+#ifdef V8_MULTI_SNAPSHOTS
+  if (blob == TrustedEmbeddedBlob()) return true;
+#endif
+  return blob == DefaultEmbeddedBlob();
+}
+
 void Isolate::SetEmbeddedBlob(const uint8_t* blob, uint32_t blob_size) {
   CHECK_NOT_NULL(blob);
 
@@ -304,7 +320,6 @@ void Isolate::InitializeOncePerProcess() {
   base::Relaxed_Store(&isolate_key_created_, 1);
 #endif
   per_isolate_thread_data_key_ = base::Thread::CreateThreadLocalKey();
-  init_memcopy_functions();
 }
 
 Address Isolate::get_address_from_id(IsolateAddressId id) {
@@ -3283,6 +3298,10 @@ bool Isolate::Init(StartupDeserializer* des) {
     CreateAndSetEmbeddedBlob();
   }
 
+  // Initialize custom memcopy and memmove functions (must happen after
+  // embedded blob setup).
+  init_memcopy_functions();
+
   if (FLAG_log_internal_timer_events) {
     set_event_logger(Logger::DefaultEventLoggerSentinel);
   }

src/isolate.h

@@ -1484,6 +1484,7 @@ class Isolate final : private HiddenFactory {
 
   static const uint8_t* CurrentEmbeddedBlob();
   static uint32_t CurrentEmbeddedBlobSize();
+  static bool CurrentEmbeddedBlobIsBinaryEmbedded();
 
   // These always return the same result as static methods above, but don't
   // access the global atomic variable (and thus *might be* slightly faster).

src/memcopy.cc

@@ -4,6 +4,8 @@
 
 #include "src/memcopy.h"
 
+#include "src/snapshot/embedded-data.h"
+
 namespace v8 {
 namespace internal {
 
@@ -15,9 +17,6 @@ static void MemMoveWrapper(void* dest, const void* src, size_t size) {
 // Initialize to library version so we can call this at any time during startup.
 static MemMoveFunction memmove_function = &MemMoveWrapper;
 
-// Defined in codegen-ia32.cc.
-MemMoveFunction CreateMemMoveFunction();
-
 // Copy memory area to disjoint memory area.
 V8_EXPORT_PRIVATE void MemMove(void* dest, const void* src, size_t size) {
   if (size == 0) return;
@@ -25,7 +24,6 @@ V8_EXPORT_PRIVATE void MemMove(void* dest, const void* src, size_t size) {
   // on all architectures we currently support.
   (*memmove_function)(dest, src, size);
 }
-
 #elif V8_OS_POSIX && V8_HOST_ARCH_ARM
 void MemCopyUint16Uint8Wrapper(uint16_t* dest, const uint8_t* src,
                                size_t chars) {
@@ -39,34 +37,33 @@ V8_EXPORT_PRIVATE MemCopyUint8Function memcopy_uint8_function =
     &MemCopyUint8Wrapper;
 MemCopyUint16Uint8Function memcopy_uint16_uint8_function =
     &MemCopyUint16Uint8Wrapper;
-// Defined in codegen-arm.cc.
-MemCopyUint8Function CreateMemCopyUint8Function(MemCopyUint8Function stub);
-MemCopyUint16Uint8Function CreateMemCopyUint16Uint8Function(
-    MemCopyUint16Uint8Function stub);
-
 #elif V8_OS_POSIX && V8_HOST_ARCH_MIPS
 V8_EXPORT_PRIVATE MemCopyUint8Function memcopy_uint8_function =
     &MemCopyUint8Wrapper;
-// Defined in codegen-mips.cc.
-MemCopyUint8Function CreateMemCopyUint8Function(MemCopyUint8Function stub);
 #endif
 
-static bool g_memcopy_functions_initialized = false;
-
 void init_memcopy_functions() {
-  if (g_memcopy_functions_initialized) return;
-  g_memcopy_functions_initialized = true;
 #if V8_TARGET_ARCH_IA32
-  MemMoveFunction generated_memmove = CreateMemMoveFunction();
-  if (generated_memmove != nullptr) {
-    memmove_function = generated_memmove;
+  if (Isolate::CurrentEmbeddedBlobIsBinaryEmbedded()) {
+    EmbeddedData d = EmbeddedData::FromBlob();
+    memmove_function = reinterpret_cast<MemMoveFunction>(
+        d.InstructionStartOfBuiltin(Builtins::kMemMove));
   }
 #elif V8_OS_POSIX && V8_HOST_ARCH_ARM
-  memcopy_uint8_function = CreateMemCopyUint8Function(&MemCopyUint8Wrapper);
+  if (Isolate::CurrentEmbeddedBlobIsBinaryEmbedded()) {
+    EmbeddedData d = EmbeddedData::FromBlob();
+    memcopy_uint8_function = reinterpret_cast<MemCopyUint8Function>(
+        d.InstructionStartOfBuiltin(Builtins::kMemCopyUint8Uint8));
     memcopy_uint16_uint8_function =
-      CreateMemCopyUint16Uint8Function(&MemCopyUint16Uint8Wrapper);
+        reinterpret_cast<MemCopyUint16Uint8Function>(
+            d.InstructionStartOfBuiltin(Builtins::kMemCopyUint16Uint8));
+  }
 #elif V8_OS_POSIX && V8_HOST_ARCH_MIPS
-  memcopy_uint8_function = CreateMemCopyUint8Function(&MemCopyUint8Wrapper);
+  if (Isolate::CurrentEmbeddedBlobIsBinaryEmbedded()) {
+    EmbeddedData d = EmbeddedData::FromBlob();
+    memcopy_uint8_function = reinterpret_cast<MemCopyUint8Function>(
+        d.InstructionStartOfBuiltin(Builtins::kMemCopyUint8Uint8));
+  }
 #endif
 }
 

src/memcopy.h

@@ -18,9 +18,8 @@ namespace internal {
 typedef uintptr_t Address;
 
 // ----------------------------------------------------------------------------
-// Generated memcpy/memmove
+// Generated memcpy/memmove for ia32, arm, and mips.
 
-// Initializes the codegen support that depends on CPU features.
 void init_memcopy_functions();
 
 #if defined(V8_TARGET_ARCH_IA32)
@@ -148,12 +147,8 @@ inline void MemsetPointer(Address* dest, Address value, size_t counter) {
 #if V8_HOST_ARCH_IA32
 #define STOS "stosl"
 #elif V8_HOST_ARCH_X64
-#if V8_HOST_ARCH_32_BIT
-#define STOS "addr32 stosl"
-#else
 #define STOS "stosq"
 #endif
-#endif
 
 #if defined(MEMORY_SANITIZER)
   // MemorySanitizer does not understand inline assembly.