[fastcall] Enable float support on arm64

This CL adds support for handling calls to C functions with arbitrary
signatures on native arm64. It introduces a new ExternalReference type
FAST_C_CALL.

The CL also splits the 10 bits used by kArchCallCFunction instruction to
store the total number of parameters into two 5-bit values, representing
the number of general purpose and floating point parameters.

Design doc:
https://docs.google.com/document/d/1ZxOF3GSyNmtU0C0YJvrsydPJj35W_tTJZymeXwfDxoI/edit

This CL is partially based on the previous attempt:
https://chromium-review.googlesource.com/c/v8/v8/+/2343072

Bug: chromium:1052746
Change-Id: Ib508626d57da26ec3c9186ee8fc46356e3c87f3a
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3182232Reviewed-by: Georg Neis <neis@chromium.org>
Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Commit-Queue: Maya Lekova <mslekova@chromium.org>
Cr-Commit-Position: refs/heads/main@{#77198}

src/codegen/arm64/macro-assembler-arm64.cc

@@ -1693,6 +1693,7 @@ void TurboAssembler::CallCFunction(ExternalReference function,
 }
 
 static const int kRegisterPassedArguments = 8;
+static const int kFPRegisterPassedArguments = 8;
 
 void TurboAssembler::CallCFunction(Register function, int num_of_reg_args,
                                    int num_of_double_args) {
@@ -1700,17 +1701,6 @@ void TurboAssembler::CallCFunction(Register function, int num_of_reg_args,
   DCHECK_LE(num_of_reg_args + num_of_double_args, kMaxCParameters);
   DCHECK(has_frame());
 
-  // If we're passing doubles, we're limited to the following prototypes
-  // (defined by ExternalReference::Type):
-  //  BUILTIN_COMPARE_CALL:  int f(double, double)
-  //  BUILTIN_FP_FP_CALL:    double f(double, double)
-  //  BUILTIN_FP_CALL:       double f(double)
-  //  BUILTIN_FP_INT_CALL:   double f(double, int)
-  if (num_of_double_args > 0) {
-    DCHECK_LE(num_of_reg_args, 1);
-    DCHECK_LE(num_of_double_args + num_of_reg_args, 2);
-  }
-
   // Save the frame pointer and PC so that the stack layout remains iterable,
   // even without an ExitFrame which normally exists between JS and C frames.
   Register pc_scratch = x4;
@@ -1761,6 +1751,13 @@ void TurboAssembler::CallCFunction(Register function, int num_of_reg_args,
     int claim_slots = RoundUp(num_of_reg_args - kRegisterPassedArguments, 2);
     Drop(claim_slots);
   }
+
+  if (num_of_double_args > kFPRegisterPassedArguments) {
+    // Drop the register passed arguments.
+    int claim_slots =
+        RoundUp(num_of_double_args - kFPRegisterPassedArguments, 2);
+    Drop(claim_slots);
+  }
 }
 
 void TurboAssembler::LoadFromConstantsTable(Register destination,

src/codegen/external-reference.cc

@@ -190,8 +190,8 @@ ExternalReference ExternalReference::Create(const Runtime::Function* f) {
 }
 
 // static
-ExternalReference ExternalReference::Create(Address address) {
-  return ExternalReference(Redirect(address));
+ExternalReference ExternalReference::Create(Address address, Type type) {
+  return ExternalReference(Redirect(address, type));
 }
 
 ExternalReference ExternalReference::isolate_address(Isolate* isolate) {

src/codegen/external-reference.h

@@ -330,6 +330,10 @@ class ExternalReference {
     // ObjectPair f(v8::internal::Arguments).
     BUILTIN_CALL_PAIR,
 
+    // TODO(mslekova): Once FAST_C_CALL is supported in the simulator,
+    // the following four specific types and their special handling
+    // can be removed, as the generic call supports them.
+
     // Builtin that takes float arguments and returns an int.
     // int f(double, double).
     BUILTIN_COMPARE_CALL,
@@ -361,7 +365,11 @@ class ExternalReference {
     // Call to accessor getter callback via InvokeAccessorGetterCallback.
     // void f(Local<Name> property, PropertyCallbackInfo& info,
     //     AccessorNameGetterCallback callback)
-    PROFILING_GETTER_CALL
+    PROFILING_GETTER_CALL,
+
+    // C call, either representing a fast API call or used in tests.
+    // Can have arbitrary signature from the types supported by the fast API.
+    FAST_C_CALL
   };
 
 #define COUNT_EXTERNAL_REFERENCE(name, desc) +1
@@ -382,7 +390,8 @@ class ExternalReference {
   static ExternalReference Create(const Runtime::Function* f);
   static ExternalReference Create(IsolateAddressId id, Isolate* isolate);
   static ExternalReference Create(Runtime::FunctionId id);
-  static V8_EXPORT_PRIVATE ExternalReference Create(Address address);
+  static V8_EXPORT_PRIVATE ExternalReference
+  Create(Address address, Type type = ExternalReference::BUILTIN_CALL);
 
   template <typename SubjectChar, typename PatternChar>
   static ExternalReference search_string_raw();

src/compiler/backend/arm/code-generator-arm.cc

@@ -767,8 +767,9 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       break;
     }
     case kArchPrepareCallCFunction: {
-      int const num_parameters = MiscField::decode(instr->opcode());
-      __ PrepareCallCFunction(num_parameters);
+      int const num_gp_parameters = ParamField::decode(instr->opcode());
+      int const num_fp_parameters = FPParamField::decode(instr->opcode());
+      __ PrepareCallCFunction(num_gp_parameters + num_fp_parameters);
       // Frame alignment requires using FP-relative frame addressing.
       frame_access_state()->SetFrameAccessToFP();
       break;

src/compiler/backend/arm64/code-generator-arm64.cc

@@ -821,7 +821,8 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       AssemblePrepareTailCall();
       break;
     case kArchCallCFunction: {
-      int const num_parameters = MiscField::decode(instr->opcode());
+      int const num_gp_parameters = ParamField::decode(instr->opcode());
+      int const num_fp_parameters = FPParamField::decode(instr->opcode());
       Label return_location;
 #if V8_ENABLE_WEBASSEMBLY
       if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
@@ -832,10 +833,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
 
       if (instr->InputAt(0)->IsImmediate()) {
         ExternalReference ref = i.InputExternalReference(0);
-        __ CallCFunction(ref, num_parameters, 0);
+        __ CallCFunction(ref, num_gp_parameters, num_fp_parameters);
       } else {
         Register func = i.InputRegister(0);
-        __ CallCFunction(func, num_parameters, 0);
+        __ CallCFunction(func, num_gp_parameters, num_fp_parameters);
       }
       __ Bind(&return_location);
 #if V8_ENABLE_WEBASSEMBLY

src/compiler/backend/ia32/code-generator-ia32.cc

@@ -791,8 +791,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
     case kArchPrepareCallCFunction: {
       // Frame alignment requires using FP-relative frame addressing.
       frame_access_state()->SetFrameAccessToFP();
-      int const num_parameters = MiscField::decode(instr->opcode());
-      __ PrepareCallCFunction(num_parameters, i.TempRegister(0));
+      int const num_gp_parameters = ParamField::decode(instr->opcode());
+      int const num_fp_parameters = FPParamField::decode(instr->opcode());
+      __ PrepareCallCFunction(num_gp_parameters + num_fp_parameters,
+                              i.TempRegister(0));
       break;
     }
     case kArchSaveCallerRegisters: {

src/compiler/backend/instruction-codes.h

@@ -341,6 +341,14 @@ using AtomicWidthField = base::BitField<AtomicWidth, 22, 2>;
 using AtomicMemoryOrderField = base::BitField<AtomicMemoryOrder, 24, 2>;
 using AtomicStoreRecordWriteModeField = base::BitField<RecordWriteMode, 26, 4>;
 
+// ParamField and FPParamField overlap with MiscField, as the latter is never
+// used for Call instructions. These 2 fields represent the general purpose
+// and floating point parameter counts of a direct call into C and are given 5
+// bits each, which allow storing a number up to the current maximum parameter
+// count, which is 20 (see kMaxCParameters defined in macro-assembler.h).
+using ParamField = base::BitField<int, 22, 5>;
+using FPParamField = base::BitField<int, 27, 5>;
+
 // This static assertion serves as an early warning if we are about to exhaust
 // the available opcode space. If we are about to exhaust it, we should start
 // looking into options to compress some opcodes (see

src/compiler/backend/instruction-selector.cc

@@ -2914,16 +2914,21 @@ void InstructionSelector::VisitCall(Node* node, BasicBlock* handler) {
   InstructionCode opcode;
   switch (call_descriptor->kind()) {
     case CallDescriptor::kCallAddress: {
-      int misc_field = static_cast<int>(call_descriptor->ParameterCount());
+      int gp_param_count =
+          static_cast<int>(call_descriptor->GPParameterCount());
+      int fp_param_count =
+          static_cast<int>(call_descriptor->FPParameterCount());
 #if ABI_USES_FUNCTION_DESCRIPTORS
-      // Highest misc_field bit is used on AIX to indicate if a CFunction call
-      // has function descriptor or not.
-      STATIC_ASSERT(MiscField::kSize == kHasFunctionDescriptorBitShift + 1);
+      // Highest fp_param_count bit is used on AIX to indicate if a CFunction
+      // call has function descriptor or not.
+      STATIC_ASSERT(ParamField::kSize + FPParamField::kSize ==
+                    kHasFunctionDescriptorBitShift + 1);
       if (!call_descriptor->NoFunctionDescriptor()) {
-        misc_field |= 1 << kHasFunctionDescriptorBitShift;
+        fp_param_count |= 1 << kHasFunctionDescriptorBitShift;
       }
 #endif
-      opcode = kArchCallCFunction | MiscField::encode(misc_field);
+      opcode = kArchCallCFunction | ParamField::encode(gp_param_count) |
+               FPParamField::encode(fp_param_count);
       break;
     }
     case CallDescriptor::kCallCodeObject:

src/compiler/backend/x64/code-generator-x64.cc

@@ -1305,7 +1305,8 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       AssemblePrepareTailCall();
       break;
     case kArchCallCFunction: {
-      int const num_parameters = MiscField::decode(instr->opcode());
+      int const num_gp_parameters = ParamField::decode(instr->opcode());
+      int const num_fp_parameters = FPParamField::decode(instr->opcode());
       Label return_location;
 #if V8_ENABLE_WEBASSEMBLY
       if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
@@ -1317,10 +1318,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
 #endif  // V8_ENABLE_WEBASSEMBLY
       if (HasImmediateInput(instr, 0)) {
         ExternalReference ref = i.InputExternalReference(0);
-        __ CallCFunction(ref, num_parameters);
+        __ CallCFunction(ref, num_gp_parameters + num_fp_parameters);
       } else {
         Register func = i.InputRegister(0);
-        __ CallCFunction(func, num_parameters);
+        __ CallCFunction(func, num_gp_parameters + num_fp_parameters);
       }
       __ bind(&return_location);
 #if V8_ENABLE_WEBASSEMBLY

src/compiler/c-linkage.cc

@@ -66,6 +66,8 @@ namespace {
 // == arm64 ====================================================================
 // ===========================================================================
 #define PARAM_REGISTERS x0, x1, x2, x3, x4, x5, x6, x7
+#define FP_PARAM_REGISTERS d0, d1, d2, d3, d4, d5, d6, d7
+#define FP_RETURN_REGISTER d0
 #define CALLEE_SAVE_REGISTERS                                     \
   (1 << x19.code()) | (1 << x20.code()) | (1 << x21.code()) |     \
       (1 << x22.code()) | (1 << x23.code()) | (1 << x24.code()) | \

src/compiler/effect-control-linearizer.cc

@@ -5171,6 +5171,8 @@ EffectControlLinearizer::AdaptOverloadedFastCallArgument(
     Node* value_is_smi = ObjectIsSmi(node);
     __ GotoIf(value_is_smi, if_error);
 
+    ExternalReference::Type ref_type = ExternalReference::FAST_C_CALL;
+
     switch (arg_type.GetSequenceType()) {
       case CTypeInfo::SequenceType::kIsSequence: {
         CHECK_EQ(arg_type.GetType(), CTypeInfo::Type::kVoid);
@@ -5191,8 +5193,8 @@ EffectControlLinearizer::AdaptOverloadedFastCallArgument(
                                      kNoWriteBarrier),
                  stack_slot, 0, node);
 
-        Node* target_address = __ ExternalConstant(
-            ExternalReference::Create(c_functions[func_index].address));
+        Node* target_address = __ ExternalConstant(ExternalReference::Create(
+            c_functions[func_index].address, ref_type));
         __ Goto(&merge, target_address, stack_slot);
         break;
       }
@@ -5205,8 +5207,8 @@ EffectControlLinearizer::AdaptOverloadedFastCallArgument(
             fast_api_call::GetTypedArrayElementsKind(
                 overloads_resolution_result.element_type),
             &next);
-        Node* target_address = __ ExternalConstant(
-            ExternalReference::Create(c_functions[func_index].address));
+        Node* target_address = __ ExternalConstant(ExternalReference::Create(
+            c_functions[func_index].address, ref_type));
         __ Goto(&merge, target_address, stack_slot);
         break;
       }
@@ -5393,6 +5395,8 @@ Node* EffectControlLinearizer::LowerFastApiCall(Node* node) {
   Node** const inputs = graph()->zone()->NewArray<Node*>(
       kFastTargetAddressInputCount + c_arg_count + n.FastCallExtraInputCount());
 
+  ExternalReference::Type ref_type = ExternalReference::FAST_C_CALL;
+
   // The inputs to {Call} node for the fast call look like:
   // [fast callee, receiver, ... C arguments, [optional Options], effect,
   //  control].
@@ -5404,7 +5408,7 @@ Node* EffectControlLinearizer::LowerFastApiCall(Node* node) {
   // with a Phi node created by AdaptOverloadedFastCallArgument.
   inputs[kFastTargetAddressInputIndex] =
       (c_functions.size() == 1) ? __ ExternalConstant(ExternalReference::Create(
-                                      c_functions[0].address))
+                                      c_functions[0].address, ref_type))
                                 : nullptr;
 
   for (int i = 0; i < c_arg_count; ++i) {

src/compiler/globals.h

@@ -92,7 +92,8 @@ const int kMaxFastLiteralProperties = JSObject::kMaxInObjectProperties;
 // to add support for IA32, because it has a totally different approach
 // (using FP stack). As support is added to more platforms, please make sure
 // to list them here in order to enable tests of this functionality.
-#if defined(V8_TARGET_ARCH_X64)
+#if defined(V8_TARGET_ARCH_X64) || \
+    (defined(V8_TARGET_ARCH_ARM64) && !defined(USE_SIMULATOR))
 #define V8_ENABLE_FP_PARAMS_IN_C_LINKAGE
 #endif
 

src/compiler/linkage.cc

@@ -208,6 +208,18 @@ int CallDescriptor::CalculateFixedFrameSize(CodeKind code_kind) const {
   UNREACHABLE();
 }
 
+void CallDescriptor::ComputeParamCounts() const {
+  gp_param_count_ = 0;
+  fp_param_count_ = 0;
+  for (size_t i = 0; i < ParameterCount(); ++i) {
+    if (IsFloatingPoint(GetParameterType(i).representation())) {
+      ++fp_param_count_.value();
+    } else {
+      ++gp_param_count_.value();
+    }
+  }
+}
+
 CallDescriptor* Linkage::ComputeIncoming(Zone* zone,
                                          OptimizedCompilationInfo* info) {
 #if V8_ENABLE_WEBASSEMBLY

src/compiler/linkage.h

@@ -305,9 +305,27 @@ class V8_EXPORT_PRIVATE CallDescriptor final
   // The number of return values from this call.
   size_t ReturnCount() const { return location_sig_->return_count(); }
 
-  // The number of C parameters to this call.
+  // The number of C parameters to this call. The following invariant
+  // should hold true:
+  // ParameterCount() == GPParameterCount() + FPParameterCount()
   size_t ParameterCount() const { return location_sig_->parameter_count(); }
 
+  // The number of general purpose C parameters to this call.
+  size_t GPParameterCount() const {
+    if (!gp_param_count_) {
+      ComputeParamCounts();
+    }
+    return gp_param_count_.value();
+  }
+
+  // The number of floating point C parameters to this call.
+  size_t FPParameterCount() const {
+    if (!fp_param_count_) {
+      ComputeParamCounts();
+    }
+    return fp_param_count_.value();
+  }
+
   // The number of stack parameter slots to the call.
   size_t ParameterSlotCount() const { return param_slot_count_; }
 
@@ -417,6 +435,8 @@ class V8_EXPORT_PRIVATE CallDescriptor final
   }
 
  private:
+  void ComputeParamCounts() const;
+
   friend class Linkage;
 
   const Kind kind_;
@@ -434,6 +454,9 @@ class V8_EXPORT_PRIVATE CallDescriptor final
   const Flags flags_;
   const StackArgumentOrder stack_order_;
   const char* const debug_name_;
+
+  mutable base::Optional<size_t> gp_param_count_;
+  mutable base::Optional<size_t> fp_param_count_;
 };
 
 DEFINE_OPERATORS_FOR_FLAGS(CallDescriptor::Flags)

src/d8/d8-test.cc

@@ -17,7 +17,8 @@
 // and resetting these counters.
 
 // Make sure to sync the following with src/compiler/globals.h.
-#if defined(V8_TARGET_ARCH_X64)
+#if defined(V8_TARGET_ARCH_X64) || \
+    (defined(V8_TARGET_ARCH_ARM64) && !defined(USE_SIMULATOR))
 #define V8_ENABLE_FP_PARAMS_IN_C_LINKAGE
 #endif
 

src/execution/arm/simulator-arm.cc

@@ -1835,8 +1835,19 @@ void Simulator::SoftwareInterrupt(Instruction* instr) {
 #endif
       } else {
         // builtin call.
+        // FAST_C_CALL is temporarily handled here as well, because we lack
+        // proper support for direct C calls with FP params in the simulator.
+        // The generic BUILTIN_CALL path assumes all parameters are passed in
+        // the GP registers, thus supporting calling the slow callback without
+        // crashing. The reason for that is that in the mjsunit tests we check
+        // the `fast_c_api.supports_fp_params` (which is false on non-simulator
+        // builds for arm/arm64), thus we expect that the slow path will be
+        // called. And since the slow path passes the arguments as a `const
+        // FunctionCallbackInfo<Value>&` (which is a GP argument), the call is
+        // made correctly.
         DCHECK(redirection->type() == ExternalReference::BUILTIN_CALL ||
-               redirection->type() == ExternalReference::BUILTIN_CALL_PAIR);
+               redirection->type() == ExternalReference::BUILTIN_CALL_PAIR ||
+               redirection->type() == ExternalReference::FAST_C_CALL);
         if (::v8::internal::FLAG_trace_sim || !stack_aligned) {
           PrintF(
               "Call to host function at %p "

src/execution/arm64/simulator-arm64.cc

@@ -538,6 +538,17 @@ void Simulator::DoRuntimeCall(Instruction* instr) {
       TraceSim("Type: Unknown.\n");
       UNREACHABLE();
 
+    // FAST_C_CALL is temporarily handled here as well, because we lack
+    // proper support for direct C calls with FP params in the simulator.
+    // The generic BUILTIN_CALL path assumes all parameters are passed in
+    // the GP registers, thus supporting calling the slow callback without
+    // crashing. The reason for that is that in the mjsunit tests we check
+    // the `fast_c_api.supports_fp_params` (which is false on non-simulator
+    // builds for arm/arm64), thus we expect that the slow path will be
+    // called. And since the slow path passes the arguments as a `const
+    // FunctionCallbackInfo<Value>&` (which is a GP argument), the call is
+    // made correctly.
+    case ExternalReference::FAST_C_CALL:
     case ExternalReference::BUILTIN_CALL:
 #if defined(V8_OS_WIN)
     {