Port direct call from JavaScript to native RegExp to x64

Code tested on both Linux and Windows.

Added a bit more abstraction to calling a C function from generated code.

Minor tweaks to the ia32 version.
Review URL: http://codereview.chromium.org/548179

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@3740 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/assembler.cc

@@ -655,7 +655,7 @@ ExternalReference ExternalReference::re_check_stack_guard_state() {
 #elif V8_TARGET_ARCH_ARM
   function = FUNCTION_ADDR(RegExpMacroAssemblerARM::CheckStackGuardState);
 #else
-  UNREACHABLE("Unexpected architecture");
+  UNREACHABLE();
 #endif
   return ExternalReference(Redirect(function));
 }

src/ia32/assembler-ia32.h

@@ -229,8 +229,9 @@ enum ScaleFactor {
   times_2 = 1,
   times_4 = 2,
   times_8 = 3,
-  times_pointer_size = times_4,
-  times_half_pointer_size = times_2
+  times_int_size = times_4,
+  times_half_pointer_size = times_2,
+  times_pointer_size = times_4
 };
 
 

src/ia32/codegen-ia32.cc

@@ -5134,7 +5134,7 @@ void CodeGenerator::GenerateFastCharCodeAt(ZoneList<Expression*>* args) {
   // flat string in a cons string).  If that is not the case we would rather go
   // to the runtime system now, to flatten the string.
   __ mov(temp.reg(), FieldOperand(object.reg(), ConsString::kSecondOffset));
-  __ cmp(Operand(temp.reg()), Immediate(Handle<String>(Heap::empty_string())));
+  __ cmp(Operand(temp.reg()), Factory::empty_string());
   __ j(not_equal, &slow_case);
   // Get the first of the two strings.
   __ mov(object.reg(), FieldOperand(object.reg(), ConsString::kFirstOffset));
@@ -8399,8 +8399,12 @@ void ArgumentsAccessStub::GenerateNewObject(MacroAssembler* masm) {
 
 
 void RegExpExecStub::Generate(MacroAssembler* masm) {
-  // Just jump directly to runtime if regexp entry in generated code is turned
-  // off.
+  // Just jump directly to runtime if native RegExp is not selected at compile
+  // time or if regexp entry in generated code is turned off runtime switch or
+  // at compilation.
+#ifndef V8_NATIVE_REGEXP
+  __ TailCallRuntime(ExternalReference(Runtime::kRegExpExec), 4, 1);
+#else  // V8_NATIVE_REGEXP
   if (!FLAG_regexp_entry_native) {
     __ TailCallRuntime(ExternalReference(Runtime::kRegExpExec), 4, 1);
     return;
@@ -8438,12 +8442,12 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   __ j(not_equal, &runtime);
   // Check that the RegExp has been compiled (data contains a fixed array).
   __ mov(ecx, FieldOperand(eax, JSRegExp::kDataOffset));
-#ifdef DEBUG
-  __ test(ecx, Immediate(kSmiTagMask));
-  __ Check(not_zero, "Unexpected type for RegExp data, FixedArray expected");
-  __ CmpObjectType(ecx, FIXED_ARRAY_TYPE, ebx);
-  __ Check(equal, "Unexpected type for RegExp data, FixedArray expected");
-#endif
+  if (FLAG_debug_code) {
+    __ test(ecx, Immediate(kSmiTagMask));
+    __ Check(not_zero, "Unexpected type for RegExp data, FixedArray expected");
+    __ CmpObjectType(ecx, FIXED_ARRAY_TYPE, ebx);
+    __ Check(equal, "Unexpected type for RegExp data, FixedArray expected");
+  }
 
   // ecx: RegExp data (FixedArray)
   // Check the type of the RegExp. Only continue if type is JSRegExp::IRREGEXP.
@@ -8478,13 +8482,12 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   // ecx: RegExp data (FixedArray)
   // edx: Number of capture registers
   // Check that the third argument is a positive smi.
+  // Check that the third argument is a positive smi less than the subject
+  // string length. A negative value will be greater (usigned comparison).
   __ mov(eax, Operand(esp, kPreviousIndexOffset));
-  __ test(eax, Immediate(kSmiTagMask | 0x80000000));
-  __ j(not_zero, &runtime);
-  // Check that it is not greater than the subject string length.
   __ SmiUntag(eax);
   __ cmp(eax, Operand(ebx));
-  __ j(greater, &runtime);
+  __ j(above, &runtime);
 
   // ecx: RegExp data (FixedArray)
   // edx: Number of capture registers
@@ -8526,17 +8529,20 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   // A flat cons string is a cons string where the second part is the empty
   // string. In that case the subject string is just the first part of the cons
   // string. Also in this case the first part of the cons string is known to be
-  // a sequential string.
+  // a sequential string or an external string.
   __ mov(edx, ebx);
   __ and_(edx, kStringRepresentationMask);
   __ cmp(edx, kConsStringTag);
   __ j(not_equal, &runtime);
   __ mov(edx, FieldOperand(eax, ConsString::kSecondOffset));
-  __ cmp(Operand(edx), Immediate(Handle<String>(Heap::empty_string())));
+  __ cmp(Operand(edx), Factory::empty_string());
   __ j(not_equal, &runtime);
   __ mov(eax, FieldOperand(eax, ConsString::kFirstOffset));
   __ mov(ebx, FieldOperand(eax, HeapObject::kMapOffset));
   __ movzx_b(ebx, FieldOperand(ebx, Map::kInstanceTypeOffset));
+  ASSERT_EQ(0, kSequentialStringTag);
+  __ test(ebx, Immediate(kStringRepresentationMask));
+  __ j(not_zero, &runtime);
   __ and_(ebx, kStringRepresentationEncodingMask);
 
   __ bind(&seq_string);
@@ -8547,10 +8553,10 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   // it has, the field contains a code object otherwise it contains the hole.
   __ cmp(ebx, kStringTag | kSeqStringTag | kTwoByteStringTag);
   __ j(equal, &seq_two_byte_string);
-#ifdef DEBUG
-  __ cmp(ebx, kStringTag | kSeqStringTag | kAsciiStringTag);
-  __ Check(equal, "Expected sequential ascii string");
-#endif
+  if (FLAG_debug_code) {
+    __ cmp(ebx, kStringTag | kSeqStringTag | kAsciiStringTag);
+    __ Check(equal, "Expected sequential ascii string");
+  }
   __ mov(edx, FieldOperand(ecx, JSRegExp::kDataAsciiCodeOffset));
   __ Set(edi, Immediate(1));  // Type is ascii.
   __ jmp(&check_code);
@@ -8562,23 +8568,24 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   __ Set(edi, Immediate(0));  // Type is two byte.
 
   __ bind(&check_code);
-  // Check that the irregexp code has been generated for If it has, the field
-  // contains a code object otherwise it contains the hole.
+  // Check that the irregexp code has been generated for the actual string
+  // encoding. If it has, the field contains a code object otherwise it contains
+  // the hole.
   __ CmpObjectType(edx, CODE_TYPE, ebx);
   __ j(not_equal, &runtime);
 
   // eax: subject string
   // edx: code
-  // edi: encoding of subject string (1 if ascii 0 if two_byte);
+  // edi: encoding of subject string (1 if ascii, 0 if two_byte);
   // Load used arguments before starting to push arguments for call to native
   // RegExp code to avoid handling changing stack height.
   __ mov(ebx, Operand(esp, kPreviousIndexOffset));
-  __ mov(ecx, Operand(esp, kJSRegExpOffset));
   __ SmiUntag(ebx);  // Previous index from smi.
 
   // eax: subject string
   // ebx: previous index
   // edx: code
+  // edi: encoding of subject string (1 if ascii 0 if two_byte);
   // All checks done. Now push arguments for native regexp code.
   __ IncrementCounter(&Counters::regexp_entry_native, 1);
 
@@ -8606,7 +8613,6 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   __ jmp(&push_rest);
 
   __ bind(&push_two_byte);
-  ASSERT(kShortSize == 2);
   __ lea(ecx, FieldOperand(eax, edi, times_2, SeqTwoByteString::kHeaderSize));
   __ push(ecx);  // Argument 4.
   __ lea(ecx, FieldOperand(eax, ebx, times_2, SeqTwoByteString::kHeaderSize));
@@ -8639,6 +8645,7 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   // Result must now be exception. If there is no pending exception already a
   // stack overflow (on the backtrack stack) was detected in RegExp code but
   // haven't created the exception yet. Handle that in the runtime system.
+  // TODO(592) Rerunning the RegExp to get the stack overflow exception.
   ExternalReference pending_exception(Top::k_pending_exception_address);
   __ mov(eax,
          Operand::StaticVariable(ExternalReference::the_hole_value_location()));
@@ -8655,6 +8662,8 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   __ mov(ecx, FieldOperand(eax, JSRegExp::kDataOffset));
   __ mov(edx, FieldOperand(ecx, JSRegExp::kIrregexpCaptureCountOffset));
   // Calculate number of capture registers (number_of_captures + 1) * 2.
+  ASSERT_EQ(0, kSmiTag);
+  ASSERT_EQ(1, kSmiTagSize + kSmiShiftSize);
   __ add(Operand(edx), Immediate(2));  // edx was a smi.
 
   // edx: Number of capture registers
@@ -8694,7 +8703,7 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   __ sub(Operand(edx), Immediate(1));
   __ j(negative, &done);
   // Read the value from the static offsets vector buffer.
-  __ mov(edi, Operand(ecx, edx, times_pointer_size, 0));
+  __ mov(edi, Operand(ecx, edx, times_int_size, 0));
   // Perform explicit shift
   ASSERT_EQ(0, kSmiTag);
   __ shl(edi, kSmiTagSize);
@@ -8720,6 +8729,7 @@ void RegExpExecStub::Generate(MacroAssembler* masm) {
   // Do the runtime call to execute the regexp.
   __ bind(&runtime);
   __ TailCallRuntime(ExternalReference(Runtime::kRegExpExec), 4, 1);
+#endif  // V8_NATIVE_REGEXP
 }
 
 

src/x64/macro-assembler-x64.cc

@@ -1553,7 +1553,7 @@ Condition MacroAssembler::IsObjectStringType(Register heap_object,
                                              Register map,
                                              Register instance_type) {
   movq(map, FieldOperand(heap_object, HeapObject::kMapOffset));
-  movzxbq(instance_type, FieldOperand(map, Map::kInstanceTypeOffset));
+  movzxbl(instance_type, FieldOperand(map, Map::kInstanceTypeOffset));
   ASSERT(kNotStringTag != 0);
   testb(instance_type, Immediate(kIsNotStringMask));
   return zero;
@@ -2473,6 +2473,51 @@ void MacroAssembler::LoadContext(Register dst, int context_chain_length) {
   }
 }
 
+int MacroAssembler::ArgumentStackSlotsForCFunctionCall(int num_arguments) {
+  // On Windows stack slots are reserved by the caller for all arguments
+  // including the ones passed in registers. On Linux 6 arguments are passed in
+  // registers and the caller does not reserve stack slots for them.
+  ASSERT(num_arguments >= 0);
+#ifdef _WIN64
+  static const int kArgumentsWithoutStackSlot = 0;
+#else
+  static const int kArgumentsWithoutStackSlot = 6;
+#endif
+  return num_arguments > kArgumentsWithoutStackSlot ?
+      num_arguments - kArgumentsWithoutStackSlot : 0;
+}
+
+void MacroAssembler::PrepareCallCFunction(int num_arguments) {
+  int frame_alignment = OS::ActivationFrameAlignment();
+  ASSERT(frame_alignment != 0);
+  ASSERT(num_arguments >= 0);
+  // Make stack end at alignment and allocate space for arguments and old rsp.
+  movq(kScratchRegister, rsp);
+  ASSERT(IsPowerOf2(frame_alignment));
+  int argument_slots_on_stack =
+      ArgumentStackSlotsForCFunctionCall(num_arguments);
+  subq(rsp, Immediate((argument_slots_on_stack + 1) * kPointerSize));
+  and_(rsp, Immediate(-frame_alignment));
+  movq(Operand(rsp, argument_slots_on_stack * kPointerSize), kScratchRegister);
+}
+
+
+void MacroAssembler::CallCFunction(ExternalReference function,
+                                   int num_arguments) {
+  movq(rax, function);
+  CallCFunction(rax, num_arguments);
+}
+
+
+void MacroAssembler::CallCFunction(Register function, int num_arguments) {
+  call(function);
+  ASSERT(OS::ActivationFrameAlignment() != 0);
+  ASSERT(num_arguments >= 0);
+  int argument_slots_on_stack =
+      ArgumentStackSlotsForCFunctionCall(num_arguments);
+  movq(rsp, Operand(rsp, argument_slots_on_stack * kPointerSize));
+}
+
 
 CodePatcher::CodePatcher(byte* address, int size)
     : address_(address), size_(size), masm_(address, size + Assembler::kGap) {

src/x64/macro-assembler-x64.h

@@ -162,7 +162,8 @@ class MacroAssembler: public Assembler {
   // Conversions between tagged smi values and non-tagged integer values.
 
   // Tag an integer value. The result must be known to be a valid smi value.
-  // Only uses the low 32 bits of the src register.
+  // Only uses the low 32 bits of the src register. Sets the N and Z flags
+  // based on the value of the resulting integer.
   void Integer32ToSmi(Register dst, Register src);
 
   // Tag an integer value if possible, or jump the integer value cannot be
@@ -644,6 +645,26 @@ class MacroAssembler: public Assembler {
   // Jump to a runtime routine.
   void JumpToRuntime(const ExternalReference& ext, int result_size);
 
+  // Before calling a C-function from generated code, align arguments on stack.
+  // After aligning the frame, arguments must be stored in esp[0], esp[4],
+  // etc., not pushed. The argument count assumes all arguments are word sized.
+  // The number of slots reserved for arguments depends on platform. On Windows
+  // stack slots are reserved for the arguments passed in registers. On other
+  // platforms stack slots are only reserved for the arguments actually passed
+  // on the stack.
+  void PrepareCallCFunction(int num_arguments);
+
+  // Calls a C function and cleans up the space for arguments allocated
+  // by PrepareCallCFunction. The called function is not allowed to trigger a
+  // garbage collection, since that might move the code and invalidate the
+  // return address (unless this is somehow accounted for by the called
+  // function).
+  void CallCFunction(ExternalReference function, int num_arguments);
+  void CallCFunction(Register function, int num_arguments);
+
+  // Calculate the number of stack slots to reserve for arguments when calling a
+  // C function.
+  int ArgumentStackSlotsForCFunctionCall(int num_arguments);
 
   // ---------------------------------------------------------------------------
   // Utilities

src/x64/regexp-macro-assembler-x64.cc

@@ -329,14 +329,14 @@ void RegExpMacroAssemblerX64::CheckNotBackReferenceIgnoreCase(
     ASSERT(mode_ == UC16);
     // Save important/volatile registers before calling C function.
 #ifndef _WIN64
-    // Callee save on Win64
+    // Caller save on Linux and callee save in Windows.
     __ push(rsi);
     __ push(rdi);
 #endif
     __ push(backtrack_stackpointer());
 
     int num_arguments = 3;
-    FrameAlign(num_arguments);
+    __ PrepareCallCFunction(num_arguments);
 
     // Put arguments into parameter registers. Parameters are
     //   Address byte_offset1 - Address captured substring's start.
@@ -361,7 +361,7 @@ void RegExpMacroAssemblerX64::CheckNotBackReferenceIgnoreCase(
 #endif
     ExternalReference compare =
         ExternalReference::re_case_insensitive_compare_uc16();
-    CallCFunction(compare, num_arguments);
+    __ CallCFunction(compare, num_arguments);
 
     // Restore original values before reacting on result value.
     __ Move(code_object_pointer(), masm_->CodeObject());
@@ -634,7 +634,6 @@ void RegExpMacroAssemblerX64::Fail() {
 Handle<Object> RegExpMacroAssemblerX64::GetCode(Handle<String> source) {
   // Finalize code - write the entry point code now we know how many
   // registers we need.
-
   // Entry code:
   __ bind(&entry_label_);
   // Start new stack frame.
@@ -671,6 +670,7 @@ Handle<Object> RegExpMacroAssemblerX64::GetCode(Handle<String> source) {
 
   __ push(rbx);  // Callee-save
 #endif
+
   __ push(Immediate(0));  // Make room for "input start - 1" constant.
   __ push(Immediate(0));  // Make room for "at start" constant.
 
@@ -850,7 +850,7 @@ Handle<Object> RegExpMacroAssemblerX64::GetCode(Handle<String> source) {
 
     // Call GrowStack(backtrack_stackpointer())
     int num_arguments = 2;
-    FrameAlign(num_arguments);
+    __ PrepareCallCFunction(num_arguments);
 #ifdef _WIN64
     // Microsoft passes parameters in rcx, rdx.
     // First argument, backtrack stackpointer, is already in rcx.
@@ -861,7 +861,7 @@ Handle<Object> RegExpMacroAssemblerX64::GetCode(Handle<String> source) {
     __ lea(rsi, Operand(rbp, kStackHighEnd));  // Second argument.
 #endif
     ExternalReference grow_stack = ExternalReference::re_grow_stack();
-    CallCFunction(grow_stack, num_arguments);
+    __ CallCFunction(grow_stack, num_arguments);
     // If return NULL, we have failed to grow the stack, and
     // must exit with a stack-overflow exception.
     __ testq(rax, rax);
@@ -1030,7 +1030,7 @@ void RegExpMacroAssemblerX64::CallCheckStackGuardState() {
   // This function call preserves no register values. Caller should
   // store anything volatile in a C call or overwritten by this function.
   int num_arguments = 3;
-  FrameAlign(num_arguments);
+  __ PrepareCallCFunction(num_arguments);
 #ifdef _WIN64
   // Second argument: Code* of self. (Do this before overwriting r8).
   __ movq(rdx, code_object_pointer());
@@ -1050,7 +1050,7 @@ void RegExpMacroAssemblerX64::CallCheckStackGuardState() {
 #endif
   ExternalReference stack_check =
       ExternalReference::re_check_stack_guard_state();
-  CallCFunction(stack_check, num_arguments);
+  __ CallCFunction(stack_check, num_arguments);
 }
 
 
@@ -1072,6 +1072,12 @@ int RegExpMacroAssemblerX64::CheckStackGuardState(Address* return_address,
   // If not real stack overflow the stack guard was used to interrupt
   // execution for another purpose.
 
+  // If this is a direct call from JavaScript retry the RegExp forcing the call
+  // through the runtime system. Currently the direct call cannot handle a GC.
+  if (frame_entry<int>(re_frame, kDirectCall) == 1) {
+    return RETRY;
+  }
+
   // Prepare for possible GC.
   HandleScope handles;
   Handle<Code> code_handle(re_code);
@@ -1266,45 +1272,6 @@ void RegExpMacroAssemblerX64::CheckStackLimit() {
 }
 
 
-void RegExpMacroAssemblerX64::FrameAlign(int num_arguments) {
-  // TODO(lrn): Since we no longer use the system stack arbitrarily (but we do
-  // use it, e.g., for SafeCall), we know the number of elements on the stack
-  // since the last frame alignment. We might be able to do this simpler then.
-  int frameAlignment = OS::ActivationFrameAlignment();
-  ASSERT(frameAlignment != 0);
-  // Make stack end at alignment and make room for num_arguments pointers
-  // (on Win64 only) and the original value of rsp.
-  __ movq(kScratchRegister, rsp);
-  ASSERT(IsPowerOf2(frameAlignment));
-#ifdef _WIN64
-  // Allocate space for parameters and old rsp.
-  __ subq(rsp, Immediate((num_arguments + 1) * kPointerSize));
-  __ and_(rsp, Immediate(-frameAlignment));
-  __ movq(Operand(rsp, num_arguments * kPointerSize), kScratchRegister);
-#else
-  // Allocate space for old rsp.
-  __ subq(rsp, Immediate(kPointerSize));
-  __ and_(rsp, Immediate(-frameAlignment));
-  __ movq(Operand(rsp, 0), kScratchRegister);
-#endif
-}
-
-
-void RegExpMacroAssemblerX64::CallCFunction(ExternalReference function,
-                                            int num_arguments) {
-  __ movq(rax, function);
-  __ call(rax);
-  ASSERT(OS::ActivationFrameAlignment() != 0);
-#ifdef _WIN64
-  __ movq(rsp, Operand(rsp, num_arguments * kPointerSize));
-#else
-  // All arguments passed in registers.
-  ASSERT(num_arguments <= 6);
-  __ pop(rsp);
-#endif
-}
-
-
 void RegExpMacroAssemblerX64::LoadCurrentCharacterUnchecked(int cp_offset,
                                                             int characters) {
   if (mode_ == ASCII) {

src/x64/regexp-macro-assembler-x64.h

@@ -247,21 +247,6 @@ class RegExpMacroAssemblerX64: public NativeRegExpMacroAssembler {
   // Increments the stack pointer (rcx) by a word size.
   inline void Drop();
 
-  // Before calling a C-function from generated code, align arguments on stack.
-  // After aligning the frame, arguments must be stored in esp[0], esp[4],
-  // etc., not pushed. The argument count assumes all arguments are word sized.
-  // Some compilers/platforms require the stack to be aligned when calling
-  // C++ code.
-  // Needs a scratch register to do some arithmetic. This register will be
-  // trashed.
-  inline void FrameAlign(int num_arguments);
-
-  // Calls a C function and cleans up the space for arguments allocated
-  // by FrameAlign. The called function is not allowed to trigger a garbage
-  // collection, since that might move the code and invalidate the return
-  // address (unless this is somehow accounted for by the called function).
-  inline void CallCFunction(ExternalReference function, int num_arguments);
-
   MacroAssembler* masm_;
 
   ZoneList<int> code_relative_fixup_positions_;