Perform string add in generated code on X64 platform

This is a port of the IA-32 version from r3400 (http://code.google.com/p/v8/source/detail?r=3400). In the X64 version the additional registers are used to avoid loading the instance type and arguments several times.
Review URL: http://codereview.chromium.org/460109

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@3434 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/ia32/codegen-ia32.cc

@@ -8306,6 +8306,7 @@ void StringAddStub::Generate(MacroAssembler* masm) {
   __ mov(edi, FieldOperand(edx, HeapObject::kMapOffset));
   __ movzx_b(edi, FieldOperand(edi, Map::kInstanceTypeOffset));
   __ and_(ecx, Operand(edi));
+  ASSERT(kStringEncodingMask == kAsciiStringTag);
   __ test(ecx, Immediate(kAsciiStringTag));
   __ j(zero, &non_ascii);
   // Allocate an acsii cons string.
@@ -8348,7 +8349,7 @@ void StringAddStub::Generate(MacroAssembler* masm) {
   Label non_ascii_string_add_flat_result;
   __ mov(ecx, FieldOperand(eax, HeapObject::kMapOffset));
   __ movzx_b(ecx, FieldOperand(ecx, Map::kInstanceTypeOffset));
-  ASSERT(kAsciiStringTag != 0);
+  ASSERT(kStringEncodingMask == kAsciiStringTag);
   __ test(ecx, Immediate(kAsciiStringTag));
   __ j(zero, &non_ascii_string_add_flat_result);
   __ mov(ecx, FieldOperand(edx, HeapObject::kMapOffset));

src/ia32/codegen-ia32.h

@@ -760,11 +760,11 @@ class StringAddStub: public CodeStub {
   void Generate(MacroAssembler* masm);
 
   void GenerateCopyCharacters(MacroAssembler* masm,
-                                   Register desc,
-                                   Register src,
-                                   Register count,
-                                   Register scratch,
-                                   bool ascii);
+                              Register desc,
+                              Register src,
+                              Register count,
+                              Register scratch,
+                              bool ascii);
 
   // Should the stub check whether arguments are strings?
   bool string_check_;

src/ia32/macro-assembler-ia32.cc

@@ -834,10 +834,9 @@ void MacroAssembler::AllocateTwoByteString(Register result,
   // Calculate the number of bytes needed for the characters in the string while
   // observing object alignment.
   ASSERT((SeqTwoByteString::kHeaderSize & kObjectAlignmentMask) == 0);
-  mov(scratch1, length);
   ASSERT(kShortSize == 2);
-  shl(scratch1, 1);
-  add(Operand(scratch1), Immediate(kObjectAlignmentMask));
+  // scratch1 = length * 2 + kObjectAlignmentMask.
+  lea(scratch1, Operand(length, length, times_1, kObjectAlignmentMask));
   and_(Operand(scratch1), Immediate(~kObjectAlignmentMask));
 
   // Allocate two byte string in new space.

src/x64/codegen-x64.cc

@@ -4051,7 +4051,8 @@ void CodeGenerator::GenerateStringAdd(ZoneList<Expression*>* args) {
   Load(args->at(0));
   Load(args->at(1));
 
-  Result answer = frame_->CallRuntime(Runtime::kStringAdd, 2);
+  StringAddStub stub(NO_STRING_ADD_FLAGS);
+  Result answer = frame_->CallStub(&stub, 2);
   frame_->Push(&answer);
 }
 
@@ -7796,8 +7797,8 @@ void GenericBinaryOpStub::Generate(MacroAssembler* masm) {
       __ j(above_equal, &string1);
 
       // First and second argument are strings.
-      Runtime::Function* f = Runtime::FunctionForId(Runtime::kStringAdd);
-      __ TailCallRuntime(ExternalReference(f), 2, f->result_size);
+      StringAddStub stub(NO_STRING_CHECK_IN_STUB);
+      __ TailCallStub(&stub);
 
       // Only first argument is a string.
       __ bind(&string1);
@@ -7880,6 +7881,234 @@ int CompareStub::MinorKey() {
   return (static_cast<unsigned>(cc_) << 1) | (strict_ ? 1 : 0);
 }
 
+
+void StringAddStub::Generate(MacroAssembler* masm) {
+  Label string_add_runtime;
+
+  // Load the two arguments.
+  __ movq(rax, Operand(rsp, 2 * kPointerSize));  // First argument.
+  __ movq(rdx, Operand(rsp, 1 * kPointerSize));  // Second argument.
+
+  // Make sure that both arguments are strings if not known in advance.
+  if (string_check_) {
+    Condition is_smi;
+    is_smi = masm->CheckSmi(rax);
+    __ j(is_smi, &string_add_runtime);
+    __ CmpObjectType(rax, FIRST_NONSTRING_TYPE, r8);
+    __ j(above_equal, &string_add_runtime);
+
+    // First argument is a a string, test second.
+    is_smi = masm->CheckSmi(rdx);
+    __ j(is_smi, &string_add_runtime);
+    __ CmpObjectType(rdx, FIRST_NONSTRING_TYPE, r9);
+    __ j(above_equal, &string_add_runtime);
+  }
+
+  // Both arguments are strings.
+  // rax: first string
+  // rdx: second string
+  // Check if either of the strings are empty. In that case return the other.
+  Label second_not_zero_length, both_not_zero_length;
+  __ movl(rcx, FieldOperand(rdx, String::kLengthOffset));
+  __ testl(rcx, rcx);
+  __ j(not_zero, &second_not_zero_length);
+  // Second string is empty, result is first string which is already in rax.
+  __ IncrementCounter(&Counters::string_add_native, 1);
+  __ ret(2 * kPointerSize);
+  __ bind(&second_not_zero_length);
+  __ movl(rbx, FieldOperand(rax, String::kLengthOffset));
+  __ testl(rbx, rbx);
+  __ j(not_zero, &both_not_zero_length);
+  // First string is empty, result is second string which is in rdx.
+  __ movq(rax, rdx);
+  __ IncrementCounter(&Counters::string_add_native, 1);
+  __ ret(2 * kPointerSize);
+
+  // Both strings are non-empty.
+  // rax: first string
+  // rbx: length of first string
+  // ecx: length of second string
+  // edx: second string
+  // r8: instance type of first string if string check was performed above
+  // r9: instance type of first string if string check was performed above
+  Label string_add_flat_result;
+  __ bind(&both_not_zero_length);
+  // Look at the length of the result of adding the two strings.
+  __ addl(rbx, rcx);
+  // Use the runtime system when adding two one character strings, as it
+  // contains optimizations for this specific case using the symbol table.
+  __ cmpl(rbx, Immediate(2));
+  __ j(equal, &string_add_runtime);
+  // If arguments where known to be strings, maps are not loaded to r8 and r9
+  // by the code above.
+  if (!string_check_) {
+    __ movq(r8, FieldOperand(rax, HeapObject::kMapOffset));
+    __ movq(r9, FieldOperand(rdx, HeapObject::kMapOffset));
+  }
+  // Get the instance types of the two strings as they will be needed soon.
+  __ movzxbl(r8, FieldOperand(r8, Map::kInstanceTypeOffset));
+  __ movzxbl(r9, FieldOperand(r9, Map::kInstanceTypeOffset));
+  // Check if resulting string will be flat.
+  __ cmpl(rbx, Immediate(String::kMinNonFlatLength));
+  __ j(below, &string_add_flat_result);
+  // Handle exceptionally long strings in the runtime system.
+  ASSERT((String::kMaxLength & 0x80000000) == 0);
+  __ cmpl(rbx, Immediate(String::kMaxLength));
+  __ j(above, &string_add_runtime);
+
+  // If result is not supposed to be flat, allocate a cons string object. If
+  // both strings are ascii the result is an ascii cons string.
+  // rax: first string
+  // ebx: length of resulting flat string
+  // rdx: second string
+  // r8: instance type of first string
+  // r9: instance type of second string
+  Label non_ascii, allocated;
+  __ movl(rcx, r8);
+  __ and_(rcx, r9);
+  ASSERT(kStringEncodingMask == kAsciiStringTag);
+  __ testl(rcx, Immediate(kAsciiStringTag));
+  __ j(zero, &non_ascii);
+  // Allocate an acsii cons string.
+  __ AllocateAsciiConsString(rcx, rdi, no_reg, &string_add_runtime);
+  __ bind(&allocated);
+  // Fill the fields of the cons string.
+  __ movl(FieldOperand(rcx, ConsString::kLengthOffset), rbx);
+  __ movl(FieldOperand(rcx, ConsString::kHashFieldOffset),
+          Immediate(String::kEmptyHashField));
+  __ movq(FieldOperand(rcx, ConsString::kFirstOffset), rax);
+  __ movq(FieldOperand(rcx, ConsString::kSecondOffset), rdx);
+  __ movq(rax, rcx);
+  __ IncrementCounter(&Counters::string_add_native, 1);
+  __ ret(2 * kPointerSize);
+  __ bind(&non_ascii);
+  // Allocate a two byte cons string.
+  __ AllocateConsString(rcx, rdi, no_reg, &string_add_runtime);
+  __ jmp(&allocated);
+
+  // Handle creating a flat result. First check that both strings are not
+  // external strings.
+  // rax: first string
+  // ebx: length of resulting flat string
+  // rdx: second string
+  // r8: instance type of first string
+  // r9: instance type of first string
+  __ bind(&string_add_flat_result);
+  __ movl(rcx, r8);
+  __ and_(rcx, Immediate(kStringRepresentationMask));
+  __ cmpl(rcx, Immediate(kExternalStringTag));
+  __ j(equal, &string_add_runtime);
+  __ movl(rcx, r9);
+  __ and_(rcx, Immediate(kStringRepresentationMask));
+  __ cmpl(rcx, Immediate(kExternalStringTag));
+  __ j(equal, &string_add_runtime);
+  // Now check if both strings are ascii strings.
+  // rax: first string
+  // ebx: length of resulting flat string
+  // rdx: second string
+  // r8: instance type of first string
+  // r9: instance type of second string
+  Label non_ascii_string_add_flat_result;
+  ASSERT(kStringEncodingMask == kAsciiStringTag);
+  __ testl(r8, Immediate(kAsciiStringTag));
+  __ j(zero, &non_ascii_string_add_flat_result);
+  __ testl(r9, Immediate(kAsciiStringTag));
+  __ j(zero, &string_add_runtime);
+  // Both strings are ascii strings. As they are short they are both flat.
+  __ AllocateAsciiString(rcx, rbx, rdi, r14, r15, &string_add_runtime);
+  // rcx: result string
+  __ movq(rbx, rcx);
+  // Locate first character of result.
+  __ addq(rcx, Immediate(SeqAsciiString::kHeaderSize - kHeapObjectTag));
+  // Locate first character of first argument
+  __ movl(rdi, FieldOperand(rax, String::kLengthOffset));
+  __ addq(rax, Immediate(SeqAsciiString::kHeaderSize - kHeapObjectTag));
+  // rax: first char of first argument
+  // rbx: result string
+  // rcx: first character of result
+  // rdx: second string
+  // rdi: length of first argument
+  GenerateCopyCharacters(masm, rcx, rax, rdi, true);
+  // Locate first character of second argument.
+  __ movl(rdi, FieldOperand(rdx, String::kLengthOffset));
+  __ addq(rdx, Immediate(SeqAsciiString::kHeaderSize - kHeapObjectTag));
+  // rbx: result string
+  // rcx: next character of result
+  // rdx: first char of second argument
+  // rdi: length of second argument
+  GenerateCopyCharacters(masm, rcx, rdx, rdi, true);
+  __ movq(rax, rbx);
+  __ IncrementCounter(&Counters::string_add_native, 1);
+  __ ret(2 * kPointerSize);
+
+  // Handle creating a flat two byte result.
+  // rax: first string - known to be two byte
+  // rbx: length of resulting flat string
+  // rdx: second string
+  // r8: instance type of first string
+  // r9: instance type of first string
+  __ bind(&non_ascii_string_add_flat_result);
+  __ and_(r9, Immediate(kAsciiStringTag));
+  __ j(not_zero, &string_add_runtime);
+  // Both strings are two byte strings. As they are short they are both
+  // flat.
+  __ AllocateTwoByteString(rcx, rbx, rdi, r14, r15, &string_add_runtime);
+  // rcx: result string
+  __ movq(rbx, rcx);
+  // Locate first character of result.
+  __ addq(rcx, Immediate(SeqTwoByteString::kHeaderSize - kHeapObjectTag));
+  // Locate first character of first argument.
+  __ movl(rdi, FieldOperand(rax, String::kLengthOffset));
+  __ addq(rax, Immediate(SeqTwoByteString::kHeaderSize - kHeapObjectTag));
+  // rax: first char of first argument
+  // rbx: result string
+  // rcx: first character of result
+  // rdx: second argument
+  // rdi: length of first argument
+  GenerateCopyCharacters(masm, rcx, rax, rdi, false);
+  // Locate first character of second argument.
+  __ movl(rdi, FieldOperand(rdx, String::kLengthOffset));
+  __ addq(rdx, Immediate(SeqTwoByteString::kHeaderSize - kHeapObjectTag));
+  // rbx: result string
+  // rcx: next character of result
+  // rdx: first char of second argument
+  // rdi: length of second argument
+  GenerateCopyCharacters(masm, rcx, rdx, rdi, false);
+  __ movq(rax, rbx);
+  __ IncrementCounter(&Counters::string_add_native, 1);
+  __ ret(2 * kPointerSize);
+
+  // Just jump to runtime to add the two strings.
+  __ bind(&string_add_runtime);
+  __ TailCallRuntime(ExternalReference(Runtime::kStringAdd), 2, 1);
+}
+
+
+void StringAddStub::GenerateCopyCharacters(MacroAssembler* masm,
+                                           Register dest,
+                                           Register src,
+                                           Register count,
+                                           bool ascii) {
+  Label loop;
+  __ bind(&loop);
+  // This loop just copies one character at a time, as it is only used for very
+  // short strings.
+  if (ascii) {
+    __ movb(kScratchRegister, Operand(src, 0));
+    __ movb(Operand(dest, 0), kScratchRegister);
+    __ addq(src, Immediate(1));
+    __ addq(dest, Immediate(1));
+  } else {
+    __ movzxwl(kScratchRegister, Operand(src, 0));
+    __ movw(Operand(dest, 0), kScratchRegister);
+    __ addq(src, Immediate(2));
+    __ addq(dest, Immediate(2));
+  }
+  __ subl(count, Immediate(1));
+  __ j(not_zero, &loop);
+}
+
+
 #undef __
 
 #define __ masm.

src/x64/codegen-x64.h

@@ -745,6 +745,36 @@ class GenericBinaryOpStub: public CodeStub {
 };
 
 
+// Flag that indicates how to generate code for the stub StringAddStub.
+enum StringAddFlags {
+  NO_STRING_ADD_FLAGS = 0,
+  NO_STRING_CHECK_IN_STUB = 1 << 0  // Omit string check in stub.
+};
+
+
+class StringAddStub: public CodeStub {
+ public:
+  explicit StringAddStub(StringAddFlags flags) {
+    string_check_ = ((flags & NO_STRING_CHECK_IN_STUB) == 0);
+  }
+
+ private:
+  Major MajorKey() { return StringAdd; }
+  int MinorKey() { return string_check_ ? 0 : 1; }
+
+  void Generate(MacroAssembler* masm);
+
+  void GenerateCopyCharacters(MacroAssembler* masm,
+                              Register desc,
+                              Register src,
+                              Register count,
+                              bool ascii);
+
+  // Should the stub check whether arguments are strings?
+  bool string_check_;
+};
+
+
 } }  // namespace v8::internal
 
 #endif  // V8_X64_CODEGEN_X64_H_

src/x64/macro-assembler-x64.cc

@@ -310,6 +310,12 @@ void MacroAssembler::CallStub(CodeStub* stub) {
 }
 
 
+void MacroAssembler::TailCallStub(CodeStub* stub) {
+  ASSERT(allow_stub_calls());  // calls are not allowed in some stubs
+  Jump(stub->GetCode(), RelocInfo::CODE_TARGET);
+}
+
+
 void MacroAssembler::StubReturn(int argc) {
   ASSERT(argc >= 1 && generating_stub());
   ret((argc - 1) * kPointerSize);
@@ -2244,6 +2250,108 @@ void MacroAssembler::AllocateHeapNumber(Register result,
 }
 
 
+void MacroAssembler::AllocateTwoByteString(Register result,
+                                           Register length,
+                                           Register scratch1,
+                                           Register scratch2,
+                                           Register scratch3,
+                                           Label* gc_required) {
+  // Calculate the number of bytes needed for the characters in the string while
+  // observing object alignment.
+  ASSERT((SeqTwoByteString::kHeaderSize & kObjectAlignmentMask) == 0);
+  ASSERT(kShortSize == 2);
+  // scratch1 = length * 2 + kObjectAlignmentMask.
+  lea(scratch1, Operand(length, length, times_1, kObjectAlignmentMask));
+  and_(scratch1, Immediate(~kObjectAlignmentMask));
+
+  // Allocate two byte string in new space.
+  AllocateInNewSpace(SeqTwoByteString::kHeaderSize,
+                     times_1,
+                     scratch1,
+                     result,
+                     scratch2,
+                     scratch3,
+                     gc_required,
+                     TAG_OBJECT);
+
+  // Set the map, length and hash field.
+  LoadRoot(kScratchRegister, Heap::kStringMapRootIndex);
+  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
+  movl(FieldOperand(result, String::kLengthOffset), length);
+  movl(FieldOperand(result, String::kHashFieldOffset),
+       Immediate(String::kEmptyHashField));
+}
+
+
+void MacroAssembler::AllocateAsciiString(Register result,
+                                         Register length,
+                                         Register scratch1,
+                                         Register scratch2,
+                                         Register scratch3,
+                                         Label* gc_required) {
+  // Calculate the number of bytes needed for the characters in the string while
+  // observing object alignment.
+  ASSERT((SeqAsciiString::kHeaderSize & kObjectAlignmentMask) == 0);
+  movl(scratch1, length);
+  ASSERT(kCharSize == 1);
+  addq(scratch1, Immediate(kObjectAlignmentMask));
+  and_(scratch1, Immediate(~kObjectAlignmentMask));
+
+  // Allocate ascii string in new space.
+  AllocateInNewSpace(SeqAsciiString::kHeaderSize,
+                     times_1,
+                     scratch1,
+                     result,
+                     scratch2,
+                     scratch3,
+                     gc_required,
+                     TAG_OBJECT);
+
+  // Set the map, length and hash field.
+  LoadRoot(kScratchRegister, Heap::kAsciiStringMapRootIndex);
+  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
+  movl(FieldOperand(result, String::kLengthOffset), length);
+  movl(FieldOperand(result, String::kHashFieldOffset),
+       Immediate(String::kEmptyHashField));
+}
+
+
+void MacroAssembler::AllocateConsString(Register result,
+                                        Register scratch1,
+                                        Register scratch2,
+                                        Label* gc_required) {
+  // Allocate heap number in new space.
+  AllocateInNewSpace(ConsString::kSize,
+                     result,
+                     scratch1,
+                     scratch2,
+                     gc_required,
+                     TAG_OBJECT);
+
+  // Set the map. The other fields are left uninitialized.
+  LoadRoot(kScratchRegister, Heap::kConsStringMapRootIndex);
+  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
+}
+
+
+void MacroAssembler::AllocateAsciiConsString(Register result,
+                                             Register scratch1,
+                                             Register scratch2,
+                                             Label* gc_required) {
+  // Allocate heap number in new space.
+  AllocateInNewSpace(ConsString::kSize,
+                     result,
+                     scratch1,
+                     scratch2,
+                     gc_required,
+                     TAG_OBJECT);
+
+  // Set the map. The other fields are left uninitialized.
+  LoadRoot(kScratchRegister, Heap::kConsAsciiStringMapRootIndex);
+  movq(FieldOperand(result, HeapObject::kMapOffset), kScratchRegister);
+}
+
+
 void MacroAssembler::LoadContext(Register dst, int context_chain_length) {
   if (context_chain_length > 0) {
     // Move up the chain of contexts to the context containing the slot.

src/x64/macro-assembler-x64.h

@@ -518,6 +518,32 @@ class MacroAssembler: public Assembler {
                           Register scratch,
                           Label* gc_required);
 
+  // Allocate a sequential string. All the header fields of the string object
+  // are initialized.
+  void AllocateTwoByteString(Register result,
+                             Register length,
+                             Register scratch1,
+                             Register scratch2,
+                             Register scratch3,
+                             Label* gc_required);
+  void AllocateAsciiString(Register result,
+                           Register length,
+                           Register scratch1,
+                           Register scratch2,
+                           Register scratch3,
+                           Label* gc_required);
+
+  // Allocate a raw cons string object. Only the map field of the result is
+  // initialized.
+  void AllocateConsString(Register result,
+                          Register scratch1,
+                          Register scratch2,
+                          Label* gc_required);
+  void AllocateAsciiConsString(Register result,
+                               Register scratch1,
+                               Register scratch2,
+                               Label* gc_required);
+
   // ---------------------------------------------------------------------------
   // Support functions.
 
@@ -557,6 +583,9 @@ class MacroAssembler: public Assembler {
   // Call a code stub.
   void CallStub(CodeStub* stub);
 
+  // Tail call a code stub (jump).
+  void TailCallStub(CodeStub* stub);
+
   // Return from a code stub after popping its arguments.
   void StubReturn(int argc);