X64: Added inline keyed load/store and a bunch of other missing functions.

Review URL: http://codereview.chromium.org/160272


git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@2585 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/ia32/ic-ia32.cc

@@ -839,7 +839,8 @@ void KeyedStoreIC::RestoreInlinedVersion(Address address) {
 
 bool LoadIC::PatchInlinedLoad(Address address, Object* map, int offset) {
   // The address of the instruction following the call.
-  Address test_instruction_address = address + 4;
+  Address test_instruction_address =
+      address + Assembler::kTargetAddrToReturnAddrDist;
   // If the instruction following the call is not a test eax, nothing
   // was inlined.
   if (*test_instruction_address != kTestEaxByte) return false;
@@ -865,7 +866,8 @@ bool LoadIC::PatchInlinedLoad(Address address, Object* map, int offset) {
 
 
 static bool PatchInlinedMapCheck(Address address, Object* map) {
-  Address test_instruction_address = address + 4;  // 4 = stub address
+  Address test_instruction_address =
+      address + Assembler::kTargetAddrToReturnAddrDist;
   // The keyed load has a fast inlined case if the IC call instruction
   // is immediately followed by a test instruction.
   if (*test_instruction_address != kTestEaxByte) return false;

src/ic.h

@@ -389,6 +389,10 @@ class KeyedStoreIC: public IC {
 
   // Support for patching the map that is checked in an inlined
   // version of keyed store.
+  // The address is the patch point for the IC call
+  // (Assembler::kTargetAddrToReturnAddrDist before the end of
+  // the call/return address).
+  // The map is the new map that the inlined code should check against.
   static bool PatchInlinedStore(Address address, Object* map);
 
   friend class IC;

src/x64/assembler-x64.cc

@@ -1140,6 +1140,9 @@ void Assembler::movq(const Operand& dst, Register src) {
 
 
 void Assembler::movq(Register dst, void* value, RelocInfo::Mode rmode) {
+  // This method must not be used with heap object references. The stored
+  // address is not GC safe. Use the handle version instead.
+  ASSERT(rmode > RelocInfo::LAST_GCED_ENUM);
   EnsureSpace ensure_space(this);
   last_pc_ = pc_;
   emit_rex_64(dst);

src/x64/assembler-x64.h

@@ -292,6 +292,7 @@ enum ScaleFactor {
   times_4 = 2,
   times_8 = 3,
   times_int_size = times_4,
+  times_half_pointer_size = times_4,
   times_pointer_size = times_8
 };
 

src/x64/ic-x64.cc

@@ -159,16 +159,64 @@ static void GenerateCheckNonObjectOrLoaded(MacroAssembler* masm, Label* miss,
 }
 
 
+// One byte opcode for test eax,0xXXXXXXXX.
+static const byte kTestEaxByte = 0xA9;
+
+
+static bool PatchInlinedMapCheck(Address address, Object* map) {
+  // Arguments are address of start of call sequence that called
+  // the IC,
+  Address test_instruction_address =
+      address + Assembler::kTargetAddrToReturnAddrDist;
+  // The keyed load has a fast inlined case if the IC call instruction
+  // is immediately followed by a test instruction.
+  if (*test_instruction_address != kTestEaxByte) return false;
+
+  // Fetch the offset from the test instruction to the map compare
+  // instructions (starting with the 64-bit immediate mov of the map
+  // address). This offset is stored in the last 4 bytes of the 5
+  // byte test instruction.
+  Address delta_address = test_instruction_address + 1;
+  int delta = *reinterpret_cast<int*>(delta_address);
+  // Compute the map address.  The map address is in the last 8 bytes
+  // of the 10-byte immediate mov instruction (incl. REX prefix), so we add 2
+  // to the offset to get the map address.
+  Address map_address = test_instruction_address + delta + 2;
+  // Patch the map check.
+  *(reinterpret_cast<Object**>(map_address)) = map;
+  return true;
+}
+
+
+bool KeyedLoadIC::PatchInlinedLoad(Address address, Object* map) {
+  return PatchInlinedMapCheck(address, map);
+}
+
+
+bool KeyedStoreIC::PatchInlinedStore(Address address, Object* map) {
+  return PatchInlinedMapCheck(address, map);
+}
+
+
 void KeyedLoadIC::ClearInlinedVersion(Address address) {
-  // TODO(X64): Implement this when LoadIC is enabled.
+  // Insert null as the map to check for to make sure the map check fails
+  // sending control flow to the IC instead of the inlined version.
+  PatchInlinedLoad(address, Heap::null_value());
 }
 
+
 void KeyedStoreIC::ClearInlinedVersion(Address address) {
-  // TODO(X64): Implement this when LoadIC is enabled.
+  // Insert null as the elements map to check for.  This will make
+  // sure that the elements fast-case map check fails so that control
+  // flows to the IC instead of the inlined version.
+  PatchInlinedStore(address, Heap::null_value());
 }
 
+
 void KeyedStoreIC::RestoreInlinedVersion(Address address) {
-  UNIMPLEMENTED();
+  // Restore the fast-case elements map check so that the inlined
+  // version can be used again.
+  PatchInlinedStore(address, Heap::fixed_array_map());
 }
 
 
@@ -310,18 +358,6 @@ void KeyedLoadIC::GenerateMiss(MacroAssembler* masm) {
 }
 
 
-bool KeyedLoadIC::PatchInlinedLoad(Address address, Object* map) {
-  // Never patch the map in the map check, so the check always fails.
-  return false;
-}
-
-
-bool KeyedStoreIC::PatchInlinedStore(Address address, Object* map) {
-  // Never patch the map in the map check, so the check always fails.
-  return false;
-}
-
-
 void KeyedStoreIC::Generate(MacroAssembler* masm, ExternalReference const& f) {
   // ----------- S t a t e -------------
   //  -- rax     : value
@@ -539,7 +575,10 @@ const int LoadIC::kOffsetToLoadInstruction = 20;
 
 
 void LoadIC::ClearInlinedVersion(Address address) {
-  // TODO(X64): Implement this when LoadIC is enabled.
+  // Reset the map check of the inlined inobject property load (if
+  // present) to guarantee failure by holding an invalid map (the null
+  // value).  The offset can be patched to anything.
+  PatchInlinedLoad(address, Heap::null_value(), kMaxInt);
 }
 
 
@@ -605,13 +644,37 @@ void LoadIC::GenerateNormal(MacroAssembler* masm) {
   Generate(masm, ExternalReference(IC_Utility(kLoadIC_Miss)));
 }
 
+
 void LoadIC::GenerateStringLength(MacroAssembler* masm) {
   Generate(masm, ExternalReference(IC_Utility(kLoadIC_Miss)));
 }
 
-bool LoadIC::PatchInlinedLoad(Address address, Object* map, int index) {
-  // TODO(X64): Implement this function.  Until then, the code is not patched.
-  return false;
+
+bool LoadIC::PatchInlinedLoad(Address address, Object* map, int offset) {
+  // The address of the instruction following the call.
+  Address test_instruction_address =
+      address + Assembler::kTargetAddrToReturnAddrDist;
+  // If the instruction following the call is not a test eax, nothing
+  // was inlined.
+  if (*test_instruction_address != kTestEaxByte) return false;
+
+  Address delta_address = test_instruction_address + 1;
+  // The delta to the start of the map check instruction.
+  int delta = *reinterpret_cast<int*>(delta_address);
+
+  // The map address is the last 8 bytes of the 10-byte
+  // immediate move instruction, so we add 2 to get the
+  // offset to the last 8 bytes.
+  Address map_address = test_instruction_address + delta + 2;
+  *(reinterpret_cast<Object**>(map_address)) = map;
+
+  // The offset is in the 32-bit displacement of a seven byte
+  // memory-to-register move instruction (REX.W 0x88 ModR/M disp32),
+  // so we add 3 to get the offset of the displacement.
+  Address offset_address =
+      test_instruction_address + delta + kOffsetToLoadInstruction + 3;
+  *reinterpret_cast<int*>(offset_address) = offset - kHeapObjectTag;
+  return true;
 }
 
 void StoreIC::Generate(MacroAssembler* masm, ExternalReference const& f) {

src/x64/macro-assembler-x64.cc

@@ -318,6 +318,17 @@ void MacroAssembler::Push(Handle<Object> source) {
 }
 
 
+void MacroAssembler::Push(Smi* source) {
+  if (IsUnsafeSmi(source)) {
+    LoadUnsafeSmi(kScratchRegister, source);
+    push(kScratchRegister);
+  } else {
+    int32_t smi = static_cast<int32_t>(reinterpret_cast<intptr_t>(source));
+    push(Immediate(smi));
+  }
+}
+
+
 void MacroAssembler::Jump(ExternalReference ext) {
   movq(kScratchRegister, ext);
   jmp(kScratchRegister);
@@ -363,6 +374,7 @@ void MacroAssembler::Call(Handle<Code> code_object, RelocInfo::Mode rmode) {
   ASSERT(RelocInfo::IsCodeTarget(rmode));
   movq(kScratchRegister, code_object, rmode);
 #ifdef DEBUG
+  // Patch target is kPointer size bytes *before* target label.
   Label target;
   bind(&target);
 #endif

src/x64/macro-assembler-x64.h

@@ -164,6 +164,7 @@ class MacroAssembler: public Assembler {
   void Cmp(Register dst, Handle<Object> source);
   void Cmp(const Operand& dst, Handle<Object> source);
   void Push(Handle<Object> source);
+  void Push(Smi* smi);
 
   // Control Flow
   void Jump(Address destination, RelocInfo::Mode rmode);

test/cctest/cctest.status

@@ -118,6 +118,7 @@ test-api/HugeConsStringOutOfMemory: CRASH || FAIL
 test-api/OutOfMemory: CRASH || FAIL
 test-api/OutOfMemoryNested: CRASH || FAIL
 test-api/Threading: CRASH || FAIL
+test-api/Threading2: PASS || TIMEOUT
 test-api/TryCatchSourceInfo: CRASH || FAIL
 test-api/RegExpInterruption: PASS || TIMEOUT
 test-api/RegExpStringModification: PASS || TIMEOUT