Commit 35a67b74 authored by mvstanton's avatar mvstanton Committed by Commit bot

Vector-ICs - speed towards the monomorphic exit as quickly as possible.

Thanks to some careful assumptions, we can examine the object found at
vector[slot] and trust it's a heap object where the second field is
either a map if it's a WeakCell, or definitely not a map if it's a
Symbol, String or FixedArray. Use this to save a memory read.

BUG=

Review URL: https://codereview.chromium.org/1032163002

Cr-Commit-Position: refs/heads/master@{#27631}
parent 6a222b8f
...@@ -162,9 +162,9 @@ const Register r3 = { kRegister_r3_Code }; ...@@ -162,9 +162,9 @@ const Register r3 = { kRegister_r3_Code };
const Register r4 = { kRegister_r4_Code }; const Register r4 = { kRegister_r4_Code };
const Register r5 = { kRegister_r5_Code }; const Register r5 = { kRegister_r5_Code };
const Register r6 = { kRegister_r6_Code }; const Register r6 = { kRegister_r6_Code };
// Used as constant pool pointer register if FLAG_enable_ool_constant_pool.
const Register r7 = { kRegister_r7_Code };
// Used as context register. // Used as context register.
const Register r7 = {kRegister_r7_Code};
// Used as constant pool pointer register if FLAG_enable_ool_constant_pool.
const Register r8 = { kRegister_r8_Code }; const Register r8 = { kRegister_r8_Code };
// Used as lithium codegen scratch register. // Used as lithium codegen scratch register.
const Register r9 = { kRegister_r9_Code }; const Register r9 = { kRegister_r9_Code };
......
...@@ -4369,21 +4369,15 @@ void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { ...@@ -4369,21 +4369,15 @@ void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) {
static void HandleArrayCases(MacroAssembler* masm, Register receiver, static void HandleArrayCases(MacroAssembler* masm, Register receiver,
Register key, Register vector, Register slot, Register key, Register vector, Register slot,
Register feedback, Register scratch1, Register feedback, Register receiver_map,
Register scratch2, Register scratch3, Register scratch1, Register scratch2,
bool is_polymorphic, Label* miss) { bool is_polymorphic, Label* miss) {
// feedback initially contains the feedback array // feedback initially contains the feedback array
Label next_loop, prepare_next; Label next_loop, prepare_next;
Label load_smi_map, compare_map;
Label start_polymorphic; Label start_polymorphic;
Register receiver_map = scratch1; Register cached_map = scratch1;
Register cached_map = scratch2;
// Receiver might not be a heap object.
__ JumpIfSmi(receiver, &load_smi_map);
__ ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset));
__ bind(&compare_map);
__ ldr(cached_map, __ ldr(cached_map,
FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0))); FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0)));
__ ldr(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); __ ldr(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset));
...@@ -4395,7 +4389,7 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, ...@@ -4395,7 +4389,7 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver,
__ add(pc, handler, Operand(Code::kHeaderSize - kHeapObjectTag)); __ add(pc, handler, Operand(Code::kHeaderSize - kHeapObjectTag));
Register length = scratch3; Register length = scratch2;
__ bind(&start_polymorphic); __ bind(&start_polymorphic);
__ ldr(length, FieldMemOperand(feedback, FixedArray::kLengthOffset)); __ ldr(length, FieldMemOperand(feedback, FixedArray::kLengthOffset));
if (!is_polymorphic) { if (!is_polymorphic) {
...@@ -4415,9 +4409,9 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, ...@@ -4415,9 +4409,9 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver,
// ^ ^ // ^ ^
// | | // | |
// pointer_reg too_far // pointer_reg too_far
// aka feedback scratch3 // aka feedback scratch2
// also need receiver_map (aka scratch1) // also need receiver_map
// use cached_map (scratch2) to look in the weak map values. // use cached_map (scratch1) to look in the weak map values.
__ add(too_far, feedback, Operand::PointerOffsetFromSmiKey(length)); __ add(too_far, feedback, Operand::PointerOffsetFromSmiKey(length));
__ add(too_far, too_far, Operand(FixedArray::kHeaderSize - kHeapObjectTag)); __ add(too_far, too_far, Operand(FixedArray::kHeaderSize - kHeapObjectTag));
__ add(pointer_reg, feedback, __ add(pointer_reg, feedback,
...@@ -4438,42 +4432,23 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, ...@@ -4438,42 +4432,23 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver,
// We exhausted our array of map handler pairs. // We exhausted our array of map handler pairs.
__ jmp(miss); __ jmp(miss);
__ bind(&load_smi_map);
__ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
__ jmp(&compare_map);
} }
static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver,
Register key, Register vector, Register slot, Register receiver_map, Register feedback,
Register weak_cell, Register scratch, Register vector, Register slot,
Label* miss) { Register scratch, Label* compare_map,
// feedback initially contains the feedback array Label* load_smi_map, Label* try_array) {
Label compare_smi_map; __ JumpIfSmi(receiver, load_smi_map);
Register receiver_map = scratch;
Register cached_map = weak_cell;
// Move the weak map into the weak_cell register.
__ ldr(cached_map, FieldMemOperand(weak_cell, WeakCell::kValueOffset));
// Receiver might not be a heap object.
__ JumpIfSmi(receiver, &compare_smi_map);
__ ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); __ ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset));
__ bind(compare_map);
Register cached_map = scratch;
// Move the weak map into the weak_cell register.
__ ldr(cached_map, FieldMemOperand(feedback, WeakCell::kValueOffset));
__ cmp(cached_map, receiver_map); __ cmp(cached_map, receiver_map);
__ b(ne, miss); __ b(ne, try_array);
Register handler = feedback;
Register handler = weak_cell;
__ add(handler, vector, Operand::PointerOffsetFromSmiKey(slot));
__ ldr(handler,
FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize));
__ add(pc, handler, Operand(Code::kHeaderSize - kHeapObjectTag));
// In microbenchmarks, it made sense to unroll this code so that the call to
// the handler is duplicated for a HeapObject receiver and a Smi receiver.
__ bind(&compare_smi_map);
__ CompareRoot(weak_cell, Heap::kHeapNumberMapRootIndex);
__ b(ne, miss);
__ add(handler, vector, Operand::PointerOffsetFromSmiKey(slot)); __ add(handler, vector, Operand::PointerOffsetFromSmiKey(slot));
__ ldr(handler, __ ldr(handler,
FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize));
...@@ -4487,26 +4462,27 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4487,26 +4462,27 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
Register vector = VectorLoadICDescriptor::VectorRegister(); // r3 Register vector = VectorLoadICDescriptor::VectorRegister(); // r3
Register slot = VectorLoadICDescriptor::SlotRegister(); // r0 Register slot = VectorLoadICDescriptor::SlotRegister(); // r0
Register feedback = r4; Register feedback = r4;
Register scratch1 = r5; Register receiver_map = r5;
Register scratch1 = r8;
__ add(feedback, vector, Operand::PointerOffsetFromSmiKey(slot)); __ add(feedback, vector, Operand::PointerOffsetFromSmiKey(slot));
__ ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); __ ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize));
// Is it a weak cell? // Try to quickly handle the monomorphic case without knowing for sure
Label try_array; // if we have a weak cell in feedback. We do know it's safe to look
Label not_array, smi_key, key_okay, miss; // at WeakCell::kValueOffset.
__ ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); Label try_array, load_smi_map, compare_map;
__ CompareRoot(scratch1, Heap::kWeakCellMapRootIndex); Label not_array, miss;
__ b(ne, &try_array); HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector, slot,
HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, scratch1, scratch1, &compare_map, &load_smi_map, &try_array);
&miss);
// Is it a fixed array? // Is it a fixed array?
__ bind(&try_array); __ bind(&try_array);
__ ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset));
__ CompareRoot(scratch1, Heap::kFixedArrayMapRootIndex); __ CompareRoot(scratch1, Heap::kFixedArrayMapRootIndex);
__ b(ne, &not_array); __ b(ne, &not_array);
HandleArrayCases(masm, receiver, name, vector, slot, feedback, scratch1, r8, HandleArrayCases(masm, receiver, name, vector, slot, feedback, receiver_map,
r9, true, &miss); scratch1, r9, true, &miss);
__ bind(&not_array); __ bind(&not_array);
__ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex); __ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex);
...@@ -4515,10 +4491,15 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4515,10 +4491,15 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
Code::ComputeHandlerFlags(Code::LOAD_IC)); Code::ComputeHandlerFlags(Code::LOAD_IC));
masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags, masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags,
false, receiver, name, feedback, false, receiver, name, feedback,
scratch1, r8, r9); receiver_map, scratch1, r9);
__ bind(&miss); __ bind(&miss);
LoadIC::GenerateMiss(masm); LoadIC::GenerateMiss(masm);
__ bind(&load_smi_map);
__ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
__ jmp(&compare_map);
} }
...@@ -4538,30 +4519,31 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4538,30 +4519,31 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
Register vector = VectorLoadICDescriptor::VectorRegister(); // r3 Register vector = VectorLoadICDescriptor::VectorRegister(); // r3
Register slot = VectorLoadICDescriptor::SlotRegister(); // r0 Register slot = VectorLoadICDescriptor::SlotRegister(); // r0
Register feedback = r4; Register feedback = r4;
Register scratch1 = r5; Register receiver_map = r5;
Register scratch1 = r8;
__ add(feedback, vector, Operand::PointerOffsetFromSmiKey(slot)); __ add(feedback, vector, Operand::PointerOffsetFromSmiKey(slot));
__ ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); __ ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize));
// Is it a weak cell? // Try to quickly handle the monomorphic case without knowing for sure
Label try_array; // if we have a weak cell in feedback. We do know it's safe to look
Label not_array, smi_key, key_okay, miss; // at WeakCell::kValueOffset.
__ ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); Label try_array, load_smi_map, compare_map;
__ CompareRoot(scratch1, Heap::kWeakCellMapRootIndex); Label not_array, miss;
__ b(ne, &try_array); HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector, slot,
HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, scratch1, scratch1, &compare_map, &load_smi_map, &try_array);
&miss);
__ bind(&try_array); __ bind(&try_array);
// Is it a fixed array? // Is it a fixed array?
__ ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset));
__ CompareRoot(scratch1, Heap::kFixedArrayMapRootIndex); __ CompareRoot(scratch1, Heap::kFixedArrayMapRootIndex);
__ b(ne, &not_array); __ b(ne, &not_array);
// We have a polymorphic element handler. // We have a polymorphic element handler.
Label polymorphic, try_poly_name; Label polymorphic, try_poly_name;
__ bind(&polymorphic); __ bind(&polymorphic);
HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, r8, HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map,
r9, true, &miss); scratch1, r9, true, &miss);
__ bind(&not_array); __ bind(&not_array);
// Is it generic? // Is it generic?
...@@ -4580,11 +4562,15 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4580,11 +4562,15 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
__ add(feedback, vector, Operand::PointerOffsetFromSmiKey(slot)); __ add(feedback, vector, Operand::PointerOffsetFromSmiKey(slot));
__ ldr(feedback, __ ldr(feedback,
FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize)); FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize));
HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, r8, HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map,
r9, false, &miss); scratch1, r9, false, &miss);
__ bind(&miss); __ bind(&miss);
KeyedLoadIC::GenerateMiss(masm); KeyedLoadIC::GenerateMiss(masm);
__ bind(&load_smi_map);
__ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
__ jmp(&compare_map);
} }
......
...@@ -4502,21 +4502,16 @@ void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) { ...@@ -4502,21 +4502,16 @@ void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) {
static void HandleArrayCases(MacroAssembler* masm, Register receiver, static void HandleArrayCases(MacroAssembler* masm, Register receiver,
Register key, Register vector, Register slot, Register key, Register vector, Register slot,
Register feedback, Register scratch1, Register feedback, Register receiver_map,
Register scratch2, Register scratch3, Register scratch1, Register scratch2,
bool is_polymorphic, Label* miss) { bool is_polymorphic, Label* miss) {
// feedback initially contains the feedback array // feedback initially contains the feedback array
Label next_loop, prepare_next; Label next_loop, prepare_next;
Label load_smi_map, compare_map; Label load_smi_map, compare_map;
Label start_polymorphic; Label start_polymorphic;
Register receiver_map = scratch1; Register cached_map = scratch1;
Register cached_map = scratch2;
// Receiver might not be a heap object.
__ JumpIfSmi(receiver, &load_smi_map);
__ Ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset));
__ Bind(&compare_map);
__ Ldr(cached_map, __ Ldr(cached_map,
FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0))); FieldMemOperand(feedback, FixedArray::OffsetOfElementAt(0)));
__ Ldr(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset)); __ Ldr(cached_map, FieldMemOperand(cached_map, WeakCell::kValueOffset));
...@@ -4528,7 +4523,7 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, ...@@ -4528,7 +4523,7 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver,
__ Add(handler, handler, Code::kHeaderSize - kHeapObjectTag); __ Add(handler, handler, Code::kHeaderSize - kHeapObjectTag);
__ Jump(feedback); __ Jump(feedback);
Register length = scratch3; Register length = scratch2;
__ Bind(&start_polymorphic); __ Bind(&start_polymorphic);
__ Ldr(length, FieldMemOperand(feedback, FixedArray::kLengthOffset)); __ Ldr(length, FieldMemOperand(feedback, FixedArray::kLengthOffset));
if (!is_polymorphic) { if (!is_polymorphic) {
...@@ -4546,9 +4541,9 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, ...@@ -4546,9 +4541,9 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver,
// ^ ^ // ^ ^
// | | // | |
// pointer_reg too_far // pointer_reg too_far
// aka feedback scratch3 // aka feedback scratch2
// also need receiver_map (aka scratch1) // also need receiver_map
// use cached_map (scratch2) to look in the weak map values. // use cached_map (scratch1) to look in the weak map values.
__ Add(too_far, feedback, __ Add(too_far, feedback,
Operand::UntagSmiAndScale(length, kPointerSizeLog2)); Operand::UntagSmiAndScale(length, kPointerSizeLog2));
__ Add(too_far, too_far, FixedArray::kHeaderSize - kHeapObjectTag); __ Add(too_far, too_far, FixedArray::kHeaderSize - kHeapObjectTag);
...@@ -4571,43 +4566,24 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, ...@@ -4571,43 +4566,24 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver,
// We exhausted our array of map handler pairs. // We exhausted our array of map handler pairs.
__ jmp(miss); __ jmp(miss);
__ Bind(&load_smi_map);
__ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
__ jmp(&compare_map);
} }
static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver,
Register key, Register vector, Register slot, Register receiver_map, Register feedback,
Register weak_cell, Register scratch, Register vector, Register slot,
Label* miss) { Register scratch, Label* compare_map,
// feedback initially contains the feedback array Label* load_smi_map, Label* try_array) {
Label compare_smi_map; __ JumpIfSmi(receiver, load_smi_map);
Register receiver_map = scratch;
Register cached_map = weak_cell;
// Move the weak map into the weak_cell register.
__ Ldr(cached_map, FieldMemOperand(weak_cell, WeakCell::kValueOffset));
// Receiver might not be a heap object.
__ JumpIfSmi(receiver, &compare_smi_map);
__ Ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset)); __ Ldr(receiver_map, FieldMemOperand(receiver, HeapObject::kMapOffset));
__ bind(compare_map);
Register cached_map = scratch;
// Move the weak map into the weak_cell register.
__ Ldr(cached_map, FieldMemOperand(feedback, WeakCell::kValueOffset));
__ Cmp(cached_map, receiver_map); __ Cmp(cached_map, receiver_map);
__ B(ne, miss); __ B(ne, try_array);
Register handler = weak_cell;
__ Add(handler, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2));
__ Ldr(handler,
FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize));
__ Add(handler, handler, Code::kHeaderSize - kHeapObjectTag);
__ Jump(weak_cell);
// In microbenchmarks, it made sense to unroll this code so that the call to Register handler = feedback;
// the handler is duplicated for a HeapObject receiver and a Smi receiver.
// TODO(mvstanton): does this hold on ARM?
__ Bind(&compare_smi_map);
__ JumpIfNotRoot(weak_cell, Heap::kHeapNumberMapRootIndex, miss);
__ Add(handler, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2)); __ Add(handler, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2));
__ Ldr(handler, __ Ldr(handler,
FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize)); FieldMemOperand(handler, FixedArray::kHeaderSize + kPointerSize));
...@@ -4622,24 +4598,26 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4622,24 +4598,26 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
Register vector = VectorLoadICDescriptor::VectorRegister(); // x3 Register vector = VectorLoadICDescriptor::VectorRegister(); // x3
Register slot = VectorLoadICDescriptor::SlotRegister(); // x0 Register slot = VectorLoadICDescriptor::SlotRegister(); // x0
Register feedback = x4; Register feedback = x4;
Register scratch1 = x5; Register receiver_map = x5;
Register scratch1 = x6;
__ Add(feedback, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2)); __ Add(feedback, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2));
__ Ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); __ Ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize));
// Is it a weak cell? // Try to quickly handle the monomorphic case without knowing for sure
Label try_array; // if we have a weak cell in feedback. We do know it's safe to look
Label not_array, smi_key, key_okay, miss; // at WeakCell::kValueOffset.
__ Ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); Label try_array, load_smi_map, compare_map;
__ JumpIfNotRoot(scratch1, Heap::kWeakCellMapRootIndex, &try_array); Label not_array, miss;
HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, scratch1, HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector, slot,
&miss); scratch1, &compare_map, &load_smi_map, &try_array);
// Is it a fixed array? // Is it a fixed array?
__ Bind(&try_array); __ Bind(&try_array);
__ Ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset));
__ JumpIfNotRoot(scratch1, Heap::kFixedArrayMapRootIndex, &not_array); __ JumpIfNotRoot(scratch1, Heap::kFixedArrayMapRootIndex, &not_array);
HandleArrayCases(masm, receiver, name, vector, slot, feedback, scratch1, x6, HandleArrayCases(masm, receiver, name, vector, slot, feedback, receiver_map,
x7, true, &miss); scratch1, x7, true, &miss);
__ Bind(&not_array); __ Bind(&not_array);
__ JumpIfNotRoot(feedback, Heap::kmegamorphic_symbolRootIndex, &miss); __ JumpIfNotRoot(feedback, Heap::kmegamorphic_symbolRootIndex, &miss);
...@@ -4647,10 +4625,14 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4647,10 +4625,14 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
Code::ComputeHandlerFlags(Code::LOAD_IC)); Code::ComputeHandlerFlags(Code::LOAD_IC));
masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags, masm->isolate()->stub_cache()->GenerateProbe(masm, Code::LOAD_IC, code_flags,
false, receiver, name, feedback, false, receiver, name, feedback,
scratch1, x6, x7); receiver_map, scratch1, x7);
__ Bind(&miss); __ Bind(&miss);
LoadIC::GenerateMiss(masm); LoadIC::GenerateMiss(masm);
__ Bind(&load_smi_map);
__ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
__ jmp(&compare_map);
} }
...@@ -4670,28 +4652,30 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4670,28 +4652,30 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
Register vector = VectorLoadICDescriptor::VectorRegister(); // x3 Register vector = VectorLoadICDescriptor::VectorRegister(); // x3
Register slot = VectorLoadICDescriptor::SlotRegister(); // x0 Register slot = VectorLoadICDescriptor::SlotRegister(); // x0
Register feedback = x4; Register feedback = x4;
Register scratch1 = x5; Register receiver_map = x5;
Register scratch1 = x6;
__ Add(feedback, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2)); __ Add(feedback, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2));
__ Ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize)); __ Ldr(feedback, FieldMemOperand(feedback, FixedArray::kHeaderSize));
// Is it a weak cell? // Try to quickly handle the monomorphic case without knowing for sure
Label try_array; // if we have a weak cell in feedback. We do know it's safe to look
Label not_array, smi_key, key_okay, miss; // at WeakCell::kValueOffset.
__ Ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset)); Label try_array, load_smi_map, compare_map;
__ JumpIfNotRoot(scratch1, Heap::kWeakCellMapRootIndex, &try_array); Label not_array, miss;
HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, scratch1, HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector, slot,
&miss); scratch1, &compare_map, &load_smi_map, &try_array);
__ Bind(&try_array); __ Bind(&try_array);
// Is it a fixed array? // Is it a fixed array?
__ Ldr(scratch1, FieldMemOperand(feedback, HeapObject::kMapOffset));
__ JumpIfNotRoot(scratch1, Heap::kFixedArrayMapRootIndex, &not_array); __ JumpIfNotRoot(scratch1, Heap::kFixedArrayMapRootIndex, &not_array);
// We have a polymorphic element handler. // We have a polymorphic element handler.
Label polymorphic, try_poly_name; Label polymorphic, try_poly_name;
__ Bind(&polymorphic); __ Bind(&polymorphic);
HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, x6, HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map,
x7, true, &miss); scratch1, x7, true, &miss);
__ Bind(&not_array); __ Bind(&not_array);
// Is it generic? // Is it generic?
...@@ -4710,11 +4694,15 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4710,11 +4694,15 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
__ Add(feedback, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2)); __ Add(feedback, vector, Operand::UntagSmiAndScale(slot, kPointerSizeLog2));
__ Ldr(feedback, __ Ldr(feedback,
FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize)); FieldMemOperand(feedback, FixedArray::kHeaderSize + kPointerSize));
HandleArrayCases(masm, receiver, key, vector, slot, feedback, scratch1, x6, HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map,
x7, false, &miss); scratch1, x7, false, &miss);
__ Bind(&miss); __ Bind(&miss);
KeyedLoadIC::GenerateMiss(masm); KeyedLoadIC::GenerateMiss(masm);
__ Bind(&load_smi_map);
__ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
__ jmp(&compare_map);
} }
......
...@@ -4351,24 +4351,18 @@ void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) { ...@@ -4351,24 +4351,18 @@ void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) {
static void HandleArrayCases(MacroAssembler* masm, Register receiver, static void HandleArrayCases(MacroAssembler* masm, Register receiver,
Register key, Register vector, Register slot, Register key, Register vector, Register slot,
Register feedback, Register scratch1, Register feedback, Register receiver_map,
Register scratch2, Register scratch3, Register scratch1, Register scratch2,
Register scratch4, bool is_polymorphic, Register scratch3, bool is_polymorphic,
Label* miss) { Label* miss) {
// feedback initially contains the feedback array // feedback initially contains the feedback array
Label next_loop, prepare_next; Label next_loop, prepare_next;
Label load_smi_map, compare_map;
Label start_polymorphic; Label start_polymorphic;
Register receiver_map = scratch1; Register counter = scratch1;
Register counter = scratch2; Register length = scratch2;
Register length = scratch3; Register cached_map = scratch3;
Register cached_map = scratch4;
// Receiver might not be a heap object.
__ JumpIfSmi(receiver, &load_smi_map);
__ movp(receiver_map, FieldOperand(receiver, 0));
__ bind(&compare_map);
__ movp(cached_map, FieldOperand(feedback, FixedArray::OffsetOfElementAt(0))); __ movp(cached_map, FieldOperand(feedback, FixedArray::OffsetOfElementAt(0)));
__ cmpp(receiver_map, FieldOperand(cached_map, WeakCell::kValueOffset)); __ cmpp(receiver_map, FieldOperand(cached_map, WeakCell::kValueOffset));
__ j(not_equal, &start_polymorphic); __ j(not_equal, &start_polymorphic);
...@@ -4407,39 +4401,21 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver, ...@@ -4407,39 +4401,21 @@ static void HandleArrayCases(MacroAssembler* masm, Register receiver,
// We exhausted our array of map handler pairs. // We exhausted our array of map handler pairs.
__ jmp(miss); __ jmp(miss);
__ bind(&load_smi_map);
__ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
__ jmp(&compare_map);
} }
static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver, static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver,
Register key, Register vector, Register slot, Register receiver_map, Register feedback,
Register weak_cell, Register integer_slot, Register vector, Register integer_slot,
Label* miss) { Label* compare_map, Label* load_smi_map,
// feedback initially contains the feedback array Label* try_array) {
Label compare_smi_map; __ JumpIfSmi(receiver, load_smi_map);
__ movp(receiver_map, FieldOperand(receiver, 0));
// Move the weak map into the weak_cell register.
Register ic_map = weak_cell;
__ movp(ic_map, FieldOperand(weak_cell, WeakCell::kValueOffset));
// Receiver might not be a heap object.
__ JumpIfSmi(receiver, &compare_smi_map);
__ cmpp(ic_map, FieldOperand(receiver, 0));
__ j(not_equal, miss);
Register handler = weak_cell;
__ movp(handler, FieldOperand(vector, integer_slot, times_pointer_size,
FixedArray::kHeaderSize + kPointerSize));
__ leap(handler, FieldOperand(handler, Code::kHeaderSize));
__ jmp(handler);
// In microbenchmarks, it made sense to unroll this code so that the call to __ bind(compare_map);
// the handler is duplicated for a HeapObject receiver and a Smi receiver. __ cmpp(receiver_map, FieldOperand(feedback, WeakCell::kValueOffset));
__ bind(&compare_smi_map); __ j(not_equal, try_array);
__ CompareRoot(ic_map, Heap::kHeapNumberMapRootIndex); Register handler = feedback;
__ j(not_equal, miss);
__ movp(handler, FieldOperand(vector, integer_slot, times_pointer_size, __ movp(handler, FieldOperand(vector, integer_slot, times_pointer_size,
FixedArray::kHeaderSize + kPointerSize)); FixedArray::kHeaderSize + kPointerSize));
__ leap(handler, FieldOperand(handler, Code::kHeaderSize)); __ leap(handler, FieldOperand(handler, Code::kHeaderSize));
...@@ -4464,25 +4440,26 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4464,25 +4440,26 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
Register slot = VectorLoadICDescriptor::SlotRegister(); // rax Register slot = VectorLoadICDescriptor::SlotRegister(); // rax
Register feedback = rdi; Register feedback = rdi;
Register integer_slot = r8; Register integer_slot = r8;
Register receiver_map = r9;
__ SmiToInteger32(integer_slot, slot); __ SmiToInteger32(integer_slot, slot);
__ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size, __ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size,
FixedArray::kHeaderSize)); FixedArray::kHeaderSize));
// Is it a weak cell? // Try to quickly handle the monomorphic case without knowing for sure
Label try_array; // if we have a weak cell in feedback. We do know it's safe to look
Label not_array, smi_key, key_okay, miss; // at WeakCell::kValueOffset.
__ CompareRoot(FieldOperand(feedback, 0), Heap::kWeakCellMapRootIndex); Label try_array, load_smi_map, compare_map;
__ j(not_equal, &try_array); Label not_array, miss;
HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback, HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector,
integer_slot, &miss); integer_slot, &compare_map, &load_smi_map, &try_array);
// Is it a fixed array? // Is it a fixed array?
__ bind(&try_array); __ bind(&try_array);
__ CompareRoot(FieldOperand(feedback, 0), Heap::kFixedArrayMapRootIndex); __ CompareRoot(FieldOperand(feedback, 0), Heap::kFixedArrayMapRootIndex);
__ j(not_equal, &not_array); __ j(not_equal, &not_array);
HandleArrayCases(masm, receiver, name, vector, slot, feedback, integer_slot, HandleArrayCases(masm, receiver, name, vector, slot, feedback, receiver_map,
r9, r11, r15, true, &miss); integer_slot, r11, r15, true, &miss);
__ bind(&not_array); __ bind(&not_array);
__ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex); __ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex);
...@@ -4494,6 +4471,10 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4494,6 +4471,10 @@ void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
__ bind(&miss); __ bind(&miss);
LoadIC::GenerateMiss(masm); LoadIC::GenerateMiss(masm);
__ bind(&load_smi_map);
__ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
__ jmp(&compare_map);
} }
...@@ -4514,18 +4495,19 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4514,18 +4495,19 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
Register slot = VectorLoadICDescriptor::SlotRegister(); // rax Register slot = VectorLoadICDescriptor::SlotRegister(); // rax
Register feedback = rdi; Register feedback = rdi;
Register integer_slot = r8; Register integer_slot = r8;
Register receiver_map = r9;
__ SmiToInteger32(integer_slot, slot); __ SmiToInteger32(integer_slot, slot);
__ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size, __ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size,
FixedArray::kHeaderSize)); FixedArray::kHeaderSize));
// Is it a weak cell? // Try to quickly handle the monomorphic case without knowing for sure
Label try_array; // if we have a weak cell in feedback. We do know it's safe to look
Label not_array, smi_key, key_okay, miss; // at WeakCell::kValueOffset.
__ CompareRoot(FieldOperand(feedback, 0), Heap::kWeakCellMapRootIndex); Label try_array, load_smi_map, compare_map;
__ j(not_equal, &try_array); Label not_array, miss;
HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback, HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector,
integer_slot, &miss); integer_slot, &compare_map, &load_smi_map, &try_array);
__ bind(&try_array); __ bind(&try_array);
// Is it a fixed array? // Is it a fixed array?
...@@ -4535,8 +4517,8 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4535,8 +4517,8 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
// We have a polymorphic element handler. // We have a polymorphic element handler.
Label polymorphic, try_poly_name; Label polymorphic, try_poly_name;
__ bind(&polymorphic); __ bind(&polymorphic);
HandleArrayCases(masm, receiver, key, vector, slot, feedback, integer_slot, HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map,
r9, r11, r15, true, &miss); integer_slot, r11, r15, true, &miss);
__ bind(&not_array); __ bind(&not_array);
// Is it generic? // Is it generic?
...@@ -4554,11 +4536,15 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) { ...@@ -4554,11 +4536,15 @@ void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
// at least one map/handler pair. // at least one map/handler pair.
__ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size, __ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size,
FixedArray::kHeaderSize + kPointerSize)); FixedArray::kHeaderSize + kPointerSize));
HandleArrayCases(masm, receiver, key, vector, slot, feedback, integer_slot, HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map,
r9, r11, r15, false, &miss); integer_slot, r11, r15, false, &miss);
__ bind(&miss); __ bind(&miss);
KeyedLoadIC::GenerateMiss(masm); KeyedLoadIC::GenerateMiss(masm);
__ bind(&load_smi_map);
__ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
__ jmp(&compare_map);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment