Commit c1a4f747 authored by Michael Stanton's avatar Michael Stanton

[ic] Record call counts for monomorphic calls made with an IC.

The idea is that TurboFan can use this information for more intelligent
inlining.

R=bmeurer@chromium.org

Review URL: https://codereview.chromium.org/1201193003

Cr-Commit-Position: refs/heads/master@{#29281}
parent 9e7af9ef
......@@ -2706,6 +2706,13 @@ void CallIC_ArrayStub::Generate(MacroAssembler* masm) {
__ CompareRoot(r5, Heap::kAllocationSiteMapRootIndex);
__ b(ne, &miss);
// Increment the call count for monomorphic function calls.
__ add(r2, r2, Operand::PointerOffsetFromSmiKey(r3));
__ add(r2, r2, Operand(FixedArray::kHeaderSize + kPointerSize));
__ ldr(r3, FieldMemOperand(r2, 0));
__ add(r3, r3, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ str(r3, FieldMemOperand(r2, 0));
__ mov(r2, r4);
__ mov(r3, r1);
ArrayConstructorStub stub(masm->isolate(), arg_count());
......@@ -2765,6 +2772,13 @@ void CallICStub::Generate(MacroAssembler* masm) {
// convincing us that we have a monomorphic JSFunction.
__ JumpIfSmi(r1, &extra_checks_or_miss);
// Increment the call count for monomorphic function calls.
__ add(r2, r2, Operand::PointerOffsetFromSmiKey(r3));
__ add(r2, r2, Operand(FixedArray::kHeaderSize + kPointerSize));
__ ldr(r3, FieldMemOperand(r2, 0));
__ add(r3, r3, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ str(r3, FieldMemOperand(r2, 0));
__ bind(&have_js_function);
if (CallAsMethod()) {
EmitContinueIfStrictOrNative(masm, &cont);
......@@ -2840,6 +2854,11 @@ void CallICStub::Generate(MacroAssembler* masm) {
__ add(r4, r4, Operand(Smi::FromInt(1)));
__ str(r4, FieldMemOperand(r2, with_types_offset));
// Initialize the call counter.
__ Move(r5, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ add(r4, r2, Operand::PointerOffsetFromSmiKey(r3));
__ str(r5, FieldMemOperand(r4, FixedArray::kHeaderSize + kPointerSize));
// Store the function. Use a stub since we need a frame for allocation.
// r2 - vector
// r3 - slot
......
......@@ -3091,10 +3091,18 @@ void CallIC_ArrayStub::Generate(MacroAssembler* masm) {
__ Ldr(map, FieldMemOperand(scratch, HeapObject::kMapOffset));
__ JumpIfNotRoot(map, Heap::kAllocationSiteMapRootIndex, &miss);
// Increment the call count for monomorphic function calls.
__ Add(feedback_vector, feedback_vector,
Operand::UntagSmiAndScale(index, kPointerSizeLog2));
__ Add(feedback_vector, feedback_vector,
Operand(FixedArray::kHeaderSize + kPointerSize));
__ Ldr(index, FieldMemOperand(feedback_vector, 0));
__ Add(index, index, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ Str(index, FieldMemOperand(feedback_vector, 0));
Register allocation_site = feedback_vector;
Register original_constructor = index;
__ Mov(allocation_site, scratch);
Register original_constructor = x3;
__ Mov(original_constructor, function);
ArrayConstructorStub stub(masm->isolate(), arg_count());
__ TailCallStub(&stub);
......@@ -3160,6 +3168,15 @@ void CallICStub::Generate(MacroAssembler* masm) {
// convincing us that we have a monomorphic JSFunction.
__ JumpIfSmi(function, &extra_checks_or_miss);
// Increment the call count for monomorphic function calls.
__ Add(feedback_vector, feedback_vector,
Operand::UntagSmiAndScale(index, kPointerSizeLog2));
__ Add(feedback_vector, feedback_vector,
Operand(FixedArray::kHeaderSize + kPointerSize));
__ Ldr(index, FieldMemOperand(feedback_vector, 0));
__ Add(index, index, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ Str(index, FieldMemOperand(feedback_vector, 0));
__ bind(&have_js_function);
if (CallAsMethod()) {
EmitContinueIfStrictOrNative(masm, &cont);
......@@ -3235,6 +3252,12 @@ void CallICStub::Generate(MacroAssembler* masm) {
__ Adds(x4, x4, Operand(Smi::FromInt(1)));
__ Str(x4, FieldMemOperand(feedback_vector, with_types_offset));
// Initialize the call counter.
__ Mov(x5, Smi::FromInt(CallICNexus::kCallCountIncrement));
__ Adds(x4, feedback_vector,
Operand::UntagSmiAndScale(index, kPointerSizeLog2));
__ Str(x5, FieldMemOperand(x4, FixedArray::kHeaderSize + kPointerSize));
// Store the function. Use a stub since we need a frame for allocation.
// x2 - vector
// x3 - slot
......
......@@ -2247,6 +2247,11 @@ void CallIC_ArrayStub::Generate(MacroAssembler* masm) {
factory->allocation_site_map());
__ j(not_equal, &miss);
// Increment the call count for monomorphic function calls.
__ add(FieldOperand(ebx, edx, times_half_pointer_size,
FixedArray::kHeaderSize + kPointerSize),
Immediate(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ mov(ebx, ecx);
__ mov(edx, edi);
ArrayConstructorStub stub(masm->isolate(), arg_count());
......@@ -2306,6 +2311,11 @@ void CallICStub::Generate(MacroAssembler* masm) {
// convincing us that we have a monomorphic JSFunction.
__ JumpIfSmi(edi, &extra_checks_or_miss);
// Increment the call count for monomorphic function calls.
__ add(FieldOperand(ebx, edx, times_half_pointer_size,
FixedArray::kHeaderSize + kPointerSize),
Immediate(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ bind(&have_js_function);
if (CallAsMethod()) {
EmitContinueIfStrictOrNative(masm, &cont);
......@@ -2377,6 +2387,11 @@ void CallICStub::Generate(MacroAssembler* masm) {
// Update stats.
__ add(FieldOperand(ebx, with_types_offset), Immediate(Smi::FromInt(1)));
// Initialize the call counter.
__ mov(FieldOperand(ebx, edx, times_half_pointer_size,
FixedArray::kHeaderSize + kPointerSize),
Immediate(Smi::FromInt(CallICNexus::kCallCountIncrement)));
// Store the function. Use a stub since we need a frame for allocation.
// ebx - vector
// edx - slot
......
......@@ -2842,6 +2842,13 @@ void CallIC_ArrayStub::Generate(MacroAssembler* masm) {
__ LoadRoot(at, Heap::kAllocationSiteMapRootIndex);
__ Branch(&miss, ne, t1, Operand(at));
// Increment the call count for monomorphic function calls.
__ sll(at, a3, kPointerSizeLog2 - kSmiTagSize);
__ Addu(at, a2, Operand(at));
__ lw(a3, FieldMemOperand(at, FixedArray::kHeaderSize + kPointerSize));
__ Addu(a3, a3, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ sw(a3, FieldMemOperand(at, FixedArray::kHeaderSize + kPointerSize));
__ mov(a2, t0);
__ mov(a3, a1);
ArrayConstructorStub stub(masm->isolate(), arg_count());
......@@ -2901,6 +2908,13 @@ void CallICStub::Generate(MacroAssembler* masm) {
// convincing us that we have a monomorphic JSFunction.
__ JumpIfSmi(a1, &extra_checks_or_miss);
// Increment the call count for monomorphic function calls.
__ sll(at, a3, kPointerSizeLog2 - kSmiTagSize);
__ Addu(at, a2, Operand(at));
__ lw(a3, FieldMemOperand(at, FixedArray::kHeaderSize + kPointerSize));
__ Addu(a3, a3, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ sw(a3, FieldMemOperand(at, FixedArray::kHeaderSize + kPointerSize));
__ bind(&have_js_function);
if (CallAsMethod()) {
EmitContinueIfStrictOrNative(masm, &cont);
......@@ -2976,6 +2990,12 @@ void CallICStub::Generate(MacroAssembler* masm) {
__ Addu(t0, t0, Operand(Smi::FromInt(1)));
__ sw(t0, FieldMemOperand(a2, with_types_offset));
// Initialize the call counter.
__ sll(at, a3, kPointerSizeLog2 - kSmiTagSize);
__ Addu(at, a2, Operand(at));
__ li(t0, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ sw(t0, FieldMemOperand(at, FixedArray::kHeaderSize + kPointerSize));
// Store the function. Use a stub since we need a frame for allocation.
// a2 - vector
// a3 - slot
......
......@@ -2918,6 +2918,13 @@ void CallIC_ArrayStub::Generate(MacroAssembler* masm) {
__ LoadRoot(at, Heap::kAllocationSiteMapRootIndex);
__ Branch(&miss, ne, a5, Operand(at));
// Increment the call count for monomorphic function calls.
__ dsrl(t0, a3, 32 - kPointerSizeLog2);
__ Daddu(a3, a2, Operand(t0));
__ ld(t0, FieldMemOperand(a3, FixedArray::kHeaderSize + kPointerSize));
__ Daddu(t0, t0, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ sd(t0, FieldMemOperand(a3, FixedArray::kHeaderSize + kPointerSize));
__ mov(a2, a4);
__ mov(a3, a1);
ArrayConstructorStub stub(masm->isolate(), arg_count());
......@@ -2977,6 +2984,13 @@ void CallICStub::Generate(MacroAssembler* masm) {
// convincing us that we have a monomorphic JSFunction.
__ JumpIfSmi(a1, &extra_checks_or_miss);
// Increment the call count for monomorphic function calls.
__ dsrl(t0, a3, 32 - kPointerSizeLog2);
__ Daddu(a3, a2, Operand(t0));
__ ld(t0, FieldMemOperand(a3, FixedArray::kHeaderSize + kPointerSize));
__ Daddu(t0, t0, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ sd(t0, FieldMemOperand(a3, FixedArray::kHeaderSize + kPointerSize));
__ bind(&have_js_function);
if (CallAsMethod()) {
EmitContinueIfStrictOrNative(masm, &cont);
......@@ -3052,6 +3066,12 @@ void CallICStub::Generate(MacroAssembler* masm) {
__ Daddu(a4, a4, Operand(Smi::FromInt(1)));
__ sd(a4, FieldMemOperand(a2, with_types_offset));
// Initialize the call counter.
__ dsrl(at, a3, 32 - kPointerSizeLog2);
__ Daddu(at, a2, Operand(at));
__ li(t0, Operand(Smi::FromInt(CallICNexus::kCallCountIncrement)));
__ sd(t0, FieldMemOperand(at, FixedArray::kHeaderSize + kPointerSize));
// Store the function. Use a stub since we need a frame for allocation.
// a2 - vector
// a3 - slot
......
......@@ -30,12 +30,6 @@ Handle<Object> TypeFeedbackVector::PremonomorphicSentinel(Isolate* isolate) {
}
Handle<Object> TypeFeedbackVector::MonomorphicArraySentinel(
Isolate* isolate, ElementsKind elements_kind) {
return Handle<Object>(Smi::FromInt(static_cast<int>(elements_kind)), isolate);
}
Object* TypeFeedbackVector::RawUninitializedSentinel(Heap* heap) {
return heap->uninitialized_symbol();
}
......
......@@ -262,11 +262,11 @@ InlineCacheState LoadICNexus::StateFromFeedback() const {
Isolate* isolate = GetIsolate();
Object* feedback = GetFeedback();
if (feedback == *vector()->UninitializedSentinel(isolate)) {
if (feedback == *TypeFeedbackVector::UninitializedSentinel(isolate)) {
return UNINITIALIZED;
} else if (feedback == *vector()->MegamorphicSentinel(isolate)) {
} else if (feedback == *TypeFeedbackVector::MegamorphicSentinel(isolate)) {
return MEGAMORPHIC;
} else if (feedback == *vector()->PremonomorphicSentinel(isolate)) {
} else if (feedback == *TypeFeedbackVector::PremonomorphicSentinel(isolate)) {
return PREMONOMORPHIC;
} else if (feedback->IsFixedArray()) {
// Determine state purely by our structure, don't check if the maps are
......@@ -285,11 +285,11 @@ InlineCacheState KeyedLoadICNexus::StateFromFeedback() const {
Isolate* isolate = GetIsolate();
Object* feedback = GetFeedback();
if (feedback == *vector()->UninitializedSentinel(isolate)) {
if (feedback == *TypeFeedbackVector::UninitializedSentinel(isolate)) {
return UNINITIALIZED;
} else if (feedback == *vector()->PremonomorphicSentinel(isolate)) {
} else if (feedback == *TypeFeedbackVector::PremonomorphicSentinel(isolate)) {
return PREMONOMORPHIC;
} else if (feedback == *vector()->MegamorphicSentinel(isolate)) {
} else if (feedback == *TypeFeedbackVector::MegamorphicSentinel(isolate)) {
return MEGAMORPHIC;
} else if (feedback->IsFixedArray()) {
// Determine state purely by our structure, don't check if the maps are
......@@ -311,25 +311,39 @@ InlineCacheState KeyedLoadICNexus::StateFromFeedback() const {
InlineCacheState CallICNexus::StateFromFeedback() const {
Isolate* isolate = GetIsolate();
Object* feedback = GetFeedback();
DCHECK(GetFeedbackExtra() == *vector()->UninitializedSentinel(isolate) ||
GetFeedbackExtra() == Smi::FromInt(kHasReturnedMinusZeroSentinel));
DCHECK(GetFeedbackExtra() ==
*TypeFeedbackVector::UninitializedSentinel(isolate) ||
GetFeedbackExtra()->IsSmi());
if (feedback == *vector()->MegamorphicSentinel(isolate)) {
if (feedback == *TypeFeedbackVector::MegamorphicSentinel(isolate)) {
return GENERIC;
} else if (feedback->IsAllocationSite() || feedback->IsWeakCell()) {
return MONOMORPHIC;
}
CHECK(feedback == *vector()->UninitializedSentinel(isolate));
CHECK(feedback == *TypeFeedbackVector::UninitializedSentinel(isolate));
return UNINITIALIZED;
}
int CallICNexus::ExtractCallCount() {
Object* call_count = GetFeedbackExtra();
if (call_count->IsSmi()) {
int value = Smi::cast(call_count)->value() / 2;
return value;
}
return -1;
}
void CallICNexus::Clear(Code* host) { CallIC::Clear(GetIsolate(), host, this); }
void CallICNexus::ConfigureGeneric() {
SetFeedback(*vector()->MegamorphicSentinel(GetIsolate()), SKIP_WRITE_BARRIER);
SetFeedback(*TypeFeedbackVector::MegamorphicSentinel(GetIsolate()),
SKIP_WRITE_BARRIER);
SetFeedbackExtra(*TypeFeedbackVector::UninitializedSentinel(GetIsolate()),
SKIP_WRITE_BARRIER);
}
......@@ -340,48 +354,55 @@ void CallICNexus::ConfigureMonomorphicArray() {
GetIsolate()->factory()->NewAllocationSite();
SetFeedback(*new_site);
}
SetFeedbackExtra(Smi::FromInt(kCallCountIncrement), SKIP_WRITE_BARRIER);
}
void CallICNexus::ConfigureUninitialized() {
SetFeedback(*vector()->UninitializedSentinel(GetIsolate()),
SetFeedback(*TypeFeedbackVector::UninitializedSentinel(GetIsolate()),
SKIP_WRITE_BARRIER);
SetFeedbackExtra(*TypeFeedbackVector::UninitializedSentinel(GetIsolate()),
SKIP_WRITE_BARRIER);
}
void CallICNexus::ConfigureMonomorphic(Handle<JSFunction> function) {
Handle<WeakCell> new_cell = GetIsolate()->factory()->NewWeakCell(function);
SetFeedback(*new_cell);
SetFeedbackExtra(Smi::FromInt(kCallCountIncrement), SKIP_WRITE_BARRIER);
}
void KeyedLoadICNexus::ConfigureMegamorphic() {
Isolate* isolate = GetIsolate();
SetFeedback(*vector()->MegamorphicSentinel(isolate), SKIP_WRITE_BARRIER);
SetFeedbackExtra(*vector()->UninitializedSentinel(isolate),
SetFeedback(*TypeFeedbackVector::MegamorphicSentinel(isolate),
SKIP_WRITE_BARRIER);
SetFeedbackExtra(*TypeFeedbackVector::UninitializedSentinel(isolate),
SKIP_WRITE_BARRIER);
}
void LoadICNexus::ConfigureMegamorphic() {
SetFeedback(*vector()->MegamorphicSentinel(GetIsolate()), SKIP_WRITE_BARRIER);
SetFeedbackExtra(*vector()->UninitializedSentinel(GetIsolate()),
SetFeedback(*TypeFeedbackVector::MegamorphicSentinel(GetIsolate()),
SKIP_WRITE_BARRIER);
SetFeedbackExtra(*TypeFeedbackVector::UninitializedSentinel(GetIsolate()),
SKIP_WRITE_BARRIER);
}
void LoadICNexus::ConfigurePremonomorphic() {
SetFeedback(*vector()->PremonomorphicSentinel(GetIsolate()),
SetFeedback(*TypeFeedbackVector::PremonomorphicSentinel(GetIsolate()),
SKIP_WRITE_BARRIER);
SetFeedbackExtra(*vector()->UninitializedSentinel(GetIsolate()),
SetFeedbackExtra(*TypeFeedbackVector::UninitializedSentinel(GetIsolate()),
SKIP_WRITE_BARRIER);
}
void KeyedLoadICNexus::ConfigurePremonomorphic() {
Isolate* isolate = GetIsolate();
SetFeedback(*vector()->PremonomorphicSentinel(isolate), SKIP_WRITE_BARRIER);
SetFeedbackExtra(*vector()->UninitializedSentinel(isolate),
SetFeedback(*TypeFeedbackVector::PremonomorphicSentinel(isolate),
SKIP_WRITE_BARRIER);
SetFeedbackExtra(*TypeFeedbackVector::UninitializedSentinel(isolate),
SKIP_WRITE_BARRIER);
}
......@@ -416,7 +437,7 @@ void LoadICNexus::ConfigurePolymorphic(MapHandleList* maps,
int receiver_count = maps->length();
Handle<FixedArray> array = EnsureArrayOfSize(receiver_count * 2);
InstallHandlers(array, maps, handlers);
SetFeedbackExtra(*vector()->UninitializedSentinel(isolate),
SetFeedbackExtra(*TypeFeedbackVector::UninitializedSentinel(isolate),
SKIP_WRITE_BARRIER);
}
......@@ -429,7 +450,7 @@ void KeyedLoadICNexus::ConfigurePolymorphic(Handle<Name> name,
Handle<FixedArray> array;
if (name.is_null()) {
array = EnsureArrayOfSize(receiver_count * 2);
SetFeedbackExtra(*vector()->UninitializedSentinel(GetIsolate()),
SetFeedbackExtra(*TypeFeedbackVector::UninitializedSentinel(GetIsolate()),
SKIP_WRITE_BARRIER);
} else {
SetFeedback(*name);
......
......@@ -214,11 +214,6 @@ class TypeFeedbackVector : public FixedArray {
// The object that indicates a premonomorphic state.
static inline Handle<Object> PremonomorphicSentinel(Isolate* isolate);
// The object that indicates a monomorphic state of Array with
// ElementsKind
static inline Handle<Object> MonomorphicArraySentinel(
Isolate* isolate, ElementsKind elements_kind);
// A raw version of the uninitialized sentinel that's safe to read during
// garbage collection (e.g., for patching the cache).
static inline Object* RawUninitializedSentinel(Heap* heap);
......@@ -339,6 +334,10 @@ class FeedbackNexus {
class CallICNexus : public FeedbackNexus {
public:
// Monomorphic call ics store call counts. Platform code needs to increment
// the count appropriately (ie, by 2).
static const int kCallCountIncrement = 2;
CallICNexus(Handle<TypeFeedbackVector> vector, FeedbackVectorICSlot slot)
: FeedbackNexus(vector, slot) {
DCHECK(vector->GetKind(slot) == Code::CALL_IC);
......@@ -368,6 +367,8 @@ class CallICNexus : public FeedbackNexus {
int length = -1) const override {
return length == 0;
}
int ExtractCallCount();
};
......
......@@ -2130,6 +2130,11 @@ void CallIC_ArrayStub::Generate(MacroAssembler* masm) {
factory->allocation_site_map());
__ j(not_equal, &miss);
// Increment the call count for monomorphic function calls.
__ SmiAddConstant(FieldOperand(rbx, rdx, times_pointer_size,
FixedArray::kHeaderSize + kPointerSize),
Smi::FromInt(CallICNexus::kCallCountIncrement));
__ movp(rbx, rcx);
__ movp(rdx, rdi);
ArrayConstructorStub stub(masm->isolate(), arg_count());
......@@ -2191,6 +2196,11 @@ void CallICStub::Generate(MacroAssembler* masm) {
// convincing us that we have a monomorphic JSFunction.
__ JumpIfSmi(rdi, &extra_checks_or_miss);
// Increment the call count for monomorphic function calls.
__ SmiAddConstant(FieldOperand(rbx, rdx, times_pointer_size,
FixedArray::kHeaderSize + kPointerSize),
Smi::FromInt(CallICNexus::kCallCountIncrement));
__ bind(&have_js_function);
if (CallAsMethod()) {
EmitContinueIfStrictOrNative(masm, &cont);
......@@ -2261,6 +2271,11 @@ void CallICStub::Generate(MacroAssembler* masm) {
// Update stats.
__ SmiAddConstant(FieldOperand(rbx, with_types_offset), Smi::FromInt(1));
// Initialize the call counter.
__ Move(FieldOperand(rbx, rdx, times_pointer_size,
FixedArray::kHeaderSize + kPointerSize),
Smi::FromInt(CallICNexus::kCallCountIncrement));
// Store the function. Use a stub since we need a frame for allocation.
// rbx - vector
// rdx - slot (needs to be in smi form)
......
// Copyright 2015 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Flags: --allow-natives-syntax --noalways-opt
// Locations in the type feedback vector where call counts are maintained for
// the two calls made from bar();
const kFooCallExtraIndex = 5;
const kArrayCallExtraIndex = 9;
function GetCallCount(func, slot) {
var vector = %GetTypeFeedbackVector(func);
// Call counts are recorded doubled.
var value = %FixedArrayGet(vector, slot);
return Math.floor(value / 2);
}
function foo(a) { return a[3] * 16; }
function bar(a) {
var result = 0;
for (var i = 0; i < 10; i++) {
result = foo(a);
if (i % 2 === 0) {
var r = Array();
r[0] = 1;
result += r[0];
}
}
return result;
}
var a = [1, 2, 3];
bar(a);
assertEquals(10, GetCallCount(bar, kFooCallExtraIndex));
assertEquals(5, GetCallCount(bar, kArrayCallExtraIndex));
%OptimizeFunctionOnNextCall(bar);
bar(a);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment