[KeyedLoadIC] Support Smi "handlers" for element loads

This is an experiment as far as performance is concerned. If Smi-configured
element loading directly from the dispatcher stub is fast enough, then we
can stop compiling LoadFastElementStubs (and drop the corresponding code).

Review-Url: https://codereview.chromium.org/2180273002
Cr-Commit-Position: refs/heads/master@{#38377}

src/code-stub-assembler.cc

@@ -6,6 +6,7 @@
 #include "src/code-factory.h"
 #include "src/frames-inl.h"
 #include "src/frames.h"
+#include "src/ic/handler-configuration.h"
 #include "src/ic/stub-cache.h"
 
 namespace v8 {
@@ -3425,31 +3426,261 @@ void CodeStubAssembler::TryProbeStubCache(
   }
 }
 
-void CodeStubAssembler::HandleLoadICHandlerCase(const LoadICParameters* p,
-                                                Node* handler, Label* miss) {
+// |is_jsarray| should be non-zero for JSArrays.
+void CodeStubAssembler::EmitBoundsCheck(Node* object, Node* elements,
+                                        Node* intptr_key, Node* is_jsarray,
+                                        Label* miss) {
+  Variable var_length(this, MachineRepresentation::kTagged);
+  Label if_array(this), length_loaded(this, &var_length);
+  GotoUnless(WordEqual(is_jsarray, IntPtrConstant(0)), &if_array);
+  {
+    var_length.Bind(SmiUntag(LoadFixedArrayBaseLength(elements)));
+    Goto(&length_loaded);
+  }
+  Bind(&if_array);
+  {
+    var_length.Bind(SmiUntag(LoadObjectField(object, JSArray::kLengthOffset)));
+    Goto(&length_loaded);
+  }
+  Bind(&length_loaded);
+  GotoUnless(UintPtrLessThan(intptr_key, var_length.value()), miss);
+}
+
+// |key| should be untagged (int32).
+void CodeStubAssembler::EmitElementLoad(Node* object, Node* elements,
+                                        Node* elements_kind, Node* key,
+                                        Label* if_hole, Label* rebox_double,
+                                        Variable* var_double_value,
+                                        Label* miss) {
+  Label if_typed_array(this), if_fast_packed(this), if_fast_holey(this),
+      if_fast_double(this), if_fast_holey_double(this),
+      unimplemented_elements_kind(this);
+  STATIC_ASSERT(LAST_ELEMENTS_KIND == LAST_FIXED_TYPED_ARRAY_ELEMENTS_KIND);
+  GotoIf(
+      IntPtrGreaterThanOrEqual(
+          elements_kind, IntPtrConstant(FIRST_FIXED_TYPED_ARRAY_ELEMENTS_KIND)),
+      &if_typed_array);
+
+  int32_t kinds[] = {// Handled by if_fast_packed.
+                     FAST_SMI_ELEMENTS, FAST_ELEMENTS,
+                     // Handled by if_fast_holey.
+                     FAST_HOLEY_SMI_ELEMENTS, FAST_HOLEY_ELEMENTS,
+                     // Handled by if_fast_double.
+                     FAST_DOUBLE_ELEMENTS,
+                     // Handled by if_fast_holey_double.
+                     FAST_HOLEY_DOUBLE_ELEMENTS};
+  Label* labels[] = {// FAST_{SMI,}_ELEMENTS
+                     &if_fast_packed, &if_fast_packed,
+                     // FAST_HOLEY_{SMI,}_ELEMENTS
+                     &if_fast_holey, &if_fast_holey,
+                     // FAST_DOUBLE_ELEMENTS
+                     &if_fast_double,
+                     // FAST_HOLEY_DOUBLE_ELEMENTS
+                     &if_fast_holey_double};
+  Switch(elements_kind, &unimplemented_elements_kind, kinds, labels,
+         arraysize(kinds));
+  Bind(&unimplemented_elements_kind);
+  {
+    // Crash if we get here.
+    DebugBreak();
+    Goto(miss);
+  }
+
+  Bind(&if_fast_packed);
+  {
+    Comment("fast packed elements");
+    // TODO(jkummerow): The Load*Element helpers add movsxlq instructions
+    // on x64 which we don't need here, because |key| is an IntPtr already.
+    // Do something about that.
+    Return(LoadFixedArrayElement(elements, key));
+  }
+
+  Bind(&if_fast_holey);
+  {
+    Comment("fast holey elements");
+    Node* element = LoadFixedArrayElement(elements, key);
+    GotoIf(WordEqual(element, TheHoleConstant()), if_hole);
+    Return(element);
+  }
+
+  Bind(&if_fast_double);
+  {
+    Comment("packed double elements");
+    var_double_value->Bind(
+        LoadFixedDoubleArrayElement(elements, key, MachineType::Float64()));
+    Goto(rebox_double);
+  }
+
+  Bind(&if_fast_holey_double);
+  {
+    Comment("holey double elements");
+    if (kPointerSize == kDoubleSize) {
+      Node* raw_element =
+          LoadFixedDoubleArrayElement(elements, key, MachineType::Uint64());
+      Node* the_hole = Int64Constant(kHoleNanInt64);
+      GotoIf(Word64Equal(raw_element, the_hole), if_hole);
+    } else {
+      Node* element_upper = LoadFixedDoubleArrayElement(
+          elements, key, MachineType::Uint32(), kIeeeDoubleExponentWordOffset);
+      GotoIf(Word32Equal(element_upper, Int32Constant(kHoleNanUpper32)),
+             if_hole);
+    }
+    var_double_value->Bind(
+        LoadFixedDoubleArrayElement(elements, key, MachineType::Float64()));
+    Goto(rebox_double);
+  }
+
+  Bind(&if_typed_array);
+  {
+    Comment("typed elements");
+    // Check if buffer has been neutered.
+    Node* buffer = LoadObjectField(object, JSArrayBufferView::kBufferOffset);
+    Node* bitfield = LoadObjectField(buffer, JSArrayBuffer::kBitFieldOffset,
+                                     MachineType::Uint32());
+    Node* neutered_bit =
+        Word32And(bitfield, Int32Constant(JSArrayBuffer::WasNeutered::kMask));
+    GotoUnless(Word32Equal(neutered_bit, Int32Constant(0)), miss);
+    // Backing store = external_pointer + base_pointer.
+    Node* external_pointer =
+        LoadObjectField(elements, FixedTypedArrayBase::kExternalPointerOffset,
+                        MachineType::Pointer());
+    Node* base_pointer =
+        LoadObjectField(elements, FixedTypedArrayBase::kBasePointerOffset);
+    Node* backing_store = IntPtrAdd(external_pointer, base_pointer);
+
+    const int kTypedElementsKindCount = LAST_FIXED_TYPED_ARRAY_ELEMENTS_KIND -
+                                        FIRST_FIXED_TYPED_ARRAY_ELEMENTS_KIND +
+                                        1;
+    Label* elements_kind_labels[kTypedElementsKindCount];
+    int32_t elements_kinds[kTypedElementsKindCount];
+    for (int i = 0; i < kTypedElementsKindCount; i++) {
+      elements_kinds[i] = i + FIRST_FIXED_TYPED_ARRAY_ELEMENTS_KIND;
+      elements_kind_labels[i] = new Label(this);
+    }
+    Switch(elements_kind, miss, elements_kinds, elements_kind_labels,
+           static_cast<size_t>(kTypedElementsKindCount));
+
+    for (int i = 0; i < kTypedElementsKindCount; i++) {
+      ElementsKind kind = static_cast<ElementsKind>(elements_kinds[i]);
+      Bind(elements_kind_labels[i]);
+      Comment(ElementsKindToString(kind));
+      switch (kind) {
+        case UINT8_ELEMENTS:
+        case UINT8_CLAMPED_ELEMENTS:
+          Return(SmiTag(Load(MachineType::Uint8(), backing_store, key)));
+          break;
+        case INT8_ELEMENTS:
+          Return(SmiTag(Load(MachineType::Int8(), backing_store, key)));
+          break;
+        case UINT16_ELEMENTS: {
+          Node* index = WordShl(key, IntPtrConstant(1));
+          Return(SmiTag(Load(MachineType::Uint16(), backing_store, index)));
+          break;
+        }
+        case INT16_ELEMENTS: {
+          Node* index = WordShl(key, IntPtrConstant(1));
+          Return(SmiTag(Load(MachineType::Int16(), backing_store, index)));
+          break;
+        }
+        case UINT32_ELEMENTS: {
+          Node* index = WordShl(key, IntPtrConstant(2));
+          Node* element = Load(MachineType::Uint32(), backing_store, index);
+          Return(ChangeUint32ToTagged(element));
+          break;
+        }
+        case INT32_ELEMENTS: {
+          Node* index = WordShl(key, IntPtrConstant(2));
+          Node* element = Load(MachineType::Int32(), backing_store, index);
+          Return(ChangeInt32ToTagged(element));
+          break;
+        }
+        case FLOAT32_ELEMENTS: {
+          Node* index = WordShl(key, IntPtrConstant(2));
+          Node* element = Load(MachineType::Float32(), backing_store, index);
+          var_double_value->Bind(ChangeFloat32ToFloat64(element));
+          Goto(rebox_double);
+          break;
+        }
+        case FLOAT64_ELEMENTS: {
+          Node* index = WordShl(key, IntPtrConstant(3));
+          Node* element = Load(MachineType::Float64(), backing_store, index);
+          var_double_value->Bind(element);
+          Goto(rebox_double);
+          break;
+        }
+        default:
+          UNREACHABLE();
+      }
+      // Don't forget to clean up.
+      delete elements_kind_labels[i];
+    }
+  }
+}
+
+void CodeStubAssembler::HandleLoadICHandlerCase(
+    const LoadICParameters* p, Node* handler, Label* miss,
+    ElementSupport support_elements) {
   Comment("have_handler");
   Label call_handler(this);
   GotoUnless(WordIsSmi(handler), &call_handler);
 
-  // |handler| is a Smi. It encodes a field index as obtained by
-  // FieldIndex.GetLoadByFieldOffset().
-  // TODO(jkummerow): For KeyedLoadICs, extend this scheme to encode
-  // fast *element* loads.
+  // |handler| is a Smi, encoding what to do. See handler-configuration.h
+  // for the encoding format.
   {
     Variable var_double_value(this, MachineRepresentation::kFloat64);
     Label rebox_double(this, &var_double_value);
 
     Node* handler_word = SmiUntag(handler);
+    if (support_elements == kSupportElements) {
+      Label property(this);
+      Node* handler_type =
+          WordAnd(handler_word, IntPtrConstant(LoadHandlerTypeBit::kMask));
+      GotoUnless(
+          WordEqual(handler_type, IntPtrConstant(kLoadICHandlerForElements)),
+          &property);
+
+      Comment("element_load");
+      GotoUnless(WordIsSmi(p->name), miss);
+      Node* key = SmiUntag(p->name);
+      Node* elements = LoadElements(p->receiver);
+      Node* is_jsarray =
+          WordAnd(handler_word, IntPtrConstant(KeyedLoadIsJsArray::kMask));
+      EmitBoundsCheck(p->receiver, elements, key, is_jsarray, miss);
+      Label if_hole(this);
+
+      Node* elements_kind = BitFieldDecode<KeyedLoadElementsKind>(handler_word);
+
+      EmitElementLoad(p->receiver, elements, elements_kind, key, &if_hole,
+                      &rebox_double, &var_double_value, miss);
+
+      Bind(&if_hole);
+      {
+        Comment("convert hole");
+        Node* convert_hole =
+            WordAnd(handler_word, IntPtrConstant(KeyedLoadConvertHole::kMask));
+        GotoIf(WordEqual(convert_hole, IntPtrConstant(0)), miss);
+        Node* protector_cell = LoadRoot(Heap::kArrayProtectorRootIndex);
+        GotoUnless(
+            WordEqual(LoadObjectField(protector_cell, Cell::kValueOffset),
+                      SmiConstant(Smi::FromInt(Isolate::kArrayProtectorValid))),
+            miss);
+        Return(UndefinedConstant());
+      }
+
+      Bind(&property);
+      Comment("property_load");
+    }
+
     // |handler_word| is a field index as obtained by
     // FieldIndex.GetLoadByFieldOffset():
     Label inobject_double(this), out_of_object(this),
         out_of_object_double(this);
-    Node* inobject_bit = WordAnd(
-        handler_word, IntPtrConstant(FieldIndex::FieldOffsetIsInobject::kMask));
-    Node* double_bit = WordAnd(
-        handler_word, IntPtrConstant(FieldIndex::FieldOffsetIsDouble::kMask));
-    Node* offset = WordSar(
-        handler_word, IntPtrConstant(FieldIndex::FieldOffsetOffset::kShift));
+    Node* inobject_bit =
+        WordAnd(handler_word, IntPtrConstant(FieldOffsetIsInobject::kMask));
+    Node* double_bit =
+        WordAnd(handler_word, IntPtrConstant(FieldOffsetIsDouble::kMask));
+    Node* offset =
+        WordSar(handler_word, IntPtrConstant(FieldOffsetOffset::kShift));
 
     GotoIf(WordEqual(inobject_bit, IntPtrConstant(0)), &out_of_object);
 
@@ -3550,7 +3781,7 @@ void CodeStubAssembler::KeyedLoadIC(const LoadICParameters* p) {
                                       &var_handler, &try_polymorphic);
   Bind(&if_handler);
   {
-    HandleLoadICHandlerCase(p, var_handler.value(), &miss);
+    HandleLoadICHandlerCase(p, var_handler.value(), &miss, kSupportElements);
   }
 
   Bind(&try_polymorphic);

src/code-stub-assembler.h

@@ -509,9 +509,6 @@ class CodeStubAssembler : public compiler::CodeAssembler {
                              Variable* var_handler, Label* if_miss,
                              int unroll_count);
 
-  void HandleLoadICHandlerCase(const LoadICParameters* p,
-                               compiler::Node* handler, Label* miss);
-
   compiler::Node* StubCachePrimaryOffset(compiler::Node* name,
                                          compiler::Node* map);
 
@@ -552,6 +549,19 @@ class CodeStubAssembler : public compiler::CodeAssembler {
       compiler::Node* value);
 
  private:
+  enum ElementSupport { kOnlyProperties, kSupportElements };
+
+  void HandleLoadICHandlerCase(
+      const LoadICParameters* p, compiler::Node* handler, Label* miss,
+      ElementSupport support_elements = kOnlyProperties);
+  void EmitBoundsCheck(compiler::Node* object, compiler::Node* elements,
+                       compiler::Node* intptr_key, compiler::Node* is_jsarray,
+                       Label* miss);
+  void EmitElementLoad(compiler::Node* object, compiler::Node* elements,
+                       compiler::Node* elements_kind, compiler::Node* key,
+                       Label* if_hole, Label* rebox_double,
+                       Variable* var_double_value, Label* miss);
+
   compiler::Node* ElementOffsetFromIndex(compiler::Node* index,
                                          ElementsKind kind, ParameterMode mode,
                                          int base_size = 0);

src/field-index-inl.h

@@ -6,6 +6,7 @@
 #define V8_FIELD_INDEX_INL_H_
 
 #include "src/field-index.h"
+#include "src/ic/handler-configuration.h"
 
 namespace v8 {
 namespace internal {
@@ -88,7 +89,7 @@ inline int FieldIndex::GetLoadByFieldIndex() const {
 // FieldIndex object from it.
 // static
 inline FieldIndex FieldIndex::ForLoadByFieldOffset(Map* map, int offset) {
-  DCHECK(offset & 1);  // Property marker (as opposed to element).
+  DCHECK(LoadHandlerTypeBit::decode(offset) == kLoadICHandlerForProperties);
   bool is_inobject = FieldOffsetIsInobject::decode(offset);
   bool is_double = FieldOffsetIsDouble::decode(offset);
   int field_index = FieldOffsetOffset::decode(offset) >> kPointerSizeLog2;
@@ -114,7 +115,7 @@ inline int FieldIndex::GetLoadByFieldOffset() const {
   return FieldOffsetIsInobject::encode(is_inobject()) |
          FieldOffsetIsDouble::encode(is_double()) |
          FieldOffsetOffset::encode(index() << kPointerSizeLog2) |
-         1;  // Property marker (as opposed to element).
+         LoadHandlerTypeBit::encode(kLoadICHandlerForProperties);
 }
 
 inline FieldIndex FieldIndex::ForDescriptor(Map* map, int descriptor_index) {

src/field-index.h

@@ -81,13 +81,6 @@ class FieldIndex final {
   }
   bool operator!=(FieldIndex const& other) const { return !(*this == other); }
 
-  // For GetLoadByFieldOffset.
-  class FieldOffsetIsInobject : public BitField<bool, 1, 1> {};
-  class FieldOffsetIsDouble : public BitField<bool, 2, 1> {};
-  class FieldOffsetOffset : public BitField<int, 3, 27> {};
-  // Make sure we don't overflow into the sign bit.
-  STATIC_ASSERT(FieldOffsetOffset::kNext <= kSmiValueSize - 1);
-
  private:
   FieldIndex(bool is_inobject, int local_index, bool is_double,
              int inobject_properties, int first_inobject_property_offset,

src/ic/handler-compiler.cc

@@ -6,6 +6,7 @@
 
 #include "src/field-type.h"
 #include "src/ic/call-optimization.h"
+#include "src/ic/handler-configuration.h"
 #include "src/ic/ic-inl.h"
 #include "src/ic/ic.h"
 #include "src/isolate-inl.h"
@@ -577,7 +578,7 @@ Handle<Code> NamedStoreHandlerCompiler::CompileStoreCallback(
 #undef __
 
 // static
-Handle<Code> ElementHandlerCompiler::GetKeyedLoadHandler(
+Handle<Object> ElementHandlerCompiler::GetKeyedLoadHandler(
     Handle<Map> receiver_map, Isolate* isolate) {
   if (receiver_map->has_indexed_interceptor() &&
       !receiver_map->GetIndexedInterceptor()->getter()->IsUndefined(isolate) &&
@@ -610,10 +611,18 @@ Handle<Code> ElementHandlerCompiler::GetKeyedLoadHandler(
   bool convert_hole_to_undefined =
       is_js_array && elements_kind == FAST_HOLEY_ELEMENTS &&
       *receiver_map == isolate->get_initial_js_array_map(elements_kind);
+  if (FLAG_tf_load_ic_stub) {
+    int config = KeyedLoadElementsKind::encode(elements_kind) |
+                 KeyedLoadConvertHole::encode(convert_hole_to_undefined) |
+                 KeyedLoadIsJsArray::encode(is_js_array) |
+                 LoadHandlerTypeBit::encode(kLoadICHandlerForElements);
+    return handle(Smi::FromInt(config), isolate);
+  } else {
     TRACE_HANDLER_STATS(isolate, KeyedLoadIC_LoadFastElementStub);
     return LoadFastElementStub(isolate, is_js_array, elements_kind,
                                convert_hole_to_undefined)
         .GetCode();
+  }
 }
 
 void ElementHandlerCompiler::CompileElementHandlers(

src/ic/handler-compiler.h

@@ -279,7 +279,7 @@ class ElementHandlerCompiler : public PropertyHandlerCompiler {
 
   virtual ~ElementHandlerCompiler() {}
 
-  static Handle<Code> GetKeyedLoadHandler(Handle<Map> receiver_map,
+  static Handle<Object> GetKeyedLoadHandler(Handle<Map> receiver_map,
                                             Isolate* isolate);
   void CompileElementHandlers(MapHandleList* receiver_maps,
                               List<Handle<Object>>* handlers);

src/ic/handler-configuration.h

+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_IC_HANDLER_CONFIGURATION_H_
+#define V8_IC_HANDLER_CONFIGURATION_H_
+
+#include "src/elements-kind.h"
+#include "src/globals.h"
+#include "src/utils.h"
+
+namespace v8 {
+namespace internal {
+
+enum LoadHandlerType {
+  kLoadICHandlerForElements = 0,
+  kLoadICHandlerForProperties = 1
+};
+
+class LoadHandlerTypeBit : public BitField<bool, 0, 1> {};
+
+// Encoding for configuration Smis for property loads:
+class FieldOffsetIsInobject
+    : public BitField<bool, LoadHandlerTypeBit::kNext, 1> {};
+class FieldOffsetIsDouble
+    : public BitField<bool, FieldOffsetIsInobject::kNext, 1> {};
+class FieldOffsetOffset : public BitField<int, FieldOffsetIsDouble::kNext, 27> {
+};
+// Make sure we don't overflow into the sign bit.
+STATIC_ASSERT(FieldOffsetOffset::kNext <= kSmiValueSize - 1);
+
+// Encoding for configuration Smis for elements loads:
+class KeyedLoadIsJsArray : public BitField<bool, LoadHandlerTypeBit::kNext, 1> {
+};
+class KeyedLoadConvertHole
+    : public BitField<bool, KeyedLoadIsJsArray::kNext, 1> {};
+class KeyedLoadElementsKind
+    : public BitField<ElementsKind, KeyedLoadConvertHole::kNext, 8> {};
+// Make sure we don't overflow into the sign bit.
+STATIC_ASSERT(KeyedLoadElementsKind::kNext <= kSmiValueSize - 1);
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_IC_HANDLER_CONFIGURATION_H_

src/ic/ic.cc

@@ -1328,7 +1328,7 @@ void KeyedLoadIC::UpdateLoadElement(Handle<HeapObject> receiver) {
   TargetMaps(&target_receiver_maps);
 
   if (target_receiver_maps.length() == 0) {
-    Handle<Code> handler =
+    Handle<Object> handler =
         ElementHandlerCompiler::GetKeyedLoadHandler(receiver_map, isolate());
     return ConfigureVectorState(Handle<Name>(), receiver_map, handler);
   }
@@ -1357,7 +1357,7 @@ void KeyedLoadIC::UpdateLoadElement(Handle<HeapObject> receiver) {
       IsMoreGeneralElementsKindTransition(
           target_receiver_maps.at(0)->elements_kind(),
           Handle<JSObject>::cast(receiver)->GetElementsKind())) {
-    Handle<Code> handler =
+    Handle<Object> handler =
         ElementHandlerCompiler::GetKeyedLoadHandler(receiver_map, isolate());
     return ConfigureVectorState(Handle<Name>(), receiver_map, handler);
   }