[builtins] TurboFan version of Array.prototype.forEach including fast path for FAST_ELEMENTS

BUG=v8:5269, v8:1956
LOG=N
R=bmeurer@chromium.org

Review-Url: https://codereview.chromium.org/2663033003
Cr-Commit-Position: refs/heads/master@{#42926}

src/bootstrapper.cc

@@ -217,6 +217,8 @@ class Genesis BASE_EMBEDDED {
   HARMONY_SHIPPING(DECLARE_FEATURE_INITIALIZATION)
 #undef DECLARE_FEATURE_INITIALIZATION
 
+  void InitializeGlobal_enable_fast_array_builtins();
+
   Handle<JSFunction> InstallArrayBuffer(Handle<JSObject> target,
                                         const char* name, Builtins::Name call,
                                         BuiltinFunctionId id);
@@ -2911,6 +2913,8 @@ void Genesis::InitializeExperimentalGlobal() {
   HARMONY_STAGED(FEATURE_INITIALIZE_GLOBAL)
   HARMONY_SHIPPING(FEATURE_INITIALIZE_GLOBAL)
 #undef FEATURE_INITIALIZE_GLOBAL
+
+  InitializeGlobal_enable_fast_array_builtins();
 }
 
 
@@ -3541,6 +3545,28 @@ void InstallPublicSymbol(Factory* factory, Handle<Context> native_context,
   JSObject::AddProperty(symbol, name_string, value, attributes);
 }
 
+void Genesis::InitializeGlobal_enable_fast_array_builtins() {
+  if (!FLAG_enable_fast_array_builtins) return;
+
+  Handle<JSGlobalObject> global(native_context()->global_object());
+  Isolate* isolate = global->GetIsolate();
+  Factory* factory = isolate->factory();
+
+  LookupIterator it1(global, factory->NewStringFromAsciiChecked("Array"),
+                     LookupIterator::OWN_SKIP_INTERCEPTOR);
+  Handle<Object> array_object = Object::GetProperty(&it1).ToHandleChecked();
+  LookupIterator it2(array_object,
+                     factory->NewStringFromAsciiChecked("prototype"),
+                     LookupIterator::OWN_SKIP_INTERCEPTOR);
+  Handle<Object> array_prototype = Object::GetProperty(&it2).ToHandleChecked();
+  LookupIterator it3(array_prototype,
+                     factory->NewStringFromAsciiChecked("forEach"),
+                     LookupIterator::OWN_SKIP_INTERCEPTOR);
+  Handle<Object> for_each_function =
+      Object::GetProperty(&it3).ToHandleChecked();
+  Handle<JSFunction>::cast(for_each_function)
+      ->set_code(isolate->builtins()->builtin(Builtins::kArrayForEach));
+}
 
 void Genesis::InitializeGlobal_harmony_sharedarraybuffer() {
   if (!FLAG_harmony_sharedarraybuffer) return;

src/builtins/builtins-array.cc

@@ -426,6 +426,242 @@ BUILTIN(ArrayUnshift) {
   return Smi::FromInt(new_length);
 }
 
+class ForEachCodeStubAssembler : public CodeStubAssembler {
+ public:
+  explicit ForEachCodeStubAssembler(compiler::CodeAssemblerState* state)
+      : CodeStubAssembler(state) {}
+
+  void VisitOneElement(Node* context, Node* this_arg, Node* o, Node* k,
+                       Node* callbackfn) {
+    Comment("begin VisitOneElement");
+
+    // a. Let Pk be ToString(k).
+    Node* p_k = ToString(context, k);
+
+    // b. Let kPresent be HasProperty(O, Pk).
+    // c. ReturnIfAbrupt(kPresent).
+    Node* k_present =
+        CallStub(CodeFactory::HasProperty(isolate()), context, p_k, o);
+
+    // d. If kPresent is true, then
+    Label not_present(this);
+    GotoIf(WordNotEqual(k_present, TrueConstant()), &not_present);
+
+    // i. Let kValue be Get(O, Pk).
+    // ii. ReturnIfAbrupt(kValue).
+    Node* k_value =
+        CallStub(CodeFactory::GetProperty(isolate()), context, o, k);
+
+    // iii. Let funcResult be Call(callbackfn, T, «kValue, k, O»).
+    // iv. ReturnIfAbrupt(funcResult).
+    CallJS(CodeFactory::Call(isolate()), context, callbackfn, this_arg, k_value,
+           k, o);
+
+    Goto(&not_present);
+    Bind(&not_present);
+    Comment("end VisitOneElement");
+  }
+
+  void VisitAllFastElements(Node* context, ElementsKind kind, Node* this_arg,
+                            Node* o, Node* len, Node* callbackfn,
+                            ParameterMode mode) {
+    Comment("begin VisitAllFastElements");
+    Variable original_map(this, MachineRepresentation::kTagged);
+    original_map.Bind(LoadMap(o));
+    VariableList list({&original_map}, zone());
+    BuildFastLoop(
+        list, IntPtrOrSmiConstant(0, mode), TaggedToParameter(len, mode),
+        [context, kind, this, o, &original_map, callbackfn, this_arg,
+         mode](Node* index) {
+          Label one_element_done(this), array_changed(this, Label::kDeferred),
+              hole_element(this);
+
+          // Check if o's map has changed during the callback. If so, we have to
+          // fall back to the slower spec implementation for the rest of the
+          // iteration.
+          Node* o_map = LoadMap(o);
+          GotoIf(WordNotEqual(o_map, original_map.value()), &array_changed);
+
+          // Check if o's length has changed during the callback and if the
+          // index is now out of range of the new length.
+          Node* tagged_index = ParameterToTagged(index, mode);
+          GotoIf(SmiGreaterThanOrEqual(tagged_index, LoadJSArrayLength(o)),
+                 &array_changed);
+
+          // Re-load the elements array. If may have been resized.
+          Node* elements = LoadElements(o);
+
+          // Fast case: load the element directly from the elements FixedArray
+          // and call the callback if the element is not the hole.
+          DCHECK(kind == FAST_ELEMENTS || kind == FAST_DOUBLE_ELEMENTS);
+          int base_size = kind == FAST_ELEMENTS
+                              ? FixedArray::kHeaderSize
+                              : (FixedArray::kHeaderSize - kHeapObjectTag);
+          Node* offset = ElementOffsetFromIndex(index, kind, mode, base_size);
+          Node* value = nullptr;
+          if (kind == FAST_ELEMENTS) {
+            value = LoadObjectField(elements, offset);
+            GotoIf(WordEqual(value, TheHoleConstant()), &hole_element);
+          } else {
+            Node* double_value =
+                LoadDoubleWithHoleCheck(elements, offset, &hole_element);
+            value = AllocateHeapNumberWithValue(double_value);
+          }
+          CallJS(CodeFactory::Call(isolate()), context, callbackfn, this_arg,
+                 value, tagged_index, o);
+          Goto(&one_element_done);
+
+          Bind(&hole_element);
+          BranchIfPrototypesHaveNoElements(o_map, &one_element_done,
+                                           &array_changed);
+
+          // O's changed during the forEach. Use the implementation precisely
+          // specified in the spec for the rest of the iteration, also making
+          // the failed original_map sticky in case of a subseuent change that
+          // goes back to the original map.
+          Bind(&array_changed);
+          VisitOneElement(context, this_arg, o, ParameterToTagged(index, mode),
+                          callbackfn);
+          original_map.Bind(UndefinedConstant());
+          Goto(&one_element_done);
+
+          Bind(&one_element_done);
+        },
+        1, mode, IndexAdvanceMode::kPost);
+    Comment("end VisitAllFastElements");
+  }
+};
+
+TF_BUILTIN(ArrayForEach, ForEachCodeStubAssembler) {
+  Label non_array(this), examine_elements(this), fast_elements(this),
+      slow(this), maybe_double_elements(this), fast_double_elements(this);
+
+  Node* receiver = Parameter(ForEachDescriptor::kReceiver);
+  Node* callbackfn = Parameter(ForEachDescriptor::kCallback);
+  Node* this_arg = Parameter(ForEachDescriptor::kThisArg);
+  Node* context = Parameter(ForEachDescriptor::kContext);
+
+  // TODO(danno): Seriously? Do we really need to throw the exact error message
+  // on null and undefined so that the webkit tests pass?
+  Label throw_null_undefined_exception(this, Label::kDeferred);
+  GotoIf(WordEqual(receiver, NullConstant()), &throw_null_undefined_exception);
+  GotoIf(WordEqual(receiver, UndefinedConstant()),
+         &throw_null_undefined_exception);
+
+  // By the book: taken directly from the ECMAScript 2015 specification
+
+  // 1. Let O be ToObject(this value).
+  // 2. ReturnIfAbrupt(O)
+  Node* o = CallStub(CodeFactory::ToObject(isolate()), context, receiver);
+
+  // 3. Let len be ToLength(Get(O, "length")).
+  // 4. ReturnIfAbrupt(len).
+  Variable merged_length(this, MachineRepresentation::kTagged);
+  Label has_length(this, &merged_length), not_js_array(this);
+  GotoIf(DoesntHaveInstanceType(o, JS_ARRAY_TYPE), &not_js_array);
+  merged_length.Bind(LoadJSArrayLength(o));
+  Goto(&has_length);
+  Bind(&not_js_array);
+  Node* len_property =
+      CallStub(CodeFactory::GetProperty(isolate()), context, o,
+               HeapConstant(isolate()->factory()->length_string()));
+  merged_length.Bind(
+      CallStub(CodeFactory::ToLength(isolate()), context, len_property));
+  Goto(&has_length);
+  Bind(&has_length);
+  Node* len = merged_length.value();
+
+  // 5. If IsCallable(callbackfn) is false, throw a TypeError exception.
+  Label type_exception(this, Label::kDeferred);
+  GotoIf(TaggedIsSmi(callbackfn), &type_exception);
+  GotoUnless(IsCallableMap(LoadMap(callbackfn)), &type_exception);
+
+  // 6. If thisArg was supplied, let T be thisArg; else let T be undefined.
+  // [Already done by the arguments adapter]
+
+  // Non-smi lengths must use the slow path.
+  GotoIf(TaggedIsNotSmi(len), &slow);
+
+  BranchIfFastJSArray(o, context,
+                      CodeStubAssembler::FastJSArrayAccessMode::INBOUNDS_READ,
+                      &examine_elements, &slow);
+
+  Bind(&examine_elements);
+
+  ParameterMode mode = OptimalParameterMode();
+
+  // Select by ElementsKind
+  Node* o_map = LoadMap(o);
+  Node* bit_field2 = LoadMapBitField2(o_map);
+  Node* kind = DecodeWord32<Map::ElementsKindBits>(bit_field2);
+  Branch(Int32GreaterThan(kind, Int32Constant(FAST_HOLEY_ELEMENTS)),
+         &maybe_double_elements, &fast_elements);
+
+  Bind(&fast_elements);
+  {
+    VisitAllFastElements(context, FAST_ELEMENTS, this_arg, o, len, callbackfn,
+                         mode);
+
+    // No exception, return success
+    Return(UndefinedConstant());
+  }
+
+  Bind(&maybe_double_elements);
+  Branch(Int32GreaterThan(kind, Int32Constant(FAST_HOLEY_DOUBLE_ELEMENTS)),
+         &slow, &fast_double_elements);
+
+  Bind(&fast_double_elements);
+  {
+    VisitAllFastElements(context, FAST_DOUBLE_ELEMENTS, this_arg, o, len,
+                         callbackfn, mode);
+
+    // No exception, return success
+    Return(UndefinedConstant());
+  }
+
+  Bind(&slow);
+  {
+    // By the book: taken from the ECMAScript 2015 specification (cont.)
+
+    // 7. Let k be 0.
+    Variable k(this, MachineRepresentation::kTagged);
+    k.Bind(SmiConstant(0));
+
+    // 8. Repeat, while k < len
+    Label loop(this, &k);
+    Label after_loop(this);
+    Goto(&loop);
+    Bind(&loop);
+    {
+      GotoUnlessNumberLessThan(k.value(), len, &after_loop);
+
+      VisitOneElement(context, this_arg, o, k.value(), callbackfn);
+
+      // e. Increase k by 1.
+      k.Bind(NumberInc(k.value()));
+      Goto(&loop);
+    }
+    Bind(&after_loop);
+    Return(UndefinedConstant());
+  }
+
+  Bind(&throw_null_undefined_exception);
+  {
+    CallRuntime(Runtime::kThrowTypeError, context,
+                SmiConstant(MessageTemplate::kCalledOnNullOrUndefined),
+                HeapConstant(isolate()->factory()->NewStringFromAsciiChecked(
+                    "Array.prototype.forEach")));
+    Return(UndefinedConstant());
+  }
+
+  Bind(&type_exception);
+  {
+    CallRuntime(Runtime::kThrowTypeError, context,
+                SmiConstant(MessageTemplate::kCalledNonCallable), callbackfn);
+    Return(UndefinedConstant());
+  }
+}
+
 BUILTIN(ArraySlice) {
   HandleScope scope(isolate);
   Handle<Object> receiver = args.receiver();

src/builtins/builtins.h

@@ -58,6 +58,7 @@ class Isolate;
   ASM(CompileLazy)                                                             \
   TFS(ToObject, BUILTIN, kNoExtraICState, TypeConversion)                      \
   TFS(FastNewObject, BUILTIN, kNoExtraICState, FastNewObject)                  \
+  TFS(HasProperty, BUILTIN, kNoExtraICState, HasProperty)                      \
                                                                                \
   /* Calls */                                                                  \
   ASM(ArgumentsAdaptorTrampoline)                                              \
@@ -273,6 +274,7 @@ class Isolate;
   CPP(ArraySlice)                                                              \
   CPP(ArraySplice)                                                             \
   CPP(ArrayUnshift)                                                            \
+  TFJ(ArrayForEach, 2)                                                         \
   /* ES6 #sec-array.prototype.entries */                                       \
   TFJ(ArrayPrototypeEntries, 0)                                                \
   /* ES6 #sec-array.prototype.keys */                                          \
@@ -642,7 +644,6 @@ class Isolate;
   CPP(ObjectSeal)                                                              \
   CPP(ObjectValues)                                                            \
                                                                                \
-  TFS(HasProperty, BUILTIN, kNoExtraICState, HasProperty)                      \
   TFS(InstanceOf, BUILTIN, kNoExtraICState, Compare)                           \
   TFS(OrdinaryHasInstance, BUILTIN, kNoExtraICState, Compare)                  \
   TFS(ForInFilter, BUILTIN, kNoExtraICState, ForInFilter)                      \

src/flag-definitions.h

@@ -257,6 +257,7 @@ DEFINE_BOOL(future, false,
 DEFINE_IMPLICATION(future, turbo)
 
 DEFINE_IMPLICATION(turbo, ignition_staging)
+DEFINE_IMPLICATION(turbo, enable_fast_array_builtins)
 
 // TODO(rmcilroy): Remove ignition-staging and set these implications directly
 // with the turbo flag.
@@ -644,6 +645,7 @@ DEFINE_BOOL(builtins_in_stack_traces, false,
             "show built-in functions in stack traces")
 
 // builtins.cc
+DEFINE_BOOL(enable_fast_array_builtins, false, "use optimized builtins")
 DEFINE_BOOL(allow_unsafe_function_constructor, false,
             "allow invoking the function constructor without security checks")
 

src/interface-descriptors.h

@@ -69,6 +69,7 @@ class PlatformInterfaceDescriptor;
   V(AllocateBool8x16)                     \
   V(Builtin)                              \
   V(ArrayConstructor)                     \
+  V(ForEach)                              \
   V(ArrayNoArgumentConstructor)           \
   V(ArraySingleArgumentConstructor)       \
   V(ArrayNArgumentsConstructor)           \
@@ -294,6 +295,41 @@ class V8_EXPORT_PRIVATE CallInterfaceDescriptor {
     kContext = kParameterCount /* implicit parameter */ \
   };
 
+#define DECLARE_BUILTIN_DESCRIPTOR(name)                                \
+  DECLARE_DESCRIPTOR_WITH_BASE(name, BuiltinDescriptor)                 \
+ protected:                                                             \
+  void InitializePlatformIndependent(CallInterfaceDescriptorData* data) \
+      override {                                                        \
+    MachineType machine_types[] = {MachineType::AnyTagged(),            \
+                                   MachineType::AnyTagged(),            \
+                                   MachineType::Int32()};               \
+    int argc = kStackParameterCount + 1 - arraysize(machine_types);     \
+    data->InitializePlatformIndependent(arraysize(machine_types), argc, \
+                                        machine_types);                 \
+  }                                                                     \
+  void InitializePlatformSpecific(CallInterfaceDescriptorData* data)    \
+      override {                                                        \
+    Register registers[] = {TargetRegister(), NewTargetRegister(),      \
+                            ArgumentsCountRegister()};                  \
+    data->InitializePlatformSpecific(arraysize(registers), registers);  \
+  }                                                                     \
+                                                                        \
+ public:
+
+#define DEFINE_BUILTIN_PARAMETERS(...)                             \
+  enum ParameterIndices {                                          \
+    kReceiver,                                                     \
+    kBeforeFirstStackParameter = kReceiver,                        \
+    __VA_ARGS__,                                                   \
+    kAfterLastStackParameter,                                      \
+    kNewTarget = kAfterLastStackParameter,                         \
+    kArgumentsCount,                                               \
+    kContext, /* implicit parameter */                             \
+    kParameterCount = kContext,                                    \
+    kStackParameterCount =                                         \
+        kAfterLastStackParameter - kBeforeFirstStackParameter - 1, \
+  };
+
 class VoidDescriptor : public CallInterfaceDescriptor {
  public:
   DECLARE_DESCRIPTOR(VoidDescriptor, CallInterfaceDescriptor)
@@ -673,6 +709,12 @@ class BuiltinDescriptor : public CallInterfaceDescriptor {
   static const Register TargetRegister();
 };
 
+class ForEachDescriptor : public BuiltinDescriptor {
+ public:
+  DEFINE_BUILTIN_PARAMETERS(kCallback, kThisArg)
+  DECLARE_BUILTIN_DESCRIPTOR(ForEachDescriptor)
+};
+
 class ArrayConstructorDescriptor : public CallInterfaceDescriptor {
  public:
   DEFINE_PARAMETERS(kTarget, kNewTarget, kActualArgumentsCount, kAllocationSite)

src/js/array.js

@@ -1569,6 +1569,8 @@ utils.InstallFunctions(GlobalArray.prototype, DONT_ENUM, [
   iteratorSymbol, ArrayValues
 ]);
 
+utils.ForEachFunction = GlobalArray.prototype.forEach;
+
 %FunctionSetName(ArrayValues, "values");
 
 %FinishArrayPrototypeSetup(GlobalArray.prototype);

src/runtime/runtime-array.cc

@@ -37,7 +37,7 @@ static void InstallCode(
     BuiltinFunctionId id = static_cast<BuiltinFunctionId>(-1)) {
   Handle<String> key = isolate->factory()->InternalizeUtf8String(name);
   Handle<JSFunction> optimized =
-      isolate->factory()->NewFunctionWithoutPrototype(key, code);
+      isolate->factory()->NewFunctionWithoutPrototype(key, code, true);
   if (argc < 0) {
     optimized->shared()->DontAdaptArguments();
   } else {
@@ -46,6 +46,8 @@ static void InstallCode(
   if (id >= 0) {
     optimized->shared()->set_builtin_function_id(id);
   }
+  optimized->shared()->set_language_mode(STRICT);
+  optimized->shared()->set_native(true);
   JSObject::AddProperty(holder, key, optimized, NONE);
 }
 
@@ -78,11 +80,9 @@ RUNTIME_FUNCTION(Runtime_SpecialArrayFunctions) {
                  kArrayValues);
   InstallBuiltin(isolate, holder, "entries", Builtins::kArrayPrototypeEntries,
                  0, kArrayEntries);
-
   return *holder;
 }
 
-
 RUNTIME_FUNCTION(Runtime_FixedArrayGet) {
   SealHandleScope shs(isolate);
   DCHECK_EQ(2, args.length());

test/mjsunit/proto-elements-add-during-foreach.js

+// Copyright 2015 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --enable-fast-array-builtins
+
+var a = [0,1,2,,,,7];
+var proto = {}
+a.__proto__ = proto;
+var visits = 0;
+Array.prototype.forEach.call(a, (v,i,o) => { ++visits; proto[4] = 4; });
+assertEquals(5, visits);