[ast] Pre-walk array literals to detect all-doubles

When buildin an array boilerplate description, we currently walk the
array literal as if it had tagged pointers, and post-hoc copy it to
a FixedDoubleArray if it had double elements kind.

Now, we calculate the elements kind during the InitDepthAndFlags walk,
and if the elements kind is Double, we allocate a FixedDoubleArray to
start with, and convert the elements of the array literal directly to
unboxed doubles in the array.

Change-Id: I56561e0af2236e785498eb70cb37eddcb09a56ca
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2002529
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Cr-Commit-Position: refs/heads/master@{#65817}

src/ast/ast.cc

@@ -12,10 +12,13 @@
 #include "src/base/hashmap.h"
 #include "src/builtins/builtins-constructor.h"
 #include "src/builtins/builtins.h"
+#include "src/common/assert-scope.h"
 #include "src/numbers/conversions-inl.h"
 #include "src/numbers/double.h"
 #include "src/objects/contexts.h"
+#include "src/objects/elements-kind.h"
 #include "src/objects/elements.h"
+#include "src/objects/fixed-array.h"
 #include "src/objects/literal-objects-inl.h"
 #include "src/objects/literal-objects.h"
 #include "src/objects/map.h"
@@ -547,6 +550,8 @@ int ArrayLiteral::InitDepthAndFlags() {
 
   // Fill in the literals.
   bool is_simple = first_spread_index_ < 0;
+  bool is_holey = false;
+  ElementsKind kind = FIRST_FAST_ELEMENTS_KIND;
   int depth_acc = 1;
   int array_index = 0;
   for (; array_index < constants_length; array_index++) {
@@ -559,11 +564,58 @@ int ArrayLiteral::InitDepthAndFlags() {
 
     if (!element->IsCompileTimeValue()) {
       is_simple = false;
+
+      // Don't change kind here: non-compile time values resolve to an unknown
+      // elements kind, so we allow them to be considered as any one of them.
+
+      // TODO(leszeks): It would be nice to DCHECK here that GetBoilerplateValue
+      // will return IsUninitialized, but that would require being on the main
+      // thread which we may not be.
+    } else {
+      Literal* literal = element->AsLiteral();
+
+      if (!literal) {
+        // Only arrays and objects are compile-time values but not (primitive)
+        // literals.
+        DCHECK(element->IsObjectLiteral() || element->IsArrayLiteral());
+        kind = PACKED_ELEMENTS;
+      } else {
+        switch (literal->type()) {
+          case Literal::kTheHole:
+            is_holey = true;
+            // The hole is allowed in holey double arrays (and holey Smi
+            // arrays), so ignore it as far as is_all_number is concerned.
+            break;
+          case Literal::kHeapNumber:
+            if (kind == PACKED_SMI_ELEMENTS) kind = PACKED_DOUBLE_ELEMENTS;
+            DCHECK_EQ(kind,
+                      GetMoreGeneralElementsKind(kind, PACKED_DOUBLE_ELEMENTS));
+            break;
+          case Literal::kSmi:
+            DCHECK_EQ(kind,
+                      GetMoreGeneralElementsKind(kind, PACKED_SMI_ELEMENTS));
+            break;
+          case Literal::kBigInt:
+          case Literal::kString:
+          case Literal::kSymbol:
+          case Literal::kBoolean:
+          case Literal::kUndefined:
+          case Literal::kNull:
+            kind = PACKED_ELEMENTS;
+            break;
+        }
+      }
     }
   }
 
+  if (is_holey) {
+    kind = GetHoleyElementsKind(kind);
+  }
+
   set_depth(depth_acc);
   set_is_simple(is_simple);
+  set_boilerplate_descriptor_kind(kind);
+
   // Array literals always need an initial allocation site to properly track
   // elements transitions.
   set_needs_initial_allocation_site(true);
@@ -575,55 +627,72 @@ void ArrayLiteral::BuildBoilerplateDescription(Isolate* isolate) {
 
   int constants_length =
       first_spread_index_ >= 0 ? first_spread_index_ : values()->length();
-  ElementsKind kind = FIRST_FAST_ELEMENTS_KIND;
-  Handle<FixedArray> fixed_array =
-      isolate->factory()->NewFixedArrayWithHoles(constants_length);
+  ElementsKind kind = boilerplate_descriptor_kind();
+  bool use_doubles = IsDoubleElementsKind(kind);
+
+  Handle<FixedArrayBase> elements;
+  if (use_doubles) {
+    elements = isolate->factory()->NewFixedDoubleArray(constants_length);
+  } else {
+    elements = isolate->factory()->NewFixedArrayWithHoles(constants_length);
+  }
 
   // Fill in the literals.
-  bool is_holey = false;
   int array_index = 0;
   for (; array_index < constants_length; array_index++) {
     Expression* element = values()->at(array_index);
     DCHECK(!element->IsSpread());
-    MaterializedLiteral* m_literal = element->AsMaterializedLiteral();
-    if (m_literal != nullptr) {
-      m_literal->BuildConstants(isolate);
-    }
+    if (use_doubles) {
+      Literal* literal = element->AsLiteral();
 
-    // New handle scope here, needs to be after BuildContants().
-    HandleScope scope(isolate);
-    Handle<Object> boilerplate_value = GetBoilerplateValue(element, isolate);
-    if (boilerplate_value->IsTheHole(isolate)) {
-      is_holey = true;
-      continue;
-    }
+      if (literal && literal->type() == Literal::kTheHole) {
+        DCHECK(IsHoleyElementsKind(kind));
+        DCHECK(GetBoilerplateValue(element, isolate)->IsTheHole(isolate));
+        FixedDoubleArray::cast(*elements).set_the_hole(array_index);
+        continue;
+      } else if (literal && literal->IsNumber()) {
+        FixedDoubleArray::cast(*elements).set(array_index, literal->AsNumber());
+      } else {
+        DCHECK(GetBoilerplateValue(element, isolate)->IsUninitialized(isolate));
+        FixedDoubleArray::cast(*elements).set(array_index, 0);
+      }
 
-    if (boilerplate_value->IsUninitialized(isolate)) {
-      boilerplate_value = handle(Smi::zero(), isolate);
-    }
+    } else {
+      MaterializedLiteral* m_literal = element->AsMaterializedLiteral();
+      if (m_literal != nullptr) {
+        m_literal->BuildConstants(isolate);
+      }
 
-    kind = GetMoreGeneralElementsKind(
-        kind, boilerplate_value->OptimalElementsKind(isolate));
-    fixed_array->set(array_index, *boilerplate_value);
-  }
+      // New handle scope here, needs to be after BuildContants().
+      HandleScope scope(isolate);
 
-  if (is_holey) kind = GetHoleyElementsKind(kind);
+      Object boilerplate_value = *GetBoilerplateValue(element, isolate);
+      // We shouldn't allocate after creating the boilerplate value.
+      DisallowHeapAllocation no_gc;
+
+      if (boilerplate_value.IsTheHole(isolate)) {
+        DCHECK(IsHoleyElementsKind(kind));
+        continue;
+      }
+
+      if (boilerplate_value.IsUninitialized(isolate)) {
+        boilerplate_value = Smi::zero();
+      }
+
+      DCHECK_EQ(boilerplate_descriptor_kind(),
+                GetMoreGeneralElementsKind(
+                    boilerplate_descriptor_kind(),
+                    boilerplate_value.OptimalElementsKind(isolate)));
+
+      Handle<FixedArray>::cast(elements)->set(array_index, boilerplate_value);
+    }
+  }
 
   // Simple and shallow arrays can be lazily copied, we transform the
   // elements array to a copy-on-write array.
   if (is_simple() && depth() == 1 && array_index > 0 &&
       IsSmiOrObjectElementsKind(kind)) {
-    fixed_array->set_map(ReadOnlyRoots(isolate).fixed_cow_array_map());
-  }
-
-  Handle<FixedArrayBase> elements = fixed_array;
-  if (IsDoubleElementsKind(kind)) {
-    ElementsAccessor* accessor = ElementsAccessor::ForKind(kind);
-    elements = isolate->factory()->NewFixedDoubleArray(constants_length);
-    // We are copying from non-fast-double to fast-double.
-    ElementsKind from_kind = TERMINAL_FAST_ELEMENTS_KIND;
-    accessor->CopyElements(isolate, fixed_array, from_kind, elements,
-                           constants_length);
+    elements->set_map(ReadOnlyRoots(isolate).fixed_cow_array_map());
   }
 
   boilerplate_description_ =

src/ast/ast.h

@@ -16,6 +16,7 @@
 #include "src/common/globals.h"
 #include "src/execution/isolate.h"
 #include "src/heap/factory.h"
+#include "src/objects/elements-kind.h"
 #include "src/objects/function-syntax-kind.h"
 #include "src/objects/literal-objects.h"
 #include "src/objects/smi.h"
@@ -1226,24 +1227,37 @@ class AggregateLiteral : public MaterializedLiteral {
   // constants and simple object and array literals.
   bool is_simple() const { return IsSimpleField::decode(bit_field_); }
 
+  ElementsKind boilerplate_descriptor_kind() const {
+    return BoilerplateDescriptorKindField::decode(bit_field_);
+  }
+
  private:
   int depth_ : 31;
   using NeedsInitialAllocationSiteField =
       MaterializedLiteral::NextBitField<bool, 1>;
   using IsSimpleField = NeedsInitialAllocationSiteField::Next<bool, 1>;
+  using BoilerplateDescriptorKindField =
+      IsSimpleField::Next<ElementsKind, kFastElementsKindBits>;
 
  protected:
   friend class AstNodeFactory;
   AggregateLiteral(int pos, NodeType type)
       : MaterializedLiteral(pos, type), depth_(0) {
-    bit_field_ |= NeedsInitialAllocationSiteField::encode(false) |
-                  IsSimpleField::encode(false);
+    bit_field_ |=
+        NeedsInitialAllocationSiteField::encode(false) |
+        IsSimpleField::encode(false) |
+        BoilerplateDescriptorKindField::encode(FIRST_FAST_ELEMENTS_KIND);
   }
 
   void set_is_simple(bool is_simple) {
     bit_field_ = IsSimpleField::update(bit_field_, is_simple);
   }
 
+  void set_boilerplate_descriptor_kind(ElementsKind kind) {
+    DCHECK(IsFastElementsKind(kind));
+    bit_field_ = BoilerplateDescriptorKindField::update(bit_field_, kind);
+  }
+
   void set_depth(int depth) {
     DCHECK(!is_initialized());
     depth_ = depth;
@@ -1254,7 +1268,7 @@ class AggregateLiteral : public MaterializedLiteral {
   }
 
   template <class T, int size>
-  using NextBitField = IsSimpleField::Next<T, size>;
+  using NextBitField = BoilerplateDescriptorKindField::Next<T, size>;
 };
 
 // Common supertype for ObjectLiteralProperty and ClassLiteralProperty

src/objects/elements-kind.h

@@ -5,6 +5,7 @@
 #ifndef V8_OBJECTS_ELEMENTS_KIND_H_
 #define V8_OBJECTS_ELEMENTS_KIND_H_
 
+#include "src/base/bits.h"
 #include "src/base/bounds.h"
 #include "src/base/macros.h"
 #include "src/common/checks.h"
@@ -102,6 +103,14 @@ constexpr int kFastElementsKindCount =
 constexpr int kFastElementsKindPackedToHoley =
     HOLEY_SMI_ELEMENTS - PACKED_SMI_ELEMENTS;
 
+constexpr int kElementsKindBits = 5;
+STATIC_ASSERT((1 << kElementsKindBits) > LAST_ELEMENTS_KIND);
+STATIC_ASSERT((1 << (kElementsKindBits - 1)) <= LAST_ELEMENTS_KIND);
+
+constexpr int kFastElementsKindBits = 3;
+STATIC_ASSERT((1 << kFastElementsKindBits) > LAST_FAST_ELEMENTS_KIND);
+STATIC_ASSERT((1 << (kFastElementsKindBits - 1)) <= LAST_FAST_ELEMENTS_KIND);
+
 V8_EXPORT_PRIVATE int ElementsKindToShiftSize(ElementsKind elements_kind);
 V8_EXPORT_PRIVATE int ElementsKindToByteSize(ElementsKind elements_kind);
 int GetDefaultHeaderSizeForElementsKind(ElementsKind elements_kind);