Reland "[deoptimizer] Staged materialization of objects."

This relands commit e71b8022.

This can now back in as the fix for chromium:787301 had enough time to
be tested in Canary.

Original change's description:
> [deoptimizer] Staged materialization of objects.
>
> The existing object materialization in the deoptimizer has the following problems:
>
> - Objects do not necessarily verify during materialization (because during the
>   depth first walk we might have inconsistent objects).
>
> - Stack can overflow (because we just materialize using recursive calls).
>
> - We generalize object fields.
>
>
> This CL re-implements the materialization algorithm to solve this problem. The
> new implementation creates the objects in two steps:
>
> 1. We allocate space for all the objects. In general, we allocate ByteArrays
>    of the right size. For leaf objects that cannot participate in cycles,
>    we build and initialize the materialized objects completely.
>
>    For JS objects, we insert markers into the byte array at the positions
>    where unboxed doubles are expected.
>
> 2. We initialize all the objects with the proper field values and change the
>    map from the ByteArray map to the correct map. This requires some sync
>    with the concurrent marker (Heap::NotifyObjectLayoutChange).
>
>    When initializing the JS object fields, we make sure that we respect
>    the unboxed double marker.
>
> Bug: chromium:770106, v8:3836
> Change-Id: I1ec466a9d19db9538df4ba915516d4c3ca825632
> Reviewed-on: https://chromium-review.googlesource.com/777559
> Commit-Queue: Jaroslav Sevcik <jarin@chromium.org>
> Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
> Reviewed-by: Michael Starzinger <mstarzinger@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#49821}

Bug: chromium:770106, v8:3836
Change-Id: Ied6c4e0fbae52713e55ae6dc13794a7521dbb8a5
Reviewed-on: https://chromium-review.googlesource.com/817745Reviewed-by: Benedikt Meurer <bmeurer@chromium.org>
Commit-Queue: Benedikt Meurer <bmeurer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#49982}

src/deoptimizer.cc

@@ -19,6 +19,8 @@
 #include "src/tracing/trace-event.h"
 #include "src/v8.h"
 
+// Has to be the last include (doesn't have include guards)
+#include "src/objects/object-macros.h"
 
 namespace v8 {
 namespace internal {
@@ -1673,6 +1675,8 @@ void Deoptimizer::MaterializeHeapObjects() {
         reinterpret_cast<intptr_t>(*value);
   }
 
+  translated_state_.VerifyMaterializedObjects();
+
   isolate_->materialized_object_store()->Remove(
       reinterpret_cast<Address>(stack_fp_));
 }
@@ -2395,9 +2399,8 @@ int TranslatedValue::object_index() const {
 
 Object* TranslatedValue::GetRawValue() const {
   // If we have a value, return it.
-  Handle<Object> result_handle;
-  if (value_.ToHandle(&result_handle)) {
-    return *result_handle;
+  if (materialization_state() == kFinished) {
+    return *storage_;
   }
 
   // Otherwise, do a best effort to get the value without allocation.
@@ -2439,11 +2442,15 @@ Object* TranslatedValue::GetRawValue() const {
   return isolate()->heap()->arguments_marker();
 }
 
+void TranslatedValue::set_initialized_storage(Handle<Object> storage) {
+  DCHECK_EQ(kUninitialized, materialization_state());
+  storage_ = storage;
+  materialization_state_ = kFinished;
+}
 
 Handle<Object> TranslatedValue::GetValue() {
-  Handle<Object> result;
   // If we already have a value, then get it.
-  if (value_.ToHandle(&result)) return result;
+  if (materialization_state() == kFinished) return storage_;
 
   // Otherwise we have to materialize.
   switch (kind()) {
@@ -2454,12 +2461,27 @@ Handle<Object> TranslatedValue::GetValue() {
     case TranslatedValue::kFloat:
     case TranslatedValue::kDouble: {
       MaterializeSimple();
-      return value_.ToHandleChecked();
+      return storage_;
     }
 
     case TranslatedValue::kCapturedObject:
-    case TranslatedValue::kDuplicatedObject:
-      return container_->MaterializeObjectAt(object_index());
+    case TranslatedValue::kDuplicatedObject: {
+      // We need to materialize the object (or possibly even object graphs).
+      // To make the object verifier happy, we materialize in two steps.
+
+      // 1. Allocate storage for reachable objects. This makes sure that for
+      //    each object we have allocated space on heap. The space will be
+      //    a byte array that will be later initialized, or a fully
+      //    initialized object if it is safe to allocate one that will
+      //    pass the verifier.
+      container_->EnsureObjectAllocatedAt(this);
+
+      // 2. Initialize the objects. If we have allocated only byte arrays
+      //    for some objects, we now overwrite the byte arrays with the
+      //    correct object fields. Note that this phase does not allocate
+      //    any new objects, so it does not trigger the object verifier.
+      return container_->InitializeObjectAt(this);
+    }
 
     case TranslatedValue::kInvalid:
       FATAL("unexpected case");
@@ -2470,36 +2492,39 @@ Handle<Object> TranslatedValue::GetValue() {
   return Handle<Object>::null();
 }
 
-
 void TranslatedValue::MaterializeSimple() {
   // If we already have materialized, return.
-  if (!value_.is_null()) return;
+  if (materialization_state() == kFinished) return;
 
   Object* raw_value = GetRawValue();
   if (raw_value != isolate()->heap()->arguments_marker()) {
     // We can get the value without allocation, just return it here.
-    value_ = Handle<Object>(raw_value, isolate());
+    set_initialized_storage(Handle<Object>(raw_value, isolate()));
     return;
   }
 
   switch (kind()) {
     case kInt32:
-      value_ = Handle<Object>(isolate()->factory()->NewNumber(int32_value()));
+      set_initialized_storage(
+          Handle<Object>(isolate()->factory()->NewNumber(int32_value())));
       return;
 
     case kUInt32:
-      value_ = Handle<Object>(isolate()->factory()->NewNumber(uint32_value()));
+      set_initialized_storage(
+          Handle<Object>(isolate()->factory()->NewNumber(uint32_value())));
       return;
 
     case kFloat: {
       double scalar_value = float_value().get_scalar();
-      value_ = Handle<Object>(isolate()->factory()->NewNumber(scalar_value));
+      set_initialized_storage(
+          Handle<Object>(isolate()->factory()->NewNumber(scalar_value)));
       return;
     }
 
     case kDouble: {
       double scalar_value = double_value().get_scalar();
-      value_ = Handle<Object>(isolate()->factory()->NewNumber(scalar_value));
+      set_initialized_storage(
+          Handle<Object>(isolate()->factory()->NewNumber(scalar_value)));
       return;
     }
 
@@ -2561,7 +2586,7 @@ Float64 TranslatedState::GetDoubleSlot(Address fp, int slot_offset) {
 
 void TranslatedValue::Handlify() {
   if (kind() == kTagged) {
-    value_ = Handle<Object>(raw_literal(), isolate());
+    set_initialized_storage(Handle<Object>(raw_literal(), isolate()));
     raw_literal_ = nullptr;
   }
 }
@@ -3224,544 +3249,439 @@ void TranslatedState::Prepare(Address stack_frame_pointer) {
   UpdateFromPreviouslyMaterializedObjects();
 }
 
-class TranslatedState::CapturedObjectMaterializer {
- public:
-  CapturedObjectMaterializer(TranslatedState* state, int frame_index,
-                             int field_count)
-      : state_(state), frame_index_(frame_index), field_count_(field_count) {}
-
-  // Ensure the properties never contain mutable heap numbers. This is necessary
-  // because the deoptimizer generalizes all maps to tagged representation
-  // fields (so mutable heap numbers are not allowed).
-  static void EnsurePropertiesGeneralized(Handle<Object> properties_or_hash) {
-    if (properties_or_hash->IsPropertyArray()) {
-      Handle<PropertyArray> properties =
-          Handle<PropertyArray>::cast(properties_or_hash);
-      int length = properties->length();
-      for (int i = 0; i < length; i++) {
-        if (properties->get(i)->IsMutableHeapNumber()) {
-          Handle<HeapObject> box(HeapObject::cast(properties->get(i)));
-          box->set_map(properties->GetIsolate()->heap()->heap_number_map());
-        }
+TranslatedValue* TranslatedState::GetValueByObjectIndex(int object_index) {
+  CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
+  TranslatedState::ObjectPosition pos = object_positions_[object_index];
+  return &(frames_[pos.frame_index_].values_[pos.value_index_]);
+}
+
+Handle<Object> TranslatedState::InitializeObjectAt(TranslatedValue* slot) {
+  slot = ResolveCapturedObject(slot);
+
+  DisallowHeapAllocation no_allocation;
+  if (slot->materialization_state() != TranslatedValue::kFinished) {
+    std::stack<int> worklist;
+    worklist.push(slot->object_index());
+    slot->mark_finished();
+
+    while (!worklist.empty()) {
+      int index = worklist.top();
+      worklist.pop();
+      InitializeCapturedObjectAt(index, &worklist, no_allocation);
+    }
+  }
+  return slot->GetStorage();
+}
+
+void TranslatedState::InitializeCapturedObjectAt(
+    int object_index, std::stack<int>* worklist,
+    const DisallowHeapAllocation& no_allocation) {
+  CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
+  TranslatedState::ObjectPosition pos = object_positions_[object_index];
+  int value_index = pos.value_index_;
+
+  TranslatedFrame* frame = &(frames_[pos.frame_index_]);
+  TranslatedValue* slot = &(frame->values_[value_index]);
+  value_index++;
+
+  CHECK_EQ(TranslatedValue::kFinished, slot->materialization_state());
+  CHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
+
+  // Ensure all fields are initialized.
+  int children_init_index = value_index;
+  for (int i = 0; i < slot->GetChildrenCount(); i++) {
+    // If the field is an object that has not been initialized yet, queue it
+    // for initialization (and mark it as such).
+    TranslatedValue* child_slot = frame->ValueAt(children_init_index);
+    if (child_slot->kind() == TranslatedValue::kCapturedObject ||
+        child_slot->kind() == TranslatedValue::kDuplicatedObject) {
+      child_slot = ResolveCapturedObject(child_slot);
+      if (child_slot->materialization_state() != TranslatedValue::kFinished) {
+        DCHECK_EQ(TranslatedValue::kAllocated,
+                  child_slot->materialization_state());
+        worklist->push(child_slot->object_index());
+        child_slot->mark_finished();
       }
     }
+    SkipSlots(1, frame, &children_init_index);
   }
 
-  Handle<Object> FieldAt(int* value_index) {
-    CHECK_GT(field_count_, 0);
-    --field_count_;
-    Handle<Object> object = state_->MaterializeAt(frame_index_, value_index);
-    // This is a big hammer to make sure that the materialized objects do not
-    // have property arrays with mutable heap numbers (mutable heap numbers are
-    // bad because we generalize maps for all materialized objects).
-    EnsurePropertiesGeneralized(object);
-    return object;
+  // Read the map.
+  // The map should never be materialized, so let us check we already have
+  // an existing object here.
+  CHECK_EQ(frame->values_[value_index].kind(), TranslatedValue::kTagged);
+  Handle<Map> map = Handle<Map>::cast(frame->values_[value_index].GetValue());
+  CHECK(map->IsMap());
+  value_index++;
+
+  // Handle the special cases.
+  switch (map->instance_type()) {
+    case MUTABLE_HEAP_NUMBER_TYPE:
+    case FIXED_DOUBLE_ARRAY_TYPE:
+      return;
+
+    case FIXED_ARRAY_TYPE:
+    case HASH_TABLE_TYPE:
+    case PROPERTY_ARRAY_TYPE:
+    case CONTEXT_EXTENSION_TYPE:
+      InitializeObjectWithTaggedFieldsAt(frame, &value_index, slot, map,
+                                         no_allocation);
+      break;
+
+    default:
+      CHECK(map->IsJSObjectMap());
+      InitializeJSObjectAt(frame, &value_index, slot, map, no_allocation);
+      break;
   }
+  CHECK_EQ(value_index, children_init_index);
+}
 
-  ~CapturedObjectMaterializer() { CHECK_EQ(0, field_count_); }
+void TranslatedState::EnsureObjectAllocatedAt(TranslatedValue* slot) {
+  slot = ResolveCapturedObject(slot);
 
- private:
-  TranslatedState* state_;
-  int frame_index_;
-  int field_count_;
+  if (slot->materialization_state() == TranslatedValue::kUninitialized) {
+    std::stack<int> worklist;
+    worklist.push(slot->object_index());
+    slot->mark_allocated();
+
+    while (!worklist.empty()) {
+      int index = worklist.top();
+      worklist.pop();
+      EnsureCapturedObjectAllocatedAt(index, &worklist);
+    }
+  }
+}
+
+void TranslatedState::MaterializeFixedDoubleArray(TranslatedFrame* frame,
+                                                  int* value_index,
+                                                  TranslatedValue* slot,
+                                                  Handle<Map> map) {
+  int length = Smi::cast(frame->values_[*value_index].GetRawValue())->value();
+  (*value_index)++;
+  Handle<FixedDoubleArray> array = Handle<FixedDoubleArray>::cast(
+      isolate()->factory()->NewFixedDoubleArray(length));
+  CHECK_GT(length, 0);
+  for (int i = 0; i < length; i++) {
+    CHECK_NE(TranslatedValue::kCapturedObject,
+             frame->values_[*value_index].kind());
+    Handle<Object> value = frame->values_[*value_index].GetValue();
+    if (value->IsNumber()) {
+      array->set(i, value->Number());
+    } else {
+      CHECK(value.is_identical_to(isolate()->factory()->the_hole_value()));
+      array->set_the_hole(isolate(), i);
+    }
+    (*value_index)++;
+  }
+  slot->set_storage(array);
+}
+
+void TranslatedState::MaterializeMutableHeapNumber(TranslatedFrame* frame,
+                                                   int* value_index,
+                                                   TranslatedValue* slot) {
+  CHECK_NE(TranslatedValue::kCapturedObject,
+           frame->values_[*value_index].kind());
+  Handle<Object> value = frame->values_[*value_index].GetValue();
+  Handle<HeapNumber> box;
+  CHECK(value->IsNumber());
+  box = isolate()->factory()->NewHeapNumber(value->Number(), MUTABLE);
+  (*value_index)++;
+  slot->set_storage(box);
+}
+
+namespace {
+
+enum DoubleStorageKind : uint8_t {
+  kStoreTagged,
+  kStoreUnboxedDouble,
+  kStoreMutableHeapNumber,
 };
 
-Handle<Object> TranslatedState::MaterializeCapturedObjectAt(
-    TranslatedValue* slot, int frame_index, int* value_index) {
-  int length = slot->GetChildrenCount();
+}  // namespace
 
-  CapturedObjectMaterializer materializer(this, frame_index, length);
+void TranslatedState::SkipSlots(int slots_to_skip, TranslatedFrame* frame,
+                                int* value_index) {
+  while (slots_to_skip > 0) {
+    TranslatedValue* slot = &(frame->values_[*value_index]);
+    (*value_index)++;
+    slots_to_skip--;
 
-  Handle<Object> result;
-  if (slot->value_.ToHandle(&result)) {
-    // This has been previously materialized, return the previous value.
-    // We still need to skip all the nested objects.
-    for (int i = 0; i < length; i++) {
-      materializer.FieldAt(value_index);
+    if (slot->kind() == TranslatedValue::kCapturedObject) {
+      slots_to_skip += slot->GetChildrenCount();
     }
-
-    return result;
   }
+}
 
-  Handle<Object> map_object = materializer.FieldAt(value_index);
-  Handle<Map> map = Map::GeneralizeAllFields(Handle<Map>::cast(map_object));
+void TranslatedState::EnsureCapturedObjectAllocatedAt(
+    int object_index, std::stack<int>* worklist) {
+  CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
+  TranslatedState::ObjectPosition pos = object_positions_[object_index];
+  int value_index = pos.value_index_;
+
+  TranslatedFrame* frame = &(frames_[pos.frame_index_]);
+  TranslatedValue* slot = &(frame->values_[value_index]);
+  value_index++;
+
+  CHECK_EQ(TranslatedValue::kAllocated, slot->materialization_state());
+  CHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
+
+  // Read the map.
+  // The map should never be materialized, so let us check we already have
+  // an existing object here.
+  CHECK_EQ(frame->values_[value_index].kind(), TranslatedValue::kTagged);
+  Handle<Map> map = Handle<Map>::cast(frame->values_[value_index].GetValue());
+  CHECK(map->IsMap());
+  value_index++;
+
+  // Handle the special cases.
   switch (map->instance_type()) {
+    case FIXED_DOUBLE_ARRAY_TYPE:
+      // Materialize (i.e. allocate&initialize) the array and return since
+      // there is no need to process the children.
+      return MaterializeFixedDoubleArray(frame, &value_index, slot, map);
+
     case MUTABLE_HEAP_NUMBER_TYPE:
-    case HEAP_NUMBER_TYPE: {
-      // Reuse the HeapNumber value directly as it is already properly
-      // tagged and skip materializing the HeapNumber explicitly.
-      Handle<Object> object = materializer.FieldAt(value_index);
-      slot->value_ = object;
-      // On 32-bit architectures, there is an extra slot there because
-      // the escape analysis calculates the number of slots as
-      // object-size/pointer-size. To account for this, we read out
-      // any extra slots.
-      for (int i = 0; i < length - 2; i++) {
-        materializer.FieldAt(value_index);
-      }
-      return object;
-    }
-    case JS_OBJECT_TYPE:
-    case JS_ERROR_TYPE:
-    case JS_ARGUMENTS_TYPE: {
-      Handle<JSObject> object =
-          isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED);
-      slot->value_ = object;
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      int in_object_properties = map->GetInObjectProperties();
-      for (int i = 0; i < in_object_properties; ++i) {
-        Handle<Object> value = materializer.FieldAt(value_index);
-        FieldIndex index = FieldIndex::ForPropertyIndex(object->map(), i);
-        object->FastPropertyAtPut(index, *value);
-      }
-      return object;
-    }
-    case JS_SET_KEY_VALUE_ITERATOR_TYPE:
-    case JS_SET_VALUE_ITERATOR_TYPE: {
-      Handle<JSSetIterator> object = Handle<JSSetIterator>::cast(
-          isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED));
-      slot->value_ = object;
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      Handle<Object> table = materializer.FieldAt(value_index);
-      Handle<Object> index = materializer.FieldAt(value_index);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      object->set_table(*table);
-      object->set_index(*index);
-      return object;
-    }
-    case JS_MAP_KEY_ITERATOR_TYPE:
-    case JS_MAP_KEY_VALUE_ITERATOR_TYPE:
-    case JS_MAP_VALUE_ITERATOR_TYPE: {
-      Handle<JSMapIterator> object = Handle<JSMapIterator>::cast(
-          isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED));
-      slot->value_ = object;
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      Handle<Object> table = materializer.FieldAt(value_index);
-      Handle<Object> index = materializer.FieldAt(value_index);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      object->set_table(*table);
-      object->set_index(*index);
-      return object;
-    }
-#define ARRAY_ITERATOR_CASE(type) case type:
-      ARRAY_ITERATOR_TYPE_LIST(ARRAY_ITERATOR_CASE)
-#undef ARRAY_ITERATOR_CASE
-      {
-        Handle<JSArrayIterator> object = Handle<JSArrayIterator>::cast(
-            isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED));
-        slot->value_ = object;
-        // Initialize the index to zero to make the heap verifier happy.
-        object->set_index(Smi::FromInt(0));
-        Handle<Object> properties = materializer.FieldAt(value_index);
-        Handle<Object> elements = materializer.FieldAt(value_index);
-        Handle<Object> iterated_object = materializer.FieldAt(value_index);
-        Handle<Object> next_index = materializer.FieldAt(value_index);
-        Handle<Object> iterated_object_map = materializer.FieldAt(value_index);
-        object->set_raw_properties_or_hash(*properties);
-        object->set_elements(FixedArrayBase::cast(*elements));
-        object->set_object(*iterated_object);
-        object->set_index(*next_index);
-        object->set_object_map(*iterated_object_map);
-        return object;
-      }
-    case JS_STRING_ITERATOR_TYPE: {
-      Handle<JSStringIterator> object = Handle<JSStringIterator>::cast(
-          isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED));
-      slot->value_ = object;
-      // Initialize the index to zero to make the heap verifier happy.
-      object->set_index(0);
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      Handle<Object> iterated_string = materializer.FieldAt(value_index);
-      Handle<Object> next_index = materializer.FieldAt(value_index);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      CHECK(iterated_string->IsString());
-      object->set_string(String::cast(*iterated_string));
-      CHECK(next_index->IsSmi());
-      object->set_index(Smi::ToInt(*next_index));
-      return object;
-    }
-    case JS_ASYNC_FROM_SYNC_ITERATOR_TYPE: {
-      Handle<JSAsyncFromSyncIterator> object =
-          Handle<JSAsyncFromSyncIterator>::cast(
-              isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED));
-      slot->value_ = object;
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      Handle<Object> sync_iterator = materializer.FieldAt(value_index);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      object->set_sync_iterator(JSReceiver::cast(*sync_iterator));
-      return object;
-    }
-    case JS_ARRAY_TYPE: {
-      Handle<JSArray> object = Handle<JSArray>::cast(
-          isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED));
-      slot->value_ = object;
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      Handle<Object> array_length = materializer.FieldAt(value_index);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      object->set_length(*array_length);
-      int in_object_properties = map->GetInObjectProperties();
-      for (int i = 0; i < in_object_properties; ++i) {
-        Handle<Object> value = materializer.FieldAt(value_index);
-        FieldIndex index = FieldIndex::ForPropertyIndex(object->map(), i);
-        object->FastPropertyAtPut(index, *value);
-      }
-      return object;
-    }
-    case JS_BOUND_FUNCTION_TYPE: {
-      Handle<JSBoundFunction> object = Handle<JSBoundFunction>::cast(
-          isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED));
-      slot->value_ = object;
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      Handle<Object> bound_target_function = materializer.FieldAt(value_index);
-      Handle<Object> bound_this = materializer.FieldAt(value_index);
-      Handle<Object> bound_arguments = materializer.FieldAt(value_index);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      object->set_bound_target_function(
-          JSReceiver::cast(*bound_target_function));
-      object->set_bound_this(*bound_this);
-      object->set_bound_arguments(FixedArray::cast(*bound_arguments));
-      return object;
-    }
-    case JS_FUNCTION_TYPE: {
-      Handle<JSFunction> object = isolate_->factory()->NewFunction(
-          map, handle(isolate_->object_function()->shared()),
-          handle(isolate_->context()), NOT_TENURED);
-      slot->value_ = object;
-      // We temporarily allocated a JSFunction for the {Object} function
-      // within the current context, to break cycles in the object graph.
-      // The correct function and context will be set below once available.
-      STATIC_ASSERT(JSFunction::kSizeWithoutPrototype == 7 * kPointerSize);
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      Handle<Object> shared = materializer.FieldAt(value_index);
-      Handle<Object> context = materializer.FieldAt(value_index);
-      Handle<Object> vector_cell = materializer.FieldAt(value_index);
-      Handle<Object> code = materializer.FieldAt(value_index);
-      bool has_prototype_slot = map->has_prototype_slot();
-      Handle<Object> prototype;
-      if (has_prototype_slot) {
-        prototype = materializer.FieldAt(value_index);
-      }
-      object->set_map(*map);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      object->set_shared(SharedFunctionInfo::cast(*shared));
-      object->set_context(Context::cast(*context));
-      object->set_feedback_vector_cell(Cell::cast(*vector_cell));
-      object->set_code(Code::cast(*code));
-      if (has_prototype_slot) {
-        object->set_prototype_or_initial_map(*prototype);
-      }
-      int in_object_properties = map->GetInObjectProperties();
-      for (int i = 0; i < in_object_properties; ++i) {
-        Handle<Object> value = materializer.FieldAt(value_index);
-        FieldIndex index = FieldIndex::ForPropertyIndex(object->map(), i);
-        object->FastPropertyAtPut(index, *value);
-      }
-      return object;
-    }
-    case JS_ASYNC_GENERATOR_OBJECT_TYPE:
-    case JS_GENERATOR_OBJECT_TYPE: {
-      Handle<JSGeneratorObject> object = Handle<JSGeneratorObject>::cast(
-          isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED));
-      slot->value_ = object;
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      Handle<Object> function = materializer.FieldAt(value_index);
-      Handle<Object> context = materializer.FieldAt(value_index);
-      Handle<Object> receiver = materializer.FieldAt(value_index);
-      Handle<Object> input_or_debug_pos = materializer.FieldAt(value_index);
-      Handle<Object> resume_mode = materializer.FieldAt(value_index);
-      Handle<Object> continuation_offset = materializer.FieldAt(value_index);
-      Handle<Object> register_file = materializer.FieldAt(value_index);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      object->set_function(JSFunction::cast(*function));
-      object->set_context(Context::cast(*context));
-      object->set_receiver(*receiver);
-      object->set_input_or_debug_pos(*input_or_debug_pos);
-      object->set_resume_mode(Smi::ToInt(*resume_mode));
-      object->set_continuation(Smi::ToInt(*continuation_offset));
-      object->set_register_file(FixedArray::cast(*register_file));
-
-      if (object->IsJSAsyncGeneratorObject()) {
-        auto generator = Handle<JSAsyncGeneratorObject>::cast(object);
-        Handle<Object> queue = materializer.FieldAt(value_index);
-        Handle<Object> awaited_promise = materializer.FieldAt(value_index);
-        generator->set_queue(HeapObject::cast(*queue));
-        generator->set_awaited_promise(HeapObject::cast(*awaited_promise));
-      }
+      // Materialize (i.e. allocate&initialize) the heap number and return.
+      // There is no need to process the children.
+      return MaterializeMutableHeapNumber(frame, &value_index, slot);
+
+    case FIXED_ARRAY_TYPE:
+    case HASH_TABLE_TYPE: {
+      // Check we have the right size.
+      int array_length =
+          Smi::cast(frame->values_[value_index].GetRawValue())->value();
+      int instance_size = FixedArray::SizeFor(array_length);
+      CHECK_EQ(instance_size, slot->GetChildrenCount() * kPointerSize);
+
+      slot->set_storage(AllocateStorageFor(slot));
+      break;
+    }
 
-      int in_object_properties = map->GetInObjectProperties();
-      for (int i = 0; i < in_object_properties; ++i) {
-        Handle<Object> value = materializer.FieldAt(value_index);
-        FieldIndex index = FieldIndex::ForPropertyIndex(object->map(), i);
-        object->FastPropertyAtPut(index, *value);
-      }
-      return object;
+    case PROPERTY_ARRAY_TYPE: {
+      // Check we have the right size.
+      int length_or_hash =
+          Smi::cast(frame->values_[value_index].GetRawValue())->value();
+      int array_length = PropertyArray::LengthField::decode(length_or_hash);
+      int instance_size = PropertyArray::SizeFor(array_length);
+      CHECK_EQ(instance_size, slot->GetChildrenCount() * kPointerSize);
+
+      slot->set_storage(AllocateStorageFor(slot));
+      break;
     }
+
     case CONTEXT_EXTENSION_TYPE: {
-      Handle<ContextExtension> object =
-          isolate_->factory()->NewContextExtension(
-              isolate_->factory()->NewScopeInfo(1),
-              isolate_->factory()->undefined_value());
-      slot->value_ = object;
-      Handle<Object> scope_info = materializer.FieldAt(value_index);
-      Handle<Object> extension = materializer.FieldAt(value_index);
-      object->set_scope_info(ScopeInfo::cast(*scope_info));
-      object->set_extension(*extension);
-      return object;
-    }
-    case HASH_TABLE_TYPE:
-    case FIXED_ARRAY_TYPE: {
-      Handle<Object> lengthObject = materializer.FieldAt(value_index);
-      int32_t array_length = 0;
-      CHECK(lengthObject->ToInt32(&array_length));
-      Handle<FixedArray> object =
-          isolate_->factory()->NewFixedArray(array_length);
-      // We need to set the map, because the fixed array we are
-      // materializing could be a context or an arguments object,
-      // in which case we must retain that information.
-      object->set_map(*map);
-      slot->value_ = object;
-      for (int i = 0; i < array_length; ++i) {
-        Handle<Object> value = materializer.FieldAt(value_index);
-        object->set(i, *value);
-      }
-      return object;
+      CHECK_EQ(map->instance_size(), slot->GetChildrenCount() * kPointerSize);
+      slot->set_storage(AllocateStorageFor(slot));
+      break;
     }
-    case PROPERTY_ARRAY_TYPE: {
-      DCHECK_EQ(*map, isolate_->heap()->property_array_map());
-      Handle<Object> lengthObject = materializer.FieldAt(value_index);
-      int32_t array_length = 0;
-      CHECK(lengthObject->ToInt32(&array_length));
-      Handle<PropertyArray> object =
-          isolate_->factory()->NewPropertyArray(array_length);
-      slot->value_ = object;
-      for (int i = 0; i < array_length; ++i) {
-        Handle<Object> value = materializer.FieldAt(value_index);
-        object->set(i, *value);
-      }
-      return object;
-    }
-    case FIXED_DOUBLE_ARRAY_TYPE: {
-      DCHECK_EQ(*map, isolate_->heap()->fixed_double_array_map());
-      Handle<Object> lengthObject = materializer.FieldAt(value_index);
-      int32_t array_length = 0;
-      CHECK(lengthObject->ToInt32(&array_length));
-      Handle<FixedArrayBase> object =
-          isolate_->factory()->NewFixedDoubleArray(array_length);
-      slot->value_ = object;
-      if (array_length > 0) {
-        Handle<FixedDoubleArray> double_array =
-            Handle<FixedDoubleArray>::cast(object);
-        for (int i = 0; i < array_length; ++i) {
-          Handle<Object> value = materializer.FieldAt(value_index);
-          if (value.is_identical_to(isolate_->factory()->the_hole_value())) {
-            double_array->set_the_hole(isolate_, i);
-          } else {
-            CHECK(value->IsNumber());
-            double_array->set(i, value->Number());
-          }
-        }
+
+    default:
+      CHECK(map->IsJSObjectMap());
+      EnsureJSObjectAllocated(slot, map);
+      TranslatedValue* properties_slot = &(frame->values_[value_index]);
+      if (properties_slot->kind() == TranslatedValue::kCapturedObject) {
+        // If we are materializing the property array, make sure we put
+        // the mutable heap numbers at the right places.
+        EnsurePropertiesAllocatedAndMarked(properties_slot, map);
+        value_index++;
+        EnsureChildrenAllocated(properties_slot->GetChildrenCount(), frame,
+                                &value_index, worklist);
       }
-      return object;
-    }
-    case JS_REGEXP_TYPE: {
-      Handle<JSRegExp> object = Handle<JSRegExp>::cast(
-          isolate_->factory()->NewJSObjectFromMap(map, NOT_TENURED));
-      slot->value_ = object;
-      Handle<Object> properties = materializer.FieldAt(value_index);
-      Handle<Object> elements = materializer.FieldAt(value_index);
-      Handle<Object> data = materializer.FieldAt(value_index);
-      Handle<Object> source = materializer.FieldAt(value_index);
-      Handle<Object> flags = materializer.FieldAt(value_index);
-      Handle<Object> last_index = materializer.FieldAt(value_index);
-      object->set_raw_properties_or_hash(*properties);
-      object->set_elements(FixedArrayBase::cast(*elements));
-      object->set_data(*data);
-      object->set_source(*source);
-      object->set_flags(*flags);
-      object->set_last_index(*last_index);
-      return object;
-    }
-    case STRING_TYPE:
-    case ONE_BYTE_STRING_TYPE:
-    case CONS_STRING_TYPE:
-    case CONS_ONE_BYTE_STRING_TYPE:
-    case SLICED_STRING_TYPE:
-    case SLICED_ONE_BYTE_STRING_TYPE:
-    case EXTERNAL_STRING_TYPE:
-    case EXTERNAL_ONE_BYTE_STRING_TYPE:
-    case EXTERNAL_STRING_WITH_ONE_BYTE_DATA_TYPE:
-    case SHORT_EXTERNAL_STRING_TYPE:
-    case SHORT_EXTERNAL_ONE_BYTE_STRING_TYPE:
-    case SHORT_EXTERNAL_STRING_WITH_ONE_BYTE_DATA_TYPE:
-    case THIN_STRING_TYPE:
-    case THIN_ONE_BYTE_STRING_TYPE:
-    case INTERNALIZED_STRING_TYPE:
-    case ONE_BYTE_INTERNALIZED_STRING_TYPE:
-    case EXTERNAL_INTERNALIZED_STRING_TYPE:
-    case EXTERNAL_ONE_BYTE_INTERNALIZED_STRING_TYPE:
-    case EXTERNAL_INTERNALIZED_STRING_WITH_ONE_BYTE_DATA_TYPE:
-    case SHORT_EXTERNAL_INTERNALIZED_STRING_TYPE:
-    case SHORT_EXTERNAL_ONE_BYTE_INTERNALIZED_STRING_TYPE:
-    case SHORT_EXTERNAL_INTERNALIZED_STRING_WITH_ONE_BYTE_DATA_TYPE:
-    case SYMBOL_TYPE:
-    case ODDBALL_TYPE:
-    case JS_GLOBAL_OBJECT_TYPE:
-    case JS_GLOBAL_PROXY_TYPE:
-    case JS_API_OBJECT_TYPE:
-    case JS_SPECIAL_API_OBJECT_TYPE:
-    case JS_VALUE_TYPE:
-    case JS_MESSAGE_OBJECT_TYPE:
-    case JS_DATE_TYPE:
-    case JS_CONTEXT_EXTENSION_OBJECT_TYPE:
-    case JS_MODULE_NAMESPACE_TYPE:
-    case JS_ARRAY_BUFFER_TYPE:
-    case JS_TYPED_ARRAY_TYPE:
-    case JS_DATA_VIEW_TYPE:
-    case JS_SET_TYPE:
-    case JS_MAP_TYPE:
-    case JS_WEAK_MAP_TYPE:
-    case JS_WEAK_SET_TYPE:
-    case JS_PROMISE_TYPE:
-    case JS_PROXY_TYPE:
-    case MAP_TYPE:
-    case ALLOCATION_SITE_TYPE:
-    case ACCESSOR_INFO_TYPE:
-    case SHARED_FUNCTION_INFO_TYPE:
-    case FUNCTION_TEMPLATE_INFO_TYPE:
-    case ACCESSOR_PAIR_TYPE:
-    case BYTE_ARRAY_TYPE:
-    case BYTECODE_ARRAY_TYPE:
-    case DESCRIPTOR_ARRAY_TYPE:
-    case TRANSITION_ARRAY_TYPE:
-    case FEEDBACK_VECTOR_TYPE:
-    case FOREIGN_TYPE:
-    case SCRIPT_TYPE:
-    case CODE_TYPE:
-    case PROPERTY_CELL_TYPE:
-    case BIGINT_TYPE:
-    case MODULE_TYPE:
-    case MODULE_INFO_ENTRY_TYPE:
-    case FREE_SPACE_TYPE:
-#define FIXED_TYPED_ARRAY_CASE(Type, type, TYPE, ctype, size) \
-  case FIXED_##TYPE##_ARRAY_TYPE:
-      TYPED_ARRAYS(FIXED_TYPED_ARRAY_CASE)
-#undef FIXED_TYPED_ARRAY_CASE
-    case FILLER_TYPE:
-    case ACCESS_CHECK_INFO_TYPE:
-    case INTERCEPTOR_INFO_TYPE:
-    case OBJECT_TEMPLATE_INFO_TYPE:
-    case ALLOCATION_MEMENTO_TYPE:
-    case ALIASED_ARGUMENTS_ENTRY_TYPE:
-    case PROMISE_RESOLVE_THENABLE_JOB_INFO_TYPE:
-    case PROMISE_REACTION_JOB_INFO_TYPE:
-    case DEBUG_INFO_TYPE:
-    case STACK_FRAME_INFO_TYPE:
-    case CELL_TYPE:
-    case WEAK_CELL_TYPE:
-    case SMALL_ORDERED_HASH_MAP_TYPE:
-    case SMALL_ORDERED_HASH_SET_TYPE:
-    case CODE_DATA_CONTAINER_TYPE:
-    case PROTOTYPE_INFO_TYPE:
-    case TUPLE2_TYPE:
-    case TUPLE3_TYPE:
-    case LOAD_HANDLER_TYPE:
-    case STORE_HANDLER_TYPE:
-    case ASYNC_GENERATOR_REQUEST_TYPE:
-    case WASM_MODULE_TYPE:
-    case WASM_INSTANCE_TYPE:
-    case WASM_MEMORY_TYPE:
-    case WASM_TABLE_TYPE:
-      OFStream os(stderr);
-      os << "[couldn't handle instance type " << map->instance_type() << "]"
-         << std::endl;
-      UNREACHABLE();
       break;
   }
-  UNREACHABLE();
+
+  EnsureChildrenAllocated(slot->GetChildrenCount() - 1, frame, &value_index,
+                          worklist);
+}
+
+void TranslatedState::EnsureChildrenAllocated(int count, TranslatedFrame* frame,
+                                              int* value_index,
+                                              std::stack<int>* worklist) {
+  // Ensure all children are allocated.
+  for (int i = 0; i < count; i++) {
+    // If the field is an object that has not been allocated yet, queue it
+    // for initialization (and mark it as such).
+    TranslatedValue* child_slot = frame->ValueAt(*value_index);
+    if (child_slot->kind() == TranslatedValue::kCapturedObject ||
+        child_slot->kind() == TranslatedValue::kDuplicatedObject) {
+      child_slot = ResolveCapturedObject(child_slot);
+      if (child_slot->materialization_state() ==
+          TranslatedValue::kUninitialized) {
+        worklist->push(child_slot->object_index());
+        child_slot->mark_allocated();
+      }
+    } else {
+      // Make sure the simple values (heap numbers, etc.) are properly
+      // initialized.
+      child_slot->MaterializeSimple();
+    }
+    SkipSlots(1, frame, value_index);
+  }
 }
 
-Handle<Object> TranslatedState::MaterializeAt(int frame_index,
-                                              int* value_index) {
-  CHECK_LT(static_cast<size_t>(frame_index), frames().size());
-  TranslatedFrame* frame = &(frames_[frame_index]);
-  CHECK_LT(static_cast<size_t>(*value_index), frame->values_.size());
+void TranslatedState::EnsurePropertiesAllocatedAndMarked(
+    TranslatedValue* properties_slot, Handle<Map> map) {
+  CHECK_EQ(TranslatedValue::kUninitialized,
+           properties_slot->materialization_state());
 
-  TranslatedValue* slot = &(frame->values_[*value_index]);
-  (*value_index)++;
+  Handle<ByteArray> object_storage = AllocateStorageFor(properties_slot);
+  properties_slot->mark_allocated();
+  properties_slot->set_storage(object_storage);
 
-  switch (slot->kind()) {
-    case TranslatedValue::kTagged:
-    case TranslatedValue::kInt32:
-    case TranslatedValue::kUInt32:
-    case TranslatedValue::kBoolBit:
-    case TranslatedValue::kFloat:
-    case TranslatedValue::kDouble: {
-      slot->MaterializeSimple();
-      Handle<Object> value = slot->GetValue();
-      if (value->IsMutableHeapNumber()) {
-        HeapNumber::cast(*value)->set_map(isolate()->heap()->heap_number_map());
-      }
-      return value;
+  // Set markers for the double properties.
+  Handle<DescriptorArray> descriptors(map->instance_descriptors());
+  int field_count = map->NumberOfOwnDescriptors();
+  for (int i = 0; i < field_count; i++) {
+    FieldIndex index = FieldIndex::ForDescriptor(*map, i);
+    if (descriptors->GetDetails(i).representation().IsDouble() &&
+        !index.is_inobject()) {
+      CHECK(!map->IsUnboxedDoubleField(index));
+      int outobject_index = index.outobject_array_index();
+      int array_index = outobject_index * kPointerSize;
+      object_storage->set(array_index, kStoreMutableHeapNumber);
     }
+  }
+}
 
-    case TranslatedValue::kCapturedObject: {
-      // The map must be a tagged object.
-      CHECK_EQ(frame->values_[*value_index].kind(), TranslatedValue::kTagged);
-      CHECK(frame->values_[*value_index].GetValue()->IsMap());
-      return MaterializeCapturedObjectAt(slot, frame_index, value_index);
+Handle<ByteArray> TranslatedState::AllocateStorageFor(TranslatedValue* slot) {
+  int allocate_size =
+      ByteArray::LengthFor(slot->GetChildrenCount() * kPointerSize);
+  // It is important to allocate all the objects tenured so that the marker
+  // does not visit them.
+  Handle<ByteArray> object_storage =
+      isolate()->factory()->NewByteArray(allocate_size, TENURED);
+  for (int i = 0; i < object_storage->length(); i++) {
+    object_storage->set(i, kStoreTagged);
+  }
+  return object_storage;
+}
+
+void TranslatedState::EnsureJSObjectAllocated(TranslatedValue* slot,
+                                              Handle<Map> map) {
+  CHECK_EQ(map->instance_size(), slot->GetChildrenCount() * kPointerSize);
+
+  Handle<ByteArray> object_storage = AllocateStorageFor(slot);
+  // Now we handle the interesting (JSObject) case.
+  Handle<DescriptorArray> descriptors(map->instance_descriptors());
+  int field_count = map->NumberOfOwnDescriptors();
+
+  // Set markers for the double properties.
+  for (int i = 0; i < field_count; i++) {
+    FieldIndex index = FieldIndex::ForDescriptor(*map, i);
+    if (descriptors->GetDetails(i).representation().IsDouble() &&
+        index.is_inobject()) {
+      CHECK_GE(index.index(), FixedArray::kHeaderSize / kPointerSize);
+      int array_index = index.index() * kPointerSize - FixedArray::kHeaderSize;
+      uint8_t marker = map->IsUnboxedDoubleField(index)
+                           ? kStoreUnboxedDouble
+                           : kStoreMutableHeapNumber;
+      object_storage->set(array_index, marker);
     }
-    case TranslatedValue::kDuplicatedObject: {
-      int object_index = slot->object_index();
-      TranslatedState::ObjectPosition pos = object_positions_[object_index];
+  }
+  slot->set_storage(object_storage);
+}
+
+Handle<Object> TranslatedState::GetValueAndAdvance(TranslatedFrame* frame,
+                                                   int* value_index) {
+  TranslatedValue* slot = frame->ValueAt(*value_index);
+  SkipSlots(1, frame, value_index);
+  if (slot->kind() == TranslatedValue::kDuplicatedObject) {
+    slot = ResolveCapturedObject(slot);
+  }
+  CHECK_NE(TranslatedValue::kUninitialized, slot->materialization_state());
+  return slot->GetStorage();
+}
 
-      // Make sure the duplicate is referring to a previous object.
-      CHECK(pos.frame_index_ < frame_index ||
-            (pos.frame_index_ == frame_index &&
-             pos.value_index_ < *value_index - 1));
+void TranslatedState::InitializeJSObjectAt(
+    TranslatedFrame* frame, int* value_index, TranslatedValue* slot,
+    Handle<Map> map, const DisallowHeapAllocation& no_allocation) {
+  Handle<HeapObject> object_storage = Handle<HeapObject>::cast(slot->storage_);
+  DCHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
 
-      Handle<Object> object =
-          frames_[pos.frame_index_].values_[pos.value_index_].GetValue();
+  // The object should have at least a map and some payload.
+  CHECK_GE(slot->GetChildrenCount(), 2);
 
-      // The object should have a (non-sentinel) value.
-      CHECK(!object.is_null() &&
-            !object.is_identical_to(isolate_->factory()->arguments_marker()));
+  // Notify the concurrent marker about the layout change.
+  isolate()->heap()->NotifyObjectLayoutChange(
+      *object_storage, slot->GetChildrenCount() * kPointerSize, no_allocation);
 
-      slot->value_ = object;
-      return object;
+  // Fill the property array field.
+  {
+    Handle<Object> properties = GetValueAndAdvance(frame, value_index);
+    WRITE_FIELD(*object_storage, JSObject::kPropertiesOrHashOffset,
+                *properties);
+    WRITE_BARRIER(isolate()->heap(), *object_storage,
+                  JSObject::kPropertiesOrHashOffset, *properties);
+  }
+
+  // For all the other fields we first look at the fixed array and check the
+  // marker to see if we store an unboxed double.
+  DCHECK_EQ(kPointerSize, JSObject::kPropertiesOrHashOffset);
+  for (int i = 2; i < slot->GetChildrenCount(); i++) {
+    // Initialize and extract the value from its slot.
+    Handle<Object> field_value = GetValueAndAdvance(frame, value_index);
+
+    // Read out the marker and ensure the field is consistent with
+    // what the markers in the storage say (note that all heap numbers
+    // should be fully initialized by now).
+    int offset = i * kPointerSize;
+    uint8_t marker = READ_UINT8_FIELD(*object_storage, offset);
+    if (marker == kStoreUnboxedDouble) {
+      double double_field_value;
+      if (field_value->IsSmi()) {
+        double_field_value = Smi::cast(*field_value)->value();
+      } else {
+        CHECK(field_value->IsHeapNumber());
+        double_field_value = HeapNumber::cast(*field_value)->value();
+      }
+      WRITE_DOUBLE_FIELD(*object_storage, offset, double_field_value);
+    } else if (marker == kStoreMutableHeapNumber) {
+      CHECK(field_value->IsMutableHeapNumber());
+      WRITE_FIELD(*object_storage, offset, *field_value);
+      WRITE_BARRIER(isolate()->heap(), *object_storage, offset, *field_value);
+    } else {
+      CHECK_EQ(kStoreTagged, marker);
+      WRITE_FIELD(*object_storage, offset, *field_value);
+      WRITE_BARRIER(isolate()->heap(), *object_storage, offset, *field_value);
     }
+  }
+  object_storage->synchronized_set_map(*map);
+}
 
-    case TranslatedValue::kInvalid:
-      UNREACHABLE();
-      break;
+void TranslatedState::InitializeObjectWithTaggedFieldsAt(
+    TranslatedFrame* frame, int* value_index, TranslatedValue* slot,
+    Handle<Map> map, const DisallowHeapAllocation& no_allocation) {
+  Handle<HeapObject> object_storage = Handle<HeapObject>::cast(slot->storage_);
+
+  // Notify the concurrent marker about the layout change.
+  isolate()->heap()->NotifyObjectLayoutChange(
+      *object_storage, slot->GetChildrenCount() * kPointerSize, no_allocation);
+
+  // Write the fields to the object.
+  for (int i = 1; i < slot->GetChildrenCount(); i++) {
+    Handle<Object> field_value = GetValueAndAdvance(frame, value_index);
+    int offset = i * kPointerSize;
+    uint8_t marker = READ_UINT8_FIELD(*object_storage, offset);
+    if (i > 1 && marker == kStoreMutableHeapNumber) {
+      CHECK(field_value->IsMutableHeapNumber());
+    } else {
+      CHECK(marker == kStoreTagged || i == 1);
+      CHECK(!field_value->IsMutableHeapNumber());
+    }
+
+    WRITE_FIELD(*object_storage, offset, *field_value);
+    WRITE_BARRIER(isolate()->heap(), *object_storage, offset, *field_value);
   }
 
-  FATAL("We should never get here - unexpected deopt slot kind.");
-  return Handle<Object>::null();
+  object_storage->synchronized_set_map(*map);
 }
 
-Handle<Object> TranslatedState::MaterializeObjectAt(int object_index) {
-  CHECK_LT(static_cast<size_t>(object_index), object_positions_.size());
-  TranslatedState::ObjectPosition pos = object_positions_[object_index];
-  return MaterializeAt(pos.frame_index_, &(pos.value_index_));
+TranslatedValue* TranslatedState::ResolveCapturedObject(TranslatedValue* slot) {
+  while (slot->kind() == TranslatedValue::kDuplicatedObject) {
+    slot = GetValueByObjectIndex(slot->object_index());
+  }
+  CHECK_EQ(TranslatedValue::kCapturedObject, slot->kind());
+  return slot;
 }
 
 TranslatedFrame* TranslatedState::GetFrameFromJSFrameIndex(int jsframe_index) {
@@ -3814,7 +3734,7 @@ void TranslatedState::StoreMaterializedValuesAndDeopt(JavaScriptFrame* frame) {
   bool new_store = false;
   if (previously_materialized_objects.is_null()) {
     previously_materialized_objects =
-        isolate_->factory()->NewFixedArray(length);
+        isolate_->factory()->NewFixedArray(length, TENURED);
     for (int i = 0; i < length; i++) {
       previously_materialized_objects->set(i, *marker);
     }
@@ -3831,6 +3751,10 @@ void TranslatedState::StoreMaterializedValuesAndDeopt(JavaScriptFrame* frame) {
 
     CHECK(value_info->IsMaterializedObject());
 
+    // Skip duplicate objects (i.e., those that point to some
+    // other object id).
+    if (value_info->object_index() != i) continue;
+
     Handle<Object> value(value_info->GetRawValue(), isolate_);
 
     if (!value.is_identical_to(marker)) {
@@ -3874,11 +3798,34 @@ void TranslatedState::UpdateFromPreviouslyMaterializedObjects() {
           &(frames_[pos.frame_index_].values_[pos.value_index_]);
       CHECK(value_info->IsMaterializedObject());
 
-      value_info->value_ =
-          Handle<Object>(previously_materialized_objects->get(i), isolate_);
+      if (value_info->kind() == TranslatedValue::kCapturedObject) {
+        value_info->set_initialized_storage(
+            Handle<Object>(previously_materialized_objects->get(i), isolate_));
+      }
+    }
+  }
+}
+
+void TranslatedState::VerifyMaterializedObjects() {
+#if VERIFY_HEAP
+  int length = static_cast<int>(object_positions_.size());
+  for (int i = 0; i < length; i++) {
+    TranslatedValue* slot = GetValueByObjectIndex(i);
+    if (slot->kind() == TranslatedValue::kCapturedObject) {
+      CHECK_EQ(slot, GetValueByObjectIndex(slot->object_index()));
+      if (slot->materialization_state() == TranslatedValue::kFinished) {
+        slot->GetStorage()->ObjectVerify();
+      } else {
+        CHECK_EQ(slot->materialization_state(),
+                 TranslatedValue::kUninitialized);
+      }
     }
   }
+#endif
 }
 
 }  // namespace internal
 }  // namespace v8
+
+// Undefine the heap manipulation macros.
+#include "src/objects/object-macros-undef.h"

src/deoptimizer.h

@@ -5,6 +5,7 @@
 #ifndef V8_DEOPTIMIZER_H_
 #define V8_DEOPTIMIZER_H_
 
+#include <stack>
 #include <vector>
 
 #include "src/allocation.h"
@@ -31,6 +32,9 @@ class TranslatedValue {
   // Returns heap()->arguments_marker() if allocation would be
   // necessary to get the value.
   Object* GetRawValue() const;
+
+  // Getter for the value, takes care of materializing the subgraph
+  // reachable from this value.
   Handle<Object> GetValue();
 
   bool IsMaterializedObject() const;
@@ -40,7 +44,7 @@ class TranslatedValue {
   friend class TranslatedState;
   friend class TranslatedFrame;
 
-  enum Kind {
+  enum Kind : uint8_t {
     kInvalid,
     kTagged,
     kInt32,
@@ -56,9 +60,20 @@ class TranslatedValue {
     kDuplicatedObject  // Duplicated object of a deferred object.
   };
 
+  enum MaterializationState : uint8_t {
+    kUninitialized,
+    kAllocated,  // Storage for the object has been allocated (or
+                 // enqueued for allocation).
+    kFinished,   // The object has been initialized (or enqueued for
+                 // initialization).
+  };
+
   TranslatedValue(TranslatedState* container, Kind kind)
       : kind_(kind), container_(container) {}
   Kind kind() const { return kind_; }
+  MaterializationState materialization_state() const {
+    return materialization_state_;
+  }
   void Handlify();
   int GetChildrenCount() const;
 
@@ -76,15 +91,25 @@ class TranslatedValue {
   Isolate* isolate() const;
   void MaterializeSimple();
 
+  void set_storage(Handle<HeapObject> storage) { storage_ = storage; }
+  void set_initialized_storage(Handle<Object> storage);
+  void mark_finished() { materialization_state_ = kFinished; }
+  void mark_allocated() { materialization_state_ = kAllocated; }
+
+  Handle<Object> GetStorage() {
+    DCHECK_NE(kUninitialized, materialization_state());
+    return storage_;
+  }
+
   Kind kind_;
+  MaterializationState materialization_state_ = kUninitialized;
   TranslatedState* container_;  // This is only needed for materialization of
                                 // objects and constructing handles (to get
                                 // to the isolate).
 
-  MaybeHandle<Object> value_;  // Before handlification, this is always null,
-                               // after materialization it is never null,
-                               // in between it is only null if the value needs
-                               // to be materialized.
+  Handle<Object> storage_;  // Contains the materialized value or the
+                            // byte-array that will be later morphed into
+                            // the materialized object.
 
   struct MaterializedObjectInfo {
     int id_;
@@ -211,6 +236,7 @@ class TranslatedFrame {
         height_(height) {}
 
   void Add(const TranslatedValue& value) { values_.push_back(value); }
+  TranslatedValue* ValueAt(int index) { return &(values_[index]); }
   void Handlify();
 
   Kind kind_;
@@ -270,6 +296,8 @@ class TranslatedState {
             FixedArray* literal_array, RegisterValues* registers,
             FILE* trace_file, int parameter_count);
 
+  void VerifyMaterializedObjects();
+
  private:
   friend TranslatedValue;
 
@@ -288,11 +316,36 @@ class TranslatedState {
                                                FILE* trace_file);
 
   void UpdateFromPreviouslyMaterializedObjects();
-  Handle<Object> MaterializeAt(int frame_index, int* value_index);
-  Handle<Object> MaterializeObjectAt(int object_index);
-  class CapturedObjectMaterializer;
-  Handle<Object> MaterializeCapturedObjectAt(TranslatedValue* slot,
-                                             int frame_index, int* value_index);
+  void MaterializeFixedDoubleArray(TranslatedFrame* frame, int* value_index,
+                                   TranslatedValue* slot, Handle<Map> map);
+  void MaterializeMutableHeapNumber(TranslatedFrame* frame, int* value_index,
+                                    TranslatedValue* slot);
+
+  void EnsureObjectAllocatedAt(TranslatedValue* slot);
+
+  void SkipSlots(int slots_to_skip, TranslatedFrame* frame, int* value_index);
+
+  Handle<ByteArray> AllocateStorageFor(TranslatedValue* slot);
+  void EnsureJSObjectAllocated(TranslatedValue* slot, Handle<Map> map);
+  void EnsurePropertiesAllocatedAndMarked(TranslatedValue* properties_slot,
+                                          Handle<Map> map);
+  void EnsureChildrenAllocated(int count, TranslatedFrame* frame,
+                               int* value_index, std::stack<int>* worklist);
+  void EnsureCapturedObjectAllocatedAt(int object_index,
+                                       std::stack<int>* worklist);
+  Handle<Object> InitializeObjectAt(TranslatedValue* slot);
+  void InitializeCapturedObjectAt(int object_index, std::stack<int>* worklist,
+                                  const DisallowHeapAllocation& no_allocation);
+  void InitializeJSObjectAt(TranslatedFrame* frame, int* value_index,
+                            TranslatedValue* slot, Handle<Map> map,
+                            const DisallowHeapAllocation& no_allocation);
+  void InitializeObjectWithTaggedFieldsAt(
+      TranslatedFrame* frame, int* value_index, TranslatedValue* slot,
+      Handle<Map> map, const DisallowHeapAllocation& no_allocation);
+
+  TranslatedValue* ResolveCapturedObject(TranslatedValue* slot);
+  TranslatedValue* GetValueByObjectIndex(int object_index);
+  Handle<Object> GetValueAndAdvance(TranslatedFrame* frame, int* value_index);
 
   static uint32_t GetUInt32Slot(Address fp, int slot_index);
   static Float32 GetFloatSlot(Address fp, int slot_index);

test/mjsunit/compiler/materialize-dictionary-properties.js

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --allow-natives-syntax
+
+function f() {
+  // Create a non-escaping object.
+  var o = Object.create(null);
+  %DeoptimizeNow();
+  // Keep it alive.
+  return o ? 1 : 0;
+}
+
+f();
+f();
+%OptimizeFunctionOnNextCall(f);
+assertEquals(1, f());

test/mjsunit/compiler/materialize-mutable-heap-number.js

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --allow-natives-syntax
+
+function C() {}
+%CompleteInobjectSlackTracking(new C());
+
+function f() {
+  // Create a non-escaping object.
+  var o = new C();
+  // Add an out-of-object double property.
+  o.x = 0.5;
+  %DeoptimizeNow();
+  return o.x + 0.25;
+}
+
+f();
+f();
+%OptimizeFunctionOnNextCall(f);
+assertEquals(0.75, f());