accessor-assembler.cc 162 KB
Newer Older
1 2 3 4 5 6
// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/ic/accessor-assembler.h"

7
#include "src/ast/ast.h"
8
#include "src/base/optional.h"
9
#include "src/codegen/code-factory.h"
10
#include "src/ic/handler-configuration.h"
11
#include "src/ic/ic.h"
12
#include "src/ic/keyed-store-generic.h"
13
#include "src/ic/stub-cache.h"
14
#include "src/logging/counters.h"
15
#include "src/objects/cell.h"
16
#include "src/objects/foreign.h"
17
#include "src/objects/heap-number.h"
18
#include "src/objects/module.h"
19
#include "src/objects/objects-inl.h"
20
#include "src/objects/property-details.h"
21
#include "src/objects/smi.h"
22 23 24 25 26

namespace v8 {
namespace internal {

using compiler::CodeAssemblerState;
27
using compiler::Node;
28 29 30

//////////////////// Private helpers.

31
// Loads dataX field from the DataHandler object.
32
TNode<MaybeObject> AccessorAssembler::LoadHandlerDataField(
33
    TNode<DataHandler> handler, int data_index) {
34
#ifdef DEBUG
35
  TNode<Map> handler_map = LoadMap(handler);
36
  TNode<Uint16T> instance_type = LoadMapInstanceType(handler_map);
37 38 39 40 41 42
#endif
  CSA_ASSERT(this,
             Word32Or(InstanceTypeEqual(instance_type, LOAD_HANDLER_TYPE),
                      InstanceTypeEqual(instance_type, STORE_HANDLER_TYPE)));
  int offset = 0;
  int minimum_size = 0;
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
  switch (data_index) {
    case 1:
      offset = DataHandler::kData1Offset;
      minimum_size = DataHandler::kSizeWithData1;
      break;
    case 2:
      offset = DataHandler::kData2Offset;
      minimum_size = DataHandler::kSizeWithData2;
      break;
    case 3:
      offset = DataHandler::kData3Offset;
      minimum_size = DataHandler::kSizeWithData3;
      break;
    default:
      UNREACHABLE();
58 59 60 61
  }
  USE(minimum_size);
  CSA_ASSERT(this, UintPtrGreaterThanOrEqual(
                       LoadMapInstanceSizeInWords(handler_map),
62
                       IntPtrConstant(minimum_size / kTaggedSize)));
63
  return LoadMaybeWeakObjectField(handler, offset);
64 65
}

66
TNode<MaybeObject> AccessorAssembler::TryMonomorphicCase(
67
    TNode<TaggedIndex> slot, TNode<FeedbackVector> vector,
68
    TNode<Map> lookup_start_object_map, Label* if_handler,
69
    TVariable<MaybeObject>* var_handler, Label* if_miss) {
70 71 72 73 74
  Comment("TryMonomorphicCase");
  DCHECK_EQ(MachineRepresentation::kTagged, var_handler->rep());

  // TODO(ishell): add helper class that hides offset computations for a series
  // of loads.
75
  int32_t header_size = FeedbackVector::kFeedbackSlotsOffset - kHeapObjectTag;
76 77 78
  // Adding |header_size| with a separate IntPtrAdd rather than passing it
  // into ElementOffsetFromIndex() allows it to be folded into a single
  // [base, index, offset] indirect memory access on x64.
79
  TNode<IntPtrT> offset = ElementOffsetFromIndex(slot, HOLEY_ELEMENTS);
80 81 82
  TNode<MaybeObject> feedback = ReinterpretCast<MaybeObject>(
      Load(MachineType::AnyTagged(), vector,
           IntPtrAdd(offset, IntPtrConstant(header_size))));
83 84

  // Try to quickly handle the monomorphic case without knowing for sure
85
  // if we have a weak reference in feedback.
86
  GotoIfNot(IsWeakReferenceTo(feedback, lookup_start_object_map), if_miss);
87

88
  TNode<MaybeObject> handler = UncheckedCast<MaybeObject>(
89
      Load(MachineType::AnyTagged(), vector,
90
           IntPtrAdd(offset, IntPtrConstant(header_size + kTaggedSize))));
91

92
  *var_handler = handler;
93 94 95 96
  Goto(if_handler);
  return feedback;
}

97
void AccessorAssembler::HandlePolymorphicCase(
98 99
    TNode<Map> lookup_start_object_map, TNode<WeakFixedArray> feedback,
    Label* if_handler, TVariable<MaybeObject>* var_handler, Label* if_miss) {
100 101 102 103 104 105
  Comment("HandlePolymorphicCase");
  DCHECK_EQ(MachineRepresentation::kTagged, var_handler->rep());

  // Iterate {feedback} array.
  const int kEntrySize = 2;

106 107
  // Load the {feedback} array length.
  TNode<IntPtrT> length = LoadAndUntagWeakFixedArrayLength(feedback);
108
  CSA_ASSERT(this, IntPtrLessThanOrEqual(IntPtrConstant(kEntrySize), length));
109

110 111 112 113
  // This is a hand-crafted loop that iterates backwards and only compares
  // against zero at the end, since we already know that we will have at least a
  // single entry in the {feedback} array anyways.
  TVARIABLE(IntPtrT, var_index, IntPtrSub(length, IntPtrConstant(kEntrySize)));
114 115 116 117
  Label loop(this, &var_index), loop_next(this);
  Goto(&loop);
  BIND(&loop);
  {
118
    TNode<MaybeObject> maybe_cached_map =
119
        LoadWeakFixedArrayElement(feedback, var_index.value());
120
    CSA_ASSERT(this, IsWeakOrCleared(maybe_cached_map));
121 122
    GotoIfNot(IsWeakReferenceTo(maybe_cached_map, lookup_start_object_map),
              &loop_next);
123 124

    // Found, now call handler.
125
    TNode<MaybeObject> handler =
126
        LoadWeakFixedArrayElement(feedback, var_index.value(), kTaggedSize);
127
    *var_handler = handler;
128 129
    Goto(if_handler);

130 131
    BIND(&loop_next);
    var_index =
132 133 134
        Signed(IntPtrSub(var_index.value(), IntPtrConstant(kEntrySize)));
    Branch(IntPtrGreaterThanOrEqual(var_index.value(), IntPtrConstant(0)),
           &loop, if_miss);
135 136 137
  }
}

138
void AccessorAssembler::HandleLoadICHandlerCase(
139
    const LazyLoadICParameters* p, TNode<Object> handler, Label* miss,
140
    ExitPoint* exit_point, ICMode ic_mode, OnNonExistent on_nonexistent,
141
    ElementSupport support_elements, LoadAccessMode access_mode) {
142
  Comment("have_handler");
143

144
  TVARIABLE(Object, var_holder, p->lookup_start_object());
145
  TVARIABLE(Object, var_smi_handler, handler);
146

147
  Label if_smi_handler(this, {&var_holder, &var_smi_handler});
jgruber's avatar
jgruber committed
148 149
  Label try_proto_handler(this, Label::kDeferred),
      call_handler(this, Label::kDeferred);
150 151 152

  Branch(TaggedIsSmi(handler), &if_smi_handler, &try_proto_handler);

153 154 155
  BIND(&try_proto_handler);
  {
    GotoIf(IsCodeMap(LoadMap(CAST(handler))), &call_handler);
156
    HandleLoadICProtoHandler(p, CAST(handler), &var_holder, &var_smi_handler,
157 158 159 160
                             &if_smi_handler, miss, exit_point, ic_mode,
                             access_mode);
  }

161 162
  // |handler| is a Smi, encoding what to do. See SmiHandler methods
  // for the encoding format.
163
  BIND(&if_smi_handler);
164
  {
165 166 167
    HandleLoadICSmiHandlerCase(
        p, var_holder.value(), CAST(var_smi_handler.value()), handler, miss,
        exit_point, ic_mode, on_nonexistent, support_elements, access_mode);
168 169
  }

170
  BIND(&call_handler);
171
  {
172
    exit_point->ReturnCallStub(LoadWithVectorDescriptor{}, CAST(handler),
173 174
                               p->context(), p->receiver(), p->name(),
                               p->slot(), p->vector());
175 176 177
  }
}

178 179 180
void AccessorAssembler::HandleLoadCallbackProperty(
    const LazyLoadICParameters* p, TNode<JSObject> holder,
    TNode<WordT> handler_word, ExitPoint* exit_point) {
181
  Comment("native_data_property_load");
182 183
  TNode<IntPtrT> descriptor =
      Signed(DecodeWord<LoadHandler::DescriptorBits>(handler_word));
184 185 186 187 188

  Callable callable = CodeFactory::ApiGetter(isolate());
  TNode<AccessorInfo> accessor_info =
      CAST(LoadDescriptorValue(LoadMap(holder), descriptor));

189
  exit_point->ReturnCallStub(callable, p->context(), p->receiver(), holder,
190 191 192 193
                             accessor_info);
}

void AccessorAssembler::HandleLoadAccessor(
194
    const LazyLoadICParameters* p, TNode<CallHandlerInfo> call_handler_info,
195 196 197 198 199 200 201 202 203 204
    TNode<WordT> handler_word, TNode<DataHandler> handler,
    TNode<IntPtrT> handler_kind, ExitPoint* exit_point) {
  Comment("api_getter");
  // Context is stored either in data2 or data3 field depending on whether
  // the access check is enabled for this handler or not.
  TNode<MaybeObject> maybe_context = Select<MaybeObject>(
      IsSetWord<LoadHandler::DoAccessCheckOnReceiverBits>(handler_word),
      [=] { return LoadHandlerDataField(handler, 3); },
      [=] { return LoadHandlerDataField(handler, 2); });

205 206
  CSA_ASSERT(this, IsWeakOrCleared(maybe_context));
  CSA_CHECK(this, IsNotCleared(maybe_context));
207
  TNode<HeapObject> context = GetHeapObjectAssumeWeak(maybe_context);
208

209 210 211
  TNode<Foreign> foreign = LoadObjectField<Foreign>(
      call_handler_info, CallHandlerInfo::kJsCallbackOffset);
  TNode<RawPtrT> callback =
212
      DecodeExternalPointer(LoadForeignForeignAddress(foreign));
213 214 215
  TNode<Object> data =
      LoadObjectField(call_handler_info, CallHandlerInfo::kDataOffset);

216
  TVARIABLE(HeapObject, api_holder, CAST(p->lookup_start_object()));
217 218 219 220 221 222 223 224 225
  Label load(this);
  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kApiGetter)),
         &load);

  CSA_ASSERT(
      this,
      WordEqual(handler_kind,
                IntPtrConstant(LoadHandler::kApiGetterHolderIsPrototype)));

226
  api_holder = LoadMapPrototype(LoadMap(CAST(p->lookup_start_object())));
227 228 229 230
  Goto(&load);

  BIND(&load);
  TNode<IntPtrT> argc = IntPtrConstant(0);
231 232
  exit_point->Return(CallApiCallback(context, callback, argc, data,
                                     api_holder.value(), p->receiver()));
233 234
}

235
void AccessorAssembler::HandleLoadField(TNode<JSObject> holder,
236
                                        TNode<WordT> handler_word,
237
                                        TVariable<Float64T>* var_double_value,
238
                                        Label* rebox_double, Label* miss,
239 240
                                        ExitPoint* exit_point) {
  Comment("field_load");
241 242 243
  TNode<IntPtrT> index =
      Signed(DecodeWord<LoadHandler::FieldIndexBits>(handler_word));
  TNode<IntPtrT> offset = IntPtrMul(index, IntPtrConstant(kTaggedSize));
244 245 246 247 248

  Label inobject(this), out_of_object(this);
  Branch(IsSetWord<LoadHandler::IsInobjectBits>(handler_word), &inobject,
         &out_of_object);

249
  BIND(&inobject);
250 251 252 253 254
  {
    Label is_double(this);
    GotoIf(IsSetWord<LoadHandler::IsDoubleBits>(handler_word), &is_double);
    exit_point->Return(LoadObjectField(holder, offset));

255
    BIND(&is_double);
256
    if (FLAG_unbox_double_fields) {
257
      *var_double_value = LoadObjectField<Float64T>(holder, offset);
258
    } else {
259 260 261 262 263 264
      TNode<Object> heap_number = LoadObjectField(holder, offset);
      // This is not an "old" Smi value from before a Smi->Double transition.
      // Rather, it's possible that since the last update of this IC, the Double
      // field transitioned to a Tagged field, and was then assigned a Smi.
      GotoIf(TaggedIsSmi(heap_number), miss);
      GotoIfNot(IsHeapNumber(CAST(heap_number)), miss);
265
      *var_double_value = LoadHeapNumberValue(CAST(heap_number));
266 267 268 269
    }
    Goto(rebox_double);
  }

270
  BIND(&out_of_object);
271 272
  {
    Label is_double(this);
273 274
    TNode<HeapObject> properties = LoadFastProperties(holder);
    TNode<Object> value = LoadObjectField(properties, offset);
275 276 277
    GotoIf(IsSetWord<LoadHandler::IsDoubleBits>(handler_word), &is_double);
    exit_point->Return(value);

278
    BIND(&is_double);
279 280 281 282 283 284 285
    if (!FLAG_unbox_double_fields) {
      // This is not an "old" Smi value from before a Smi->Double transition.
      // Rather, it's possible that since the last update of this IC, the Double
      // field transitioned to a Tagged field, and was then assigned a Smi.
      GotoIf(TaggedIsSmi(value), miss);
      GotoIfNot(IsHeapNumber(CAST(value)), miss);
    }
286
    *var_double_value = LoadHeapNumberValue(CAST(value));
287 288 289 290
    Goto(rebox_double);
  }
}

291 292 293
TNode<Object> AccessorAssembler::LoadDescriptorValue(
    TNode<Map> map, TNode<IntPtrT> descriptor_entry) {
  return CAST(LoadDescriptorValueOrFieldType(map, descriptor_entry));
294 295 296
}

TNode<MaybeObject> AccessorAssembler::LoadDescriptorValueOrFieldType(
297
    TNode<Map> map, TNode<IntPtrT> descriptor_entry) {
298
  TNode<DescriptorArray> descriptors = LoadMapDescriptors(map);
299
  return LoadFieldTypeByDescriptorEntry(descriptors, descriptor_entry);
300 301
}

302
void AccessorAssembler::HandleLoadICSmiHandlerCase(
303
    const LazyLoadICParameters* p, TNode<Object> holder, TNode<Smi> smi_handler,
304 305 306 307
    TNode<Object> handler, Label* miss, ExitPoint* exit_point, ICMode ic_mode,
    OnNonExistent on_nonexistent, ElementSupport support_elements,
    LoadAccessMode access_mode) {
  TVARIABLE(Float64T, var_double_value);
308 309
  Label rebox_double(this, &var_double_value);

310
  TNode<IntPtrT> handler_word = SmiUntag(smi_handler);
311 312
  TNode<IntPtrT> handler_kind =
      Signed(DecodeWord<LoadHandler::KindBits>(handler_word));
313

314
  if (support_elements == kSupportElements) {
315 316
    Label if_element(this), if_indexed_string(this), if_property(this),
        if_hole(this), unimplemented_elements_kind(this),
317 318 319
        if_oob(this, Label::kDeferred), try_string_to_array_index(this),
        emit_element_load(this);
    TVARIABLE(IntPtrT, var_intptr_index);
320 321
    GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kElement)),
           &if_element);
322 323 324 325 326 327 328 329 330 331 332

    if (access_mode == LoadAccessMode::kHas) {
      CSA_ASSERT(this,
                 WordNotEqual(handler_kind,
                              IntPtrConstant(LoadHandler::kIndexedString)));
      Goto(&if_property);
    } else {
      Branch(
          WordEqual(handler_kind, IntPtrConstant(LoadHandler::kIndexedString)),
          &if_indexed_string, &if_property);
    }
333

334
    BIND(&if_element);
335 336 337 338 339 340 341 342 343 344 345 346
    {
      Comment("element_load");
      TVARIABLE(Int32T, var_instance_type);
      TNode<IntPtrT> intptr_index = TryToIntptr(
          p->name(), &try_string_to_array_index, &var_instance_type);
      var_intptr_index = intptr_index;
      Goto(&emit_element_load);

      BIND(&try_string_to_array_index);
      {
        GotoIfNot(IsStringInstanceType(var_instance_type.value()), miss);

347
        TNode<ExternalReference> function = ExternalConstant(
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
            ExternalReference::string_to_array_index_function());
        TNode<Int32T> result = UncheckedCast<Int32T>(
            CallCFunction(function, MachineType::Int32(),
                          std::make_pair(MachineType::AnyTagged(), p->name())));
        GotoIf(Word32Equal(Int32Constant(-1), result), miss);
        CSA_ASSERT(this, Int32GreaterThanOrEqual(result, Int32Constant(0)));
        var_intptr_index = ChangeInt32ToIntPtr(result);

        Goto(&emit_element_load);
      }

      BIND(&emit_element_load);
      {
        TNode<BoolT> is_jsarray_condition =
            IsSetWord<LoadHandler::IsJsArrayBits>(handler_word);
        TNode<Uint32T> elements_kind =
            DecodeWord32FromWord<LoadHandler::ElementsKindBits>(handler_word);
365
        EmitElementLoad(CAST(holder), elements_kind, var_intptr_index.value(),
366 367 368 369 370
                        is_jsarray_condition, &if_hole, &rebox_double,
                        &var_double_value, &unimplemented_elements_kind,
                        &if_oob, miss, exit_point, access_mode);
      }
    }
371

372
    BIND(&unimplemented_elements_kind);
373 374 375 376 377 378 379
    {
      // Smi handlers should only be installed for supported elements kinds.
      // Crash if we get here.
      DebugBreak();
      Goto(miss);
    }

380 381 382 383 384 385
    BIND(&if_oob);
    {
      Comment("out of bounds elements access");
      Label return_undefined(this);

      // Check if we're allowed to handle OOB accesses.
386
      TNode<BoolT> allow_out_of_bounds =
387 388 389
          IsSetWord<LoadHandler::AllowOutOfBoundsBits>(handler_word);
      GotoIfNot(allow_out_of_bounds, miss);

390 391 392 393 394
      // Negative indices aren't valid array indices (according to
      // the ECMAScript specification), and are stored as properties
      // in V8, not elements. So we cannot handle them here, except
      // in case of typed arrays, where integer indexed properties
      // aren't looked up in the prototype chain.
395
      GotoIf(IsJSTypedArray(CAST(holder)), &return_undefined);
396
      if (Is64()) {
397 398 399 400
        GotoIfNot(
            UintPtrLessThanOrEqual(var_intptr_index.value(),
                                   IntPtrConstant(JSArray::kMaxArrayIndex)),
            miss);
401 402 403 404
      } else {
        GotoIf(IntPtrLessThan(var_intptr_index.value(), IntPtrConstant(0)),
               miss);
      }
405 406 407

      // For all other receivers we need to check that the prototype chain
      // doesn't contain any elements.
408
      BranchIfPrototypesHaveNoElements(LoadMap(CAST(holder)), &return_undefined,
409 410 411
                                       miss);

      BIND(&return_undefined);
412 413 414
      exit_point->Return(access_mode == LoadAccessMode::kHas
                             ? FalseConstant()
                             : UndefinedConstant());
415 416
    }

417
    BIND(&if_hole);
418 419
    {
      Comment("convert hole");
420

421
      GotoIfNot(IsSetWord<LoadHandler::ConvertHoleBits>(handler_word), miss);
422
      GotoIf(IsNoElementsProtectorCellInvalid(), miss);
423 424 425
      exit_point->Return(access_mode == LoadAccessMode::kHas
                             ? FalseConstant()
                             : UndefinedConstant());
426 427
    }

428 429 430 431 432 433
    if (access_mode != LoadAccessMode::kHas) {
      BIND(&if_indexed_string);
      {
        Label if_oob(this, Label::kDeferred);

        Comment("indexed string");
434
        TNode<String> string_holder = CAST(holder);
435 436 437 438 439
        TNode<UintPtrT> index = Unsigned(TryToIntptr(p->name(), miss));
        TNode<UintPtrT> length =
            Unsigned(LoadStringLengthAsWord(string_holder));
        GotoIf(UintPtrGreaterThanOrEqual(index, length), &if_oob);
        TNode<Int32T> code = StringCharCodeAt(string_holder, index);
440 441 442 443
        TNode<String> result = StringFromSingleCharCode(code);
        Return(result);

        BIND(&if_oob);
444
        TNode<BoolT> allow_out_of_bounds =
445 446 447 448 449
            IsSetWord<LoadHandler::AllowOutOfBoundsBits>(handler_word);
        GotoIfNot(allow_out_of_bounds, miss);
        GotoIf(IsNoElementsProtectorCellInvalid(), miss);
        Return(UndefinedConstant());
      }
450 451 452
    }

    BIND(&if_property);
453 454 455
    Comment("property_load");
  }

456 457
  if (access_mode == LoadAccessMode::kHas) {
    HandleLoadICSmiHandlerHasNamedCase(p, holder, handler_kind, miss,
458
                                       exit_point, ic_mode);
459 460 461
  } else {
    HandleLoadICSmiHandlerLoadNamedCase(
        p, holder, handler_kind, handler_word, &rebox_double, &var_double_value,
462
        handler, miss, exit_point, ic_mode, on_nonexistent, support_elements);
463 464 465 466
  }
}

void AccessorAssembler::HandleLoadICSmiHandlerLoadNamedCase(
467
    const LazyLoadICParameters* p, TNode<Object> holder,
468
    TNode<IntPtrT> handler_kind, TNode<WordT> handler_word, Label* rebox_double,
469
    TVariable<Float64T>* var_double_value, TNode<Object> handler, Label* miss,
470
    ExitPoint* exit_point, ICMode ic_mode, OnNonExistent on_nonexistent,
471
    ElementSupport support_elements) {
472
  Label constant(this), field(this), normal(this, Label::kDeferred),
473 474 475 476
      slow(this, Label::kDeferred), interceptor(this, Label::kDeferred),
      nonexistent(this), accessor(this, Label::kDeferred),
      global(this, Label::kDeferred), module_export(this, Label::kDeferred),
      proxy(this, Label::kDeferred),
477 478
      native_data_property(this, Label::kDeferred),
      api_getter(this, Label::kDeferred);
479

480
  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kField)), &field);
481

482 483
  GotoIf(WordEqual(handler_kind,
                   IntPtrConstant(LoadHandler::kConstantFromPrototype)),
484 485 486 487 488
         &constant);

  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kNonExistent)),
         &nonexistent);

489 490 491
  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kNormal)),
         &normal);

492 493 494
  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kAccessor)),
         &accessor);

495 496 497 498 499 500 501 502 503 504 505
  GotoIf(
      WordEqual(handler_kind, IntPtrConstant(LoadHandler::kNativeDataProperty)),
      &native_data_property);

  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kApiGetter)),
         &api_getter);

  GotoIf(WordEqual(handler_kind,
                   IntPtrConstant(LoadHandler::kApiGetterHolderIsPrototype)),
         &api_getter);

506 507 508
  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kGlobal)),
         &global);

509 510
  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kSlow)), &slow);

511 512
  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kProxy)), &proxy);

513 514
  Branch(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kModuleExport)),
         &module_export, &interceptor);
515

516
  BIND(&field);
517 518
  HandleLoadField(CAST(holder), handler_word, var_double_value, rebox_double,
                  miss, exit_point);
519

520
  BIND(&nonexistent);
521
  // This is a handler for a load of a non-existent value.
522
  if (on_nonexistent == OnNonExistent::kThrowReferenceError) {
523 524
    exit_point->ReturnCallRuntime(Runtime::kThrowReferenceError, p->context(),
                                  p->name());
525
  } else {
526
    DCHECK_EQ(OnNonExistent::kReturnUndefined, on_nonexistent);
527 528 529
    exit_point->Return(UndefinedConstant());
  }

530
  BIND(&constant);
531 532
  {
    Comment("constant_load");
533
    exit_point->Return(holder);
534
  }
535

536
  BIND(&normal);
537 538
  {
    Comment("load_normal");
539
    TNode<NameDictionary> properties = CAST(LoadSlowProperties(CAST(holder)));
540
    TVARIABLE(IntPtrT, var_name_index);
541
    Label found(this, &var_name_index);
542
    NameDictionaryLookup<NameDictionary>(properties, CAST(p->name()), &found,
543
                                         &var_name_index, miss);
544
    BIND(&found);
545
    {
546 547
      TVARIABLE(Uint32T, var_details);
      TVARIABLE(Object, var_value);
548 549
      LoadPropertyFromNameDictionary(properties, var_name_index.value(),
                                     &var_details, &var_value);
550 551 552
      TNode<Object> value = CallGetterIfAccessor(
          var_value.value(), CAST(holder), var_details.value(), p->context(),
          p->receiver(), miss);
553 554 555 556
      exit_point->Return(value);
    }
  }

557
  BIND(&accessor);
558 559
  {
    Comment("accessor_load");
560 561
    TNode<IntPtrT> descriptor =
        Signed(DecodeWord<LoadHandler::DescriptorBits>(handler_word));
562
    TNode<AccessorPair> accessor_pair =
563
        CAST(LoadDescriptorValue(LoadMap(CAST(holder)), descriptor));
564 565
    TNode<Object> getter =
        LoadObjectField(accessor_pair, AccessorPair::kGetterOffset);
566 567
    CSA_ASSERT(this, Word32BinaryNot(IsTheHole(getter)));

568
    exit_point->Return(Call(p->context(), getter, p->receiver()));
569 570
  }

571
  BIND(&native_data_property);
572
  HandleLoadCallbackProperty(p, CAST(holder), handler_word, exit_point);
573 574

  BIND(&api_getter);
575 576
  HandleLoadAccessor(p, CAST(holder), handler_word, CAST(handler), handler_kind,
                     exit_point);
577

578 579
  BIND(&proxy);
  {
580 581 582 583 584 585 586 587
    // TODO(mythria): LoadGlobals don't use this path. LoadGlobals need special
    // handling with proxies which is currently not supported by builtins. So
    // for such cases, we should install a slow path and never reach here. Fix
    // it to not generate this for LoadGlobals.
    CSA_ASSERT(this,
               WordNotEqual(IntPtrConstant(static_cast<int>(on_nonexistent)),
                            IntPtrConstant(static_cast<int>(
                                OnNonExistent::kThrowReferenceError))));
588 589
    TVARIABLE(IntPtrT, var_index);
    TVARIABLE(Name, var_unique);
590 591 592 593 594

    Label if_index(this), if_unique_name(this),
        to_name_failed(this, Label::kDeferred);

    if (support_elements == kSupportElements) {
595 596
      DCHECK_NE(on_nonexistent, OnNonExistent::kThrowReferenceError);

597
      TryToName(p->name(), &if_index, &var_index, &if_unique_name, &var_unique,
598 599 600 601 602
                &to_name_failed);

      BIND(&if_unique_name);
      exit_point->ReturnCallStub(
          Builtins::CallableFor(isolate(), Builtins::kProxyGetProperty),
603
          p->context(), holder, var_unique.value(), p->receiver(),
604
          SmiConstant(on_nonexistent));
605 606 607 608 609 610 611

      BIND(&if_index);
      // TODO(mslekova): introduce TryToName that doesn't try to compute
      // the intptr index value
      Goto(&to_name_failed);

      BIND(&to_name_failed);
612 613
      // TODO(duongn): use GetPropertyWithReceiver builtin once
      // |lookup_element_in_holder| supports elements.
614
      exit_point->ReturnCallRuntime(Runtime::kGetPropertyWithReceiver,
615 616
                                    p->context(), holder, p->name(),
                                    p->receiver(), SmiConstant(on_nonexistent));
617 618 619
    } else {
      exit_point->ReturnCallStub(
          Builtins::CallableFor(isolate(), Builtins::kProxyGetProperty),
620
          p->context(), holder, p->name(), p->receiver(),
621
          SmiConstant(on_nonexistent));
622
    }
623 624
  }

625
  BIND(&global);
626
  {
627
    CSA_ASSERT(this, IsPropertyCell(CAST(holder)));
628
    // Ensure the property cell doesn't contain the hole.
629 630
    TNode<Object> value =
        LoadObjectField(CAST(holder), PropertyCell::kValueOffset);
631 632
    TNode<Uint32T> details = Unsigned(LoadAndUntagToWord32ObjectField(
        CAST(holder), PropertyCell::kPropertyDetailsRawOffset));
633 634
    GotoIf(IsTheHole(value), miss);

635 636
    exit_point->Return(CallGetterIfAccessor(value, CAST(holder), details,
                                            p->context(), p->receiver(), miss));
637 638
  }

639
  BIND(&interceptor);
640 641 642
  {
    Comment("load_interceptor");
    exit_point->ReturnCallRuntime(Runtime::kLoadPropertyWithInterceptor,
643 644
                                  p->context(), p->name(), p->receiver(),
                                  holder, p->slot(), p->vector());
645
  }
646 647 648 649 650 651 652 653
  BIND(&slow);
  {
    Comment("load_slow");
    if (ic_mode == ICMode::kGlobalIC) {
      exit_point->ReturnCallRuntime(Runtime::kLoadGlobalIC_Slow, p->context(),
                                    p->name(), p->slot(), p->vector());

    } else {
654 655
      exit_point->ReturnCallRuntime(Runtime::kGetProperty, p->context(), holder,
                                    p->name(), p->receiver());
656 657
    }
  }
658

659 660 661
  BIND(&module_export);
  {
    Comment("module export");
662 663
    TNode<UintPtrT> index =
        DecodeWord<LoadHandler::ExportsIndexBits>(handler_word);
664 665
    TNode<Module> module = LoadObjectField<Module>(
        CAST(p->receiver()), JSModuleNamespace::kModuleOffset);
666 667
    TNode<ObjectHashTable> exports =
        LoadObjectField<ObjectHashTable>(module, Module::kExportsOffset);
668
    TNode<Cell> cell = CAST(LoadFixedArrayElement(exports, index));
669
    // The handler is only installed for exports that exist.
670
    TNode<Object> value = LoadCellValue(cell);
671 672 673 674 675 676
    Label is_the_hole(this, Label::kDeferred);
    GotoIf(IsTheHole(value), &is_the_hole);
    exit_point->Return(value);

    BIND(&is_the_hole);
    {
677
      TNode<Smi> message = SmiConstant(MessageTemplate::kNotDefined);
678 679
      exit_point->ReturnCallRuntime(Runtime::kThrowReferenceError, p->context(),
                                    message, p->name());
680 681 682
    }
  }

683 684 685 686 687
  BIND(rebox_double);
  exit_point->Return(AllocateHeapNumberWithValue(var_double_value->value()));
}

void AccessorAssembler::HandleLoadICSmiHandlerHasNamedCase(
688
    const LazyLoadICParameters* p, TNode<Object> holder,
689 690
    TNode<IntPtrT> handler_kind, Label* miss, ExitPoint* exit_point,
    ICMode ic_mode) {
691
  Label return_true(this), return_false(this), return_lookup(this),
692
      normal(this), global(this), slow(this);
693 694 695 696

  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kField)),
         &return_true);

697 698
  GotoIf(WordEqual(handler_kind,
                   IntPtrConstant(LoadHandler::kConstantFromPrototype)),
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720
         &return_true);

  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kNonExistent)),
         &return_false);

  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kNormal)),
         &normal);

  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kAccessor)),
         &return_true);

  GotoIf(
      WordEqual(handler_kind, IntPtrConstant(LoadHandler::kNativeDataProperty)),
      &return_true);

  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kApiGetter)),
         &return_true);

  GotoIf(WordEqual(handler_kind,
                   IntPtrConstant(LoadHandler::kApiGetterHolderIsPrototype)),
         &return_true);

721 722
  GotoIf(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kSlow)), &slow);

723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
  Branch(WordEqual(handler_kind, IntPtrConstant(LoadHandler::kGlobal)), &global,
         &return_lookup);

  BIND(&return_true);
  exit_point->Return(TrueConstant());

  BIND(&return_false);
  exit_point->Return(FalseConstant());

  BIND(&return_lookup);
  {
    CSA_ASSERT(
        this,
        Word32Or(
            WordEqual(handler_kind, IntPtrConstant(LoadHandler::kInterceptor)),
            Word32Or(
                WordEqual(handler_kind, IntPtrConstant(LoadHandler::kProxy)),
                WordEqual(handler_kind,
                          IntPtrConstant(LoadHandler::kModuleExport)))));
    exit_point->ReturnCallStub(
743 744
        Builtins::CallableFor(isolate(), Builtins::kHasProperty), p->context(),
        p->receiver(), p->name());
745 746 747 748 749
  }

  BIND(&normal);
  {
    Comment("has_normal");
750
    TNode<NameDictionary> properties = CAST(LoadSlowProperties(CAST(holder)));
751 752
    TVARIABLE(IntPtrT, var_name_index);
    Label found(this);
753
    NameDictionaryLookup<NameDictionary>(properties, CAST(p->name()), &found,
754 755 756 757 758 759 760 761
                                         &var_name_index, miss);

    BIND(&found);
    exit_point->Return(TrueConstant());
  }

  BIND(&global);
  {
762
    CSA_ASSERT(this, IsPropertyCell(CAST(holder)));
763
    // Ensure the property cell doesn't contain the hole.
764 765
    TNode<Object> value =
        LoadObjectField(CAST(holder), PropertyCell::kValueOffset);
766 767 768 769
    GotoIf(IsTheHole(value), miss);

    exit_point->Return(TrueConstant());
  }
770 771 772 773 774 775 776 777 778 779 780 781

  BIND(&slow);
  {
    Comment("load_slow");
    if (ic_mode == ICMode::kGlobalIC) {
      exit_point->ReturnCallRuntime(Runtime::kLoadGlobalIC_Slow, p->context(),
                                    p->name(), p->slot(), p->vector());
    } else {
      exit_point->ReturnCallRuntime(Runtime::kHasProperty, p->context(),
                                    p->receiver(), p->name());
    }
  }
782 783
}

784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
// Performs actions common to both load and store handlers:
// 1. Checks prototype validity cell.
// 2. If |on_code_handler| is provided, then it checks if the sub handler is
//    a smi or code and if it's a code then it calls |on_code_handler| to
//    generate a code that handles Code handlers.
//    If |on_code_handler| is not provided, then only smi sub handler are
//    expected.
// 3. Does access check on receiver if ICHandler::DoAccessCheckOnReceiverBits
//    bit is set in the smi handler.
// 4. Does dictionary lookup on receiver if ICHandler::LookupOnReceiverBits bit
//    is set in the smi handler. If |on_found_on_receiver| is provided then
//    it calls it to generate a code that handles the "found on receiver case"
//    or just misses if the |on_found_on_receiver| is not provided.
// 5. Falls through in a case of a smi handler which is returned from this
//    function (tagged!).
// TODO(ishell): Remove templatezation once we move common bits from
// Load/StoreHandler to the base class.
template <typename ICHandler, typename ICParameters>
802 803 804
TNode<Object> AccessorAssembler::HandleProtoHandler(
    const ICParameters* p, TNode<DataHandler> handler,
    const OnCodeHandler& on_code_handler,
805
    const OnFoundOnReceiver& on_found_on_receiver, Label* miss,
806
    ICMode ic_mode) {
807 808 809 810
  //
  // Check prototype validity cell.
  //
  {
811
    TNode<Object> maybe_validity_cell =
812
        LoadObjectField(handler, ICHandler::kValidityCellOffset);
813
    CheckPrototypeValidityCell(maybe_validity_cell, miss);
814
  }
815

816 817 818 819
  //
  // Check smi handler bits.
  //
  {
820
    TNode<Object> smi_or_code_handler =
821 822 823 824
        LoadObjectField(handler, ICHandler::kSmiHandlerOffset);
    if (on_code_handler) {
      Label if_smi_handler(this);
      GotoIf(TaggedIsSmi(smi_or_code_handler), &if_smi_handler);
825

826
      on_code_handler(CAST(smi_or_code_handler));
827

828 829
      BIND(&if_smi_handler);
    }
830
    TNode<IntPtrT> handler_flags = SmiUntag(CAST(smi_or_code_handler));
831 832 833 834 835 836 837

    // Lookup on receiver and access checks are not necessary for global ICs
    // because in the former case the validity cell check guards modifications
    // of the global object and the latter is not applicable to the global
    // object.
    int mask = ICHandler::LookupOnReceiverBits::kMask |
               ICHandler::DoAccessCheckOnReceiverBits::kMask;
838
    if (ic_mode == ICMode::kGlobalIC) {
839
      CSA_ASSERT(this, IsClearWord(handler_flags, mask));
840 841
    } else {
      DCHECK_EQ(ICMode::kNonGlobalIC, ic_mode);
842 843 844 845 846 847 848

      Label done(this), if_do_access_check(this), if_lookup_on_receiver(this);
      GotoIf(IsClearWord(handler_flags, mask), &done);
      // Only one of the bits can be set at a time.
      CSA_ASSERT(this,
                 WordNotEqual(WordAnd(handler_flags, IntPtrConstant(mask)),
                              IntPtrConstant(mask)));
849 850
      Branch(IsSetWord<LoadHandler::DoAccessCheckOnReceiverBits>(handler_flags),
             &if_do_access_check, &if_lookup_on_receiver);
851

852 853
      BIND(&if_do_access_check);
      {
854
        TNode<MaybeObject> data2 = LoadHandlerDataField(handler, 2);
855
        CSA_ASSERT(this, IsWeakOrCleared(data2));
856 857
        TNode<Context> expected_native_context =
            CAST(GetHeapObjectAssumeWeak(data2, miss));
858 859
        EmitAccessCheck(expected_native_context, p->context(), p->receiver(),
                        &done, miss);
860 861 862 863 864 865 866 867 868
      }

      // Dictionary lookup on receiver is not necessary for Load/StoreGlobalIC
      // because prototype validity cell check already guards modifications of
      // the global object.
      BIND(&if_lookup_on_receiver);
      {
        DCHECK_EQ(ICMode::kNonGlobalIC, ic_mode);
        CSA_ASSERT(this, Word32BinaryNot(HasInstanceType(
869
                             CAST(p->receiver()), JS_GLOBAL_OBJECT_TYPE)));
870

871
        TNode<NameDictionary> properties =
872
            CAST(LoadSlowProperties(CAST(p->receiver())));
873
        TVARIABLE(IntPtrT, var_name_index);
874
        Label found(this, &var_name_index);
875 876
        NameDictionaryLookup<NameDictionary>(properties, CAST(p->name()),
                                             &found, &var_name_index, &done);
877 878 879 880 881 882 883 884 885 886 887
        BIND(&found);
        {
          if (on_found_on_receiver) {
            on_found_on_receiver(properties, var_name_index.value());
          } else {
            Goto(miss);
          }
        }
      }

      BIND(&done);
888
    }
889 890 891
    return smi_or_code_handler;
  }
}
892

893
void AccessorAssembler::HandleLoadICProtoHandler(
894
    const LazyLoadICParameters* p, TNode<DataHandler> handler,
895 896
    TVariable<Object>* var_holder, TVariable<Object>* var_smi_handler,
    Label* if_smi_handler, Label* miss, ExitPoint* exit_point, ICMode ic_mode,
897 898
    LoadAccessMode access_mode) {
  TNode<Smi> smi_handler = CAST(HandleProtoHandler<LoadHandler>(
899 900 901 902
      p, handler,
      // Code sub-handlers are not expected in LoadICs, so no |on_code_handler|.
      nullptr,
      // on_found_on_receiver
903
      [=](TNode<NameDictionary> properties, TNode<IntPtrT> name_index) {
904 905 906
        if (access_mode == LoadAccessMode::kHas) {
          exit_point->Return(TrueConstant());
        } else {
907 908
          TVARIABLE(Uint32T, var_details);
          TVARIABLE(Object, var_value);
909 910
          LoadPropertyFromNameDictionary(properties, name_index, &var_details,
                                         &var_value);
911 912 913
          TNode<Object> value = CallGetterIfAccessor(
              var_value.value(), CAST(var_holder->value()), var_details.value(),
              p->context(), p->receiver(), miss);
914 915
          exit_point->Return(value);
        }
916
      },
917
      miss, ic_mode));
918

919 920
  TNode<MaybeObject> maybe_holder_or_constant =
      LoadHandlerDataField(handler, 1);
921

922
  Label load_from_cached_holder(this), is_smi(this), done(this);
923

924
  GotoIf(TaggedIsSmi(maybe_holder_or_constant), &is_smi);
925
  Branch(TaggedEqual(maybe_holder_or_constant, NullConstant()), &done,
926
         &load_from_cached_holder);
927

928
  BIND(&is_smi);
929
  {
930 931 932 933 934 935 936 937
    CSA_ASSERT(
        this,
        WordEqual(
            Signed(DecodeWord<LoadHandler::KindBits>(SmiUntag(smi_handler))),
            IntPtrConstant(LoadHandler::kConstantFromPrototype)));
    if (access_mode == LoadAccessMode::kHas) {
      exit_point->Return(TrueConstant());
    } else {
938
      exit_point->Return(CAST(maybe_holder_or_constant));
939 940
    }
  }
941

942 943 944 945 946 947 948 949 950
  BIND(&load_from_cached_holder);
  {
    // For regular holders, having passed the receiver map check and
    // the validity cell check implies that |holder| is
    // alive. However, for global object receivers, |maybe_holder| may
    // be cleared.
    CSA_ASSERT(this, IsWeakOrCleared(maybe_holder_or_constant));
    TNode<HeapObject> holder =
        GetHeapObjectAssumeWeak(maybe_holder_or_constant, miss);
951
    *var_holder = holder;
952
    Goto(&done);
953
  }
954

955
  BIND(&done);
956
  {
957
    *var_smi_handler = smi_handler;
958 959
    Goto(if_smi_handler);
  }
960 961
}

962
void AccessorAssembler::EmitAccessCheck(TNode<Context> expected_native_context,
963 964
                                        TNode<Context> context,
                                        TNode<Object> receiver,
965 966 967
                                        Label* can_access, Label* miss) {
  CSA_ASSERT(this, IsNativeContext(expected_native_context));

968
  TNode<NativeContext> native_context = LoadNativeContext(context);
969
  GotoIf(TaggedEqual(expected_native_context, native_context), can_access);
970
  // If the receiver is not a JSGlobalProxy then we miss.
971
  GotoIfNot(IsJSGlobalProxy(CAST(receiver)), miss);
972 973
  // For JSGlobalProxy receiver try to compare security tokens of current
  // and expected native contexts.
974 975 976
  TNode<Object> expected_token = LoadContextElement(
      expected_native_context, Context::SECURITY_TOKEN_INDEX);
  TNode<Object> current_token =
977
      LoadContextElement(native_context, Context::SECURITY_TOKEN_INDEX);
978
  Branch(TaggedEqual(expected_token, current_token), can_access, miss);
979 980
}

981 982
void AccessorAssembler::JumpIfDataProperty(TNode<Uint32T> details,
                                           Label* writable, Label* readonly) {
983 984 985 986 987 988 989 990
  if (readonly) {
    // Accessor properties never have the READ_ONLY attribute set.
    GotoIf(IsSetWord32(details, PropertyDetails::kAttributesReadOnlyMask),
           readonly);
  } else {
    CSA_ASSERT(this, IsNotSetWord32(details,
                                    PropertyDetails::kAttributesReadOnlyMask));
  }
991
  TNode<Uint32T> kind = DecodeWord32<PropertyDetails::KindField>(details);
992 993 994 995
  GotoIf(Word32Equal(kind, Int32Constant(kData)), writable);
  // Fall through if it's an accessor property.
}

996
void AccessorAssembler::HandleStoreICNativeDataProperty(
997
    const StoreICParameters* p, TNode<HeapObject> holder,
998
    TNode<Word32T> handler_word) {
999
  Comment("native_data_property_store");
1000
  TNode<IntPtrT> descriptor =
1001
      Signed(DecodeWordFromWord32<StoreHandler::DescriptorBits>(handler_word));
1002 1003
  TNode<AccessorInfo> accessor_info =
      CAST(LoadDescriptorValue(LoadMap(holder), descriptor));
1004

1005 1006
  TailCallRuntime(Runtime::kStoreCallbackProperty, p->context(), p->receiver(),
                  holder, accessor_info, p->name(), p->value());
1007 1008
}

1009
void AccessorAssembler::HandleStoreICHandlerCase(
1010 1011
    const StoreICParameters* p, TNode<MaybeObject> handler, Label* miss,
    ICMode ic_mode, ElementSupport support_elements) {
1012
  Label if_smi_handler(this), if_nonsmi_handler(this);
1013
  Label if_proto_handler(this), call_handler(this),
1014
      store_transition_or_global(this);
1015

1016
  Branch(TaggedIsSmi(handler), &if_smi_handler, &if_nonsmi_handler);
1017 1018 1019

  // |handler| is a Smi, encoding what to do. See SmiHandler methods
  // for the encoding format.
1020
  BIND(&if_smi_handler);
1021
  {
1022
    TNode<Object> holder = p->receiver();
1023
    TNode<Int32T> handler_word = SmiToInt32(CAST(handler));
1024

1025 1026
    Label if_fast_smi(this), if_proxy(this), if_interceptor(this),
        if_slow(this);
1027

1028
    STATIC_ASSERT(StoreHandler::kGlobalProxy + 1 == StoreHandler::kNormal);
1029 1030 1031
    STATIC_ASSERT(StoreHandler::kNormal + 1 == StoreHandler::kInterceptor);
    STATIC_ASSERT(StoreHandler::kInterceptor + 1 == StoreHandler::kSlow);
    STATIC_ASSERT(StoreHandler::kSlow + 1 == StoreHandler::kProxy);
1032 1033
    STATIC_ASSERT(StoreHandler::kProxy + 1 == StoreHandler::kKindsNumber);

1034 1035 1036 1037 1038 1039
    TNode<Uint32T> handler_kind =
        DecodeWord32<StoreHandler::KindBits>(handler_word);
    GotoIf(
        Int32LessThan(handler_kind, Int32Constant(StoreHandler::kGlobalProxy)),
        &if_fast_smi);
    GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kProxy)),
1040
           &if_proxy);
1041
    GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kInterceptor)),
1042
           &if_interceptor);
1043
    GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kSlow)),
1044
           &if_slow);
1045
    CSA_ASSERT(this,
1046
               Word32Equal(handler_kind, Int32Constant(StoreHandler::kNormal)));
1047
    TNode<NameDictionary> properties = CAST(LoadSlowProperties(CAST(holder)));
1048

1049
    TVARIABLE(IntPtrT, var_name_index);
1050
    Label dictionary_found(this, &var_name_index);
1051
    NameDictionaryLookup<NameDictionary>(
1052
        properties, CAST(p->name()), &dictionary_found, &var_name_index, miss);
1053
    BIND(&dictionary_found);
1054
    {
1055 1056
      TNode<Uint32T> details =
          LoadDetailsByKeyIndex(properties, var_name_index.value());
1057 1058 1059 1060 1061 1062 1063
      // Check that the property is a writable data property (no accessor).
      const int kTypeAndReadOnlyMask = PropertyDetails::KindField::kMask |
                                       PropertyDetails::kAttributesReadOnlyMask;
      STATIC_ASSERT(kData == 0);
      GotoIf(IsSetWord32(details, kTypeAndReadOnlyMask), miss);

      StoreValueByKeyIndex<NameDictionary>(properties, var_name_index.value(),
1064 1065
                                           p->value());
      Return(p->value());
1066 1067
    }

1068
    BIND(&if_fast_smi);
1069
    {
1070 1071
      TNode<Uint32T> handler_kind =
          DecodeWord32<StoreHandler::KindBits>(handler_word);
1072

1073
      Label data(this), accessor(this), native_data_property(this);
1074
      GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kAccessor)),
1075
             &accessor);
1076 1077
      Branch(Word32Equal(handler_kind,
                         Int32Constant(StoreHandler::kNativeDataProperty)),
1078
             &native_data_property, &data);
1079

1080
      BIND(&accessor);
1081
      HandleStoreAccessor(p, CAST(holder), handler_word);
1082 1083

      BIND(&native_data_property);
1084
      HandleStoreICNativeDataProperty(p, CAST(holder), handler_word);
1085 1086 1087

      BIND(&data);
      // Handle non-transitioning field stores.
1088
      HandleStoreICSmiHandlerCase(handler_word, CAST(holder), p->value(), miss);
1089
    }
1090 1091

    BIND(&if_proxy);
1092
    HandleStoreToProxy(p, CAST(holder), miss, support_elements);
1093 1094 1095 1096 1097

    BIND(&if_interceptor);
    {
      Comment("store_interceptor");
      TailCallRuntime(Runtime::kStorePropertyWithInterceptor, p->context(),
1098
                      p->value(), p->receiver(), p->name());
1099 1100 1101 1102 1103 1104 1105 1106
    }

    BIND(&if_slow);
    {
      Comment("store_slow");
      // The slow case calls into the runtime to complete the store without
      // causing an IC miss that would otherwise cause a transition to the
      // generic stub.
1107 1108 1109 1110 1111 1112 1113
      if (ic_mode == ICMode::kGlobalIC) {
        TailCallRuntime(Runtime::kStoreGlobalIC_Slow, p->context(), p->value(),
                        p->slot(), p->vector(), p->receiver(), p->name());
      } else {
        TailCallRuntime(Runtime::kKeyedStoreIC_Slow, p->context(), p->value(),
                        p->receiver(), p->name());
      }
1114
    }
1115 1116
  }

1117
  BIND(&if_nonsmi_handler);
1118
  {
1119
    GotoIf(IsWeakOrCleared(handler), &store_transition_or_global);
1120
    TNode<HeapObject> strong_handler = CAST(handler);
1121
    TNode<Map> handler_map = LoadMap(strong_handler);
1122
    Branch(IsCodeMap(handler_map), &call_handler, &if_proto_handler);
1123

1124 1125 1126 1127 1128
    BIND(&if_proto_handler);
    {
      HandleStoreICProtoHandler(p, CAST(strong_handler), miss, ic_mode,
                                support_elements);
    }
1129

1130 1131 1132
    // |handler| is a heap object. Must be code, call it.
    BIND(&call_handler);
    {
1133
      TailCallStub(StoreWithVectorDescriptor{}, CAST(strong_handler),
1134 1135
                   p->context(), p->receiver(), p->name(), p->value(),
                   p->slot(), p->vector());
1136
    }
1137
  }
1138

1139
  BIND(&store_transition_or_global);
1140 1141
  {
    // Load value or miss if the {handler} weak cell is cleared.
1142 1143 1144
    CSA_ASSERT(this, IsWeakOrCleared(handler));
    TNode<HeapObject> map_or_property_cell =
        GetHeapObjectAssumeWeak(handler, miss);
1145 1146 1147 1148 1149 1150 1151 1152

    Label store_global(this), store_transition(this);
    Branch(IsMap(map_or_property_cell), &store_transition, &store_global);

    BIND(&store_global);
    {
      TNode<PropertyCell> property_cell = CAST(map_or_property_cell);
      ExitPoint direct_exit(this);
1153
      StoreGlobalIC_PropertyCellCase(property_cell, p->value(), &direct_exit,
1154 1155 1156 1157 1158
                                     miss);
    }
    BIND(&store_transition);
    {
      TNode<Map> map = CAST(map_or_property_cell);
1159 1160
      HandleStoreICTransitionMapHandlerCase(p, map, miss,
                                            kCheckPrototypeValidity);
1161
      Return(p->value());
1162
    }
1163
  }
1164 1165 1166
}

void AccessorAssembler::HandleStoreICTransitionMapHandlerCase(
1167
    const StoreICParameters* p, TNode<Map> transition_map, Label* miss,
1168 1169 1170
    StoreTransitionMapFlags flags) {
  DCHECK_EQ(0, flags & ~kStoreTransitionMapFlagsMask);
  if (flags & kCheckPrototypeValidity) {
1171
    TNode<Object> maybe_validity_cell =
1172 1173 1174
        LoadObjectField(transition_map, Map::kPrototypeValidityCellOffset);
    CheckPrototypeValidityCell(maybe_validity_cell, miss);
  }
1175 1176

  TNode<Uint32T> bitfield3 = LoadMapBitField3(transition_map);
1177 1178
  CSA_ASSERT(this, IsClearWord32<Map::Bits3::IsDictionaryMapBit>(bitfield3));
  GotoIf(IsSetWord32<Map::Bits3::IsDeprecatedBit>(bitfield3), miss);
1179 1180

  // Load last descriptor details.
1181
  TNode<UintPtrT> nof =
1182
      DecodeWordFromWord32<Map::Bits3::NumberOfOwnDescriptorsBits>(bitfield3);
1183
  CSA_ASSERT(this, WordNotEqual(nof, IntPtrConstant(0)));
1184
  TNode<DescriptorArray> descriptors = LoadMapDescriptors(transition_map);
1185

1186
  TNode<IntPtrT> factor = IntPtrConstant(DescriptorArray::kEntrySize);
1187 1188
  TNode<IntPtrT> last_key_index = UncheckedCast<IntPtrT>(IntPtrAdd(
      IntPtrConstant(DescriptorArray::ToKeyIndex(-1)), IntPtrMul(nof, factor)));
1189
  if (flags & kValidateTransitionHandler) {
1190
    TNode<Name> key = LoadKeyByKeyIndex(descriptors, last_key_index);
1191
    GotoIf(TaggedNotEqual(key, p->name()), miss);
1192
  } else {
1193 1194
    CSA_ASSERT(this, TaggedEqual(LoadKeyByKeyIndex(descriptors, last_key_index),
                                 p->name()));
1195
  }
1196
  TNode<Uint32T> details = LoadDetailsByKeyIndex(descriptors, last_key_index);
1197
  if (flags & kValidateTransitionHandler) {
1198 1199 1200 1201
    // Follow transitions only in the following cases:
    // 1) name is a non-private symbol and attributes equal to NONE,
    // 2) name is a private symbol and attributes equal to DONT_ENUM.
    Label attributes_ok(this);
1202 1203
    const int kKindAndAttributesDontDeleteReadOnlyMask =
        PropertyDetails::KindField::kMask |
1204 1205
        PropertyDetails::kAttributesDontDeleteMask |
        PropertyDetails::kAttributesReadOnlyMask;
1206 1207 1208 1209 1210
    STATIC_ASSERT(kData == 0);
    // Both DontDelete and ReadOnly attributes must not be set and it has to be
    // a kData property.
    GotoIf(IsSetWord32(details, kKindAndAttributesDontDeleteReadOnlyMask),
           miss);
1211 1212 1213 1214

    // DontEnum attribute is allowed only for private symbols and vice versa.
    Branch(Word32Equal(
               IsSetWord32(details, PropertyDetails::kAttributesDontEnumMask),
1215
               IsPrivateSymbol(CAST(p->name()))),
1216 1217 1218 1219 1220
           &attributes_ok, miss);

    BIND(&attributes_ok);
  }

1221 1222 1223
  OverwriteExistingFastDataProperty(CAST(p->receiver()), transition_map,
                                    descriptors, last_key_index, details,
                                    p->value(), miss, true);
1224 1225
}

1226
void AccessorAssembler::CheckFieldType(TNode<DescriptorArray> descriptors,
1227
                                       TNode<IntPtrT> name_index,
1228
                                       TNode<Word32T> representation,
1229
                                       TNode<Object> value, Label* bailout) {
1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252
  Label r_smi(this), r_double(this), r_heapobject(this), all_fine(this);
  // Ignore FLAG_track_fields etc. and always emit code for all checks,
  // because this builtin is part of the snapshot and therefore should
  // be flag independent.
  GotoIf(Word32Equal(representation, Int32Constant(Representation::kSmi)),
         &r_smi);
  GotoIf(Word32Equal(representation, Int32Constant(Representation::kDouble)),
         &r_double);
  GotoIf(
      Word32Equal(representation, Int32Constant(Representation::kHeapObject)),
      &r_heapobject);
  GotoIf(Word32Equal(representation, Int32Constant(Representation::kNone)),
         bailout);
  CSA_ASSERT(this, Word32Equal(representation,
                               Int32Constant(Representation::kTagged)));
  Goto(&all_fine);

  BIND(&r_smi);
  { Branch(TaggedIsSmi(value), &all_fine, bailout); }

  BIND(&r_double);
  {
    GotoIf(TaggedIsSmi(value), &all_fine);
1253
    Branch(IsHeapNumber(CAST(value)), &all_fine, bailout);
1254 1255 1256 1257 1258
  }

  BIND(&r_heapobject);
  {
    GotoIf(TaggedIsSmi(value), bailout);
1259 1260
    TNode<MaybeObject> field_type =
        LoadFieldTypeByKeyIndex(descriptors, name_index);
1261 1262
    const Address kNoneType = FieldType::None().ptr();
    const Address kAnyType = FieldType::Any().ptr();
1263 1264
    DCHECK_NE(static_cast<uint32_t>(kNoneType), kClearedWeakHeapObjectLower32);
    DCHECK_NE(static_cast<uint32_t>(kAnyType), kClearedWeakHeapObjectLower32);
1265
    // FieldType::None can't hold any value.
1266 1267 1268
    GotoIf(
        TaggedEqual(field_type, BitcastWordToTagged(IntPtrConstant(kNoneType))),
        bailout);
1269
    // FieldType::Any can hold any value.
1270 1271 1272
    GotoIf(
        TaggedEqual(field_type, BitcastWordToTagged(IntPtrConstant(kAnyType))),
        &all_fine);
1273 1274
    // Cleared weak references count as FieldType::None, which can't hold any
    // value.
1275 1276
    TNode<Map> field_type_map =
        CAST(GetHeapObjectAssumeWeak(field_type, bailout));
1277
    // FieldType::Class(...) performs a map check.
1278 1279
    Branch(TaggedEqual(LoadMap(CAST(value)), field_type_map), &all_fine,
           bailout);
1280 1281 1282 1283 1284
  }

  BIND(&all_fine);
}

1285
TNode<BoolT> AccessorAssembler::IsPropertyDetailsConst(TNode<Uint32T> details) {
1286 1287 1288 1289
  return Word32Equal(DecodeWord32<PropertyDetails::ConstnessField>(details),
                     Int32Constant(static_cast<int32_t>(VariableMode::kConst)));
}

1290
void AccessorAssembler::OverwriteExistingFastDataProperty(
1291
    TNode<HeapObject> object, TNode<Map> object_map,
1292 1293 1294
    TNode<DescriptorArray> descriptors, TNode<IntPtrT> descriptor_name_index,
    TNode<Uint32T> details, TNode<Object> value, Label* slow,
    bool do_transitioning_store) {
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
  Label done(this), if_field(this), if_descriptor(this);

  CSA_ASSERT(this,
             Word32Equal(DecodeWord32<PropertyDetails::KindField>(details),
                         Int32Constant(kData)));

  Branch(Word32Equal(DecodeWord32<PropertyDetails::LocationField>(details),
                     Int32Constant(kField)),
         &if_field, &if_descriptor);

  BIND(&if_field);
  {
1307
    TNode<Uint32T> representation =
1308 1309
        DecodeWord32<PropertyDetails::RepresentationField>(details);

1310 1311
    CheckFieldType(descriptors, descriptor_name_index, representation, value,
                   slow);
1312

1313
    TNode<UintPtrT> field_index =
1314
        DecodeWordFromWord32<PropertyDetails::FieldIndexField>(details);
1315 1316 1317 1318 1319
    field_index = Unsigned(
        IntPtrAdd(field_index,
                  Unsigned(LoadMapInobjectPropertiesStartInWords(object_map))));
    TNode<IntPtrT> instance_size_in_words =
        LoadMapInstanceSizeInWords(object_map);
1320 1321 1322 1323 1324 1325 1326

    Label inobject(this), backing_store(this);
    Branch(UintPtrLessThan(field_index, instance_size_in_words), &inobject,
           &backing_store);

    BIND(&inobject);
    {
1327
      TNode<IntPtrT> field_offset = Signed(TimesTaggedSize(field_index));
1328 1329 1330 1331 1332 1333
      Label tagged_rep(this), double_rep(this);
      Branch(
          Word32Equal(representation, Int32Constant(Representation::kDouble)),
          &double_rep, &tagged_rep);
      BIND(&double_rep);
      {
1334
        TNode<Float64T> double_value = ChangeNumberToFloat64(CAST(value));
1335 1336 1337
        if (FLAG_unbox_double_fields) {
          if (do_transitioning_store) {
            StoreMap(object, object_map);
1338
          } else {
1339 1340
            Label store_value(this);
            GotoIfNot(IsPropertyDetailsConst(details), &store_value);
1341
            TNode<Float64T> current_value =
1342
                LoadObjectField<Float64T>(object, field_offset);
1343 1344 1345
            BranchIfSameNumberValue(current_value, double_value, &store_value,
                                    slow);
            BIND(&store_value);
1346
          }
1347
          StoreObjectFieldNoWriteBarrier(object, field_offset, double_value);
1348 1349
        } else {
          if (do_transitioning_store) {
1350 1351
            TNode<HeapNumber> heap_number =
                AllocateHeapNumberWithValue(double_value);
1352
            StoreMap(object, object_map);
1353
            StoreObjectField(object, field_offset, heap_number);
1354
          } else {
1355 1356
            TNode<HeapNumber> heap_number =
                CAST(LoadObjectField(object, field_offset));
1357 1358
            Label store_value(this);
            GotoIfNot(IsPropertyDetailsConst(details), &store_value);
1359
            TNode<Float64T> current_value = LoadHeapNumberValue(heap_number);
1360 1361 1362
            BranchIfSameNumberValue(current_value, double_value, &store_value,
                                    slow);
            BIND(&store_value);
1363
            StoreHeapNumberValue(heap_number, double_value);
1364 1365 1366 1367 1368 1369 1370 1371 1372
          }
        }
        Goto(&done);
      }

      BIND(&tagged_rep);
      {
        if (do_transitioning_store) {
          StoreMap(object, object_map);
1373
        } else {
1374 1375
          Label if_mutable(this);
          GotoIfNot(IsPropertyDetailsConst(details), &if_mutable);
1376
          TNode<Object> current_value = LoadObjectField(object, field_offset);
1377 1378
          BranchIfSameValue(current_value, value, &done, slow,
                            SameValueMode::kNumbersOnly);
1379
          BIND(&if_mutable);
1380 1381 1382 1383 1384 1385 1386 1387
        }
        StoreObjectField(object, field_offset, value);
        Goto(&done);
      }
    }

    BIND(&backing_store);
    {
1388 1389
      TNode<IntPtrT> backing_store_index =
          Signed(IntPtrSub(field_index, instance_size_in_words));
1390 1391 1392 1393 1394

      if (do_transitioning_store) {
        // Allocate mutable heap number before extending properties backing
        // store to ensure that heap verifier will not see the heap in
        // inconsistent state.
1395
        TVARIABLE(Object, var_value, value);
1396 1397 1398 1399 1400 1401
        {
          Label cont(this);
          GotoIf(Word32NotEqual(representation,
                                Int32Constant(Representation::kDouble)),
                 &cont);
          {
1402 1403 1404
            TNode<Float64T> double_value = ChangeNumberToFloat64(CAST(value));
            TNode<HeapNumber> heap_number =
                AllocateHeapNumberWithValue(double_value);
1405
            var_value = heap_number;
1406 1407 1408 1409 1410
            Goto(&cont);
          }
          BIND(&cont);
        }

1411
        TNode<PropertyArray> properties =
1412
            ExtendPropertiesBackingStore(object, backing_store_index);
1413 1414
        StorePropertyArrayElement(properties, backing_store_index,
                                  var_value.value());
1415 1416 1417 1418 1419
        StoreMap(object, object_map);
        Goto(&done);

      } else {
        Label tagged_rep(this), double_rep(this);
1420 1421
        TNode<PropertyArray> properties =
            CAST(LoadFastProperties(CAST(object)));
1422 1423 1424 1425 1426
        Branch(
            Word32Equal(representation, Int32Constant(Representation::kDouble)),
            &double_rep, &tagged_rep);
        BIND(&double_rep);
        {
1427 1428
          TNode<HeapNumber> heap_number =
              CAST(LoadPropertyArrayElement(properties, backing_store_index));
1429
          TNode<Float64T> double_value = ChangeNumberToFloat64(CAST(value));
1430 1431 1432

          Label if_mutable(this);
          GotoIfNot(IsPropertyDetailsConst(details), &if_mutable);
1433
          TNode<Float64T> current_value = LoadHeapNumberValue(heap_number);
1434 1435 1436
          BranchIfSameNumberValue(current_value, double_value, &done, slow);

          BIND(&if_mutable);
1437
          StoreHeapNumberValue(heap_number, double_value);
1438 1439 1440 1441
          Goto(&done);
        }
        BIND(&tagged_rep);
        {
1442 1443 1444 1445 1446 1447 1448 1449
          Label if_mutable(this);
          GotoIfNot(IsPropertyDetailsConst(details), &if_mutable);
          TNode<Object> current_value =
              LoadPropertyArrayElement(properties, backing_store_index);
          BranchIfSameValue(current_value, value, &done, slow,
                            SameValueMode::kNumbersOnly);

          BIND(&if_mutable);
1450
          StorePropertyArrayElement(properties, backing_store_index, value);
1451 1452 1453 1454 1455 1456 1457 1458 1459
          Goto(&done);
        }
      }
    }
  }

  BIND(&if_descriptor);
  {
    // Check that constant matches value.
1460 1461
    TNode<Object> constant =
        LoadValueByKeyIndex(descriptors, descriptor_name_index);
1462
    GotoIf(TaggedNotEqual(value, constant), slow);
1463 1464 1465 1466 1467

    if (do_transitioning_store) {
      StoreMap(object, object_map);
    }
    Goto(&done);
1468
  }
1469
  BIND(&done);
1470 1471
}

1472 1473
void AccessorAssembler::CheckPrototypeValidityCell(
    TNode<Object> maybe_validity_cell, Label* miss) {
1474
  Label done(this);
1475 1476 1477
  GotoIf(
      TaggedEqual(maybe_validity_cell, SmiConstant(Map::kPrototypeChainValid)),
      &done);
1478 1479
  CSA_ASSERT(this, TaggedIsNotSmi(maybe_validity_cell));

1480 1481 1482
  TNode<Object> cell_value =
      LoadObjectField(CAST(maybe_validity_cell), Cell::kValueOffset);
  Branch(TaggedEqual(cell_value, SmiConstant(Map::kPrototypeChainValid)), &done,
1483 1484 1485 1486 1487
         miss);

  BIND(&done);
}

1488
void AccessorAssembler::HandleStoreAccessor(const StoreICParameters* p,
1489
                                            TNode<HeapObject> holder,
1490
                                            TNode<Word32T> handler_word) {
1491
  Comment("accessor_store");
1492
  TNode<IntPtrT> descriptor =
1493
      Signed(DecodeWordFromWord32<StoreHandler::DescriptorBits>(handler_word));
1494 1495
  TNode<HeapObject> accessor_pair =
      CAST(LoadDescriptorValue(LoadMap(holder), descriptor));
1496
  CSA_ASSERT(this, IsAccessorPair(accessor_pair));
1497 1498
  TNode<Object> setter =
      LoadObjectField(accessor_pair, AccessorPair::kSetterOffset);
1499 1500 1501
  CSA_ASSERT(this, Word32BinaryNot(IsTheHole(setter)));

  Callable callable = CodeFactory::Call(isolate());
1502
  Return(Call(p->context(), setter, p->receiver(), p->value()));
1503 1504
}

1505
void AccessorAssembler::HandleStoreICProtoHandler(
1506 1507
    const StoreICParameters* p, TNode<StoreHandler> handler, Label* miss,
    ICMode ic_mode, ElementSupport support_elements) {
1508 1509
  Comment("HandleStoreICProtoHandler");

1510 1511 1512
  OnCodeHandler on_code_handler;
  if (support_elements == kSupportElements) {
    // Code sub-handlers are expected only in KeyedStoreICs.
1513
    on_code_handler = [=](TNode<Code> code_handler) {
1514 1515 1516 1517 1518 1519
      // This is either element store or transitioning element store.
      Label if_element_store(this), if_transitioning_element_store(this);
      Branch(IsStoreHandler0Map(LoadMap(handler)), &if_element_store,
             &if_transitioning_element_store);
      BIND(&if_element_store);
      {
1520 1521 1522
        TailCallStub(StoreWithVectorDescriptor{}, code_handler, p->context(),
                     p->receiver(), p->name(), p->value(), p->slot(),
                     p->vector());
1523 1524 1525 1526
      }

      BIND(&if_transitioning_element_store);
      {
1527 1528 1529
        TNode<MaybeObject> maybe_transition_map =
            LoadHandlerDataField(handler, 1);
        TNode<Map> transition_map =
1530
            CAST(GetHeapObjectAssumeWeak(maybe_transition_map, miss));
1531 1532 1533

        GotoIf(IsDeprecatedMap(transition_map), miss);

1534 1535 1536
        TailCallStub(StoreTransitionDescriptor{}, code_handler, p->context(),
                     p->receiver(), p->name(), transition_map, p->value(),
                     p->slot(), p->vector());
1537 1538 1539 1540
      }
    };
  }

1541
  TNode<Object> smi_handler = HandleProtoHandler<StoreHandler>(
1542
      p, handler, on_code_handler,
1543
      // on_found_on_receiver
1544
      [=](TNode<NameDictionary> properties, TNode<IntPtrT> name_index) {
1545
        TNode<Uint32T> details = LoadDetailsByKeyIndex(properties, name_index);
1546 1547 1548 1549 1550 1551
        // Check that the property is a writable data property (no accessor).
        const int kTypeAndReadOnlyMask =
            PropertyDetails::KindField::kMask |
            PropertyDetails::kAttributesReadOnlyMask;
        STATIC_ASSERT(kData == 0);
        GotoIf(IsSetWord32(details, kTypeAndReadOnlyMask), miss);
1552

1553 1554
        StoreValueByKeyIndex<NameDictionary>(properties, name_index,
                                             p->value());
1555
        Return(p->value());
1556 1557
      },
      miss, ic_mode);
1558 1559

  {
1560
    Label if_add_normal(this), if_store_global_proxy(this), if_api_setter(this),
1561
        if_accessor(this), if_native_data_property(this), if_slow(this);
1562

1563
    CSA_ASSERT(this, TaggedIsSmi(smi_handler));
1564
    TNode<Int32T> handler_word = SmiToInt32(CAST(smi_handler));
1565

1566 1567 1568
    TNode<Uint32T> handler_kind =
        DecodeWord32<StoreHandler::KindBits>(handler_word);
    GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kNormal)),
1569
           &if_add_normal);
1570

1571 1572 1573
    GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kSlow)),
           &if_slow);

1574
    TNode<MaybeObject> maybe_holder = LoadHandlerDataField(handler, 1);
1575
    CSA_ASSERT(this, IsWeakOrCleared(maybe_holder));
1576
    TNode<HeapObject> holder = GetHeapObjectAssumeWeak(maybe_holder, miss);
1577

1578
    GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kGlobalProxy)),
1579
           &if_store_global_proxy);
1580

1581
    GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kAccessor)),
1582 1583
           &if_accessor);

1584 1585
    GotoIf(Word32Equal(handler_kind,
                       Int32Constant(StoreHandler::kNativeDataProperty)),
1586 1587
           &if_native_data_property);

1588
    GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kApiSetter)),
1589 1590
           &if_api_setter);

1591 1592 1593 1594
    GotoIf(
        Word32Equal(handler_kind,
                    Int32Constant(StoreHandler::kApiSetterHolderIsPrototype)),
        &if_api_setter);
1595

1596
    CSA_ASSERT(this,
1597
               Word32Equal(handler_kind, Int32Constant(StoreHandler::kProxy)));
1598
    HandleStoreToProxy(p, CAST(holder), miss, support_elements);
1599

1600 1601 1602 1603 1604 1605
    BIND(&if_slow);
    {
      Comment("store_slow");
      // The slow case calls into the runtime to complete the store without
      // causing an IC miss that would otherwise cause a transition to the
      // generic stub.
1606 1607 1608 1609 1610 1611 1612
      if (ic_mode == ICMode::kGlobalIC) {
        TailCallRuntime(Runtime::kStoreGlobalIC_Slow, p->context(), p->value(),
                        p->slot(), p->vector(), p->receiver(), p->name());
      } else {
        TailCallRuntime(Runtime::kKeyedStoreIC_Slow, p->context(), p->value(),
                        p->receiver(), p->name());
      }
1613 1614
    }

1615 1616 1617
    BIND(&if_add_normal);
    {
      // This is a case of "transitioning store" to a dictionary mode object
1618 1619
      // when the property does not exist. The "existing property" case is
      // covered above by LookupOnReceiver bit handling of the smi handler.
1620
      Label slow(this);
1621
      TNode<Map> receiver_map = LoadMap(CAST(p->receiver()));
1622 1623
      InvalidateValidityCellIfPrototype(receiver_map);

1624
      TNode<NameDictionary> properties =
1625
          CAST(LoadSlowProperties(CAST(p->receiver())));
1626 1627
      Add<NameDictionary>(properties, CAST(p->name()), p->value(), &slow);
      Return(p->value());
1628 1629

      BIND(&slow);
1630 1631
      TailCallRuntime(Runtime::kAddDictionaryProperty, p->context(),
                      p->receiver(), p->name(), p->value());
1632 1633
    }

1634 1635 1636
    BIND(&if_accessor);
    HandleStoreAccessor(p, holder, handler_word);

1637 1638 1639
    BIND(&if_native_data_property);
    HandleStoreICNativeDataProperty(p, holder, handler_word);

1640 1641 1642
    BIND(&if_api_setter);
    {
      Comment("api_setter");
1643
      CSA_ASSERT(this, TaggedIsNotSmi(handler));
1644
      TNode<CallHandlerInfo> call_handler_info = CAST(holder);
1645 1646 1647

      // Context is stored either in data2 or data3 field depending on whether
      // the access check is enabled for this handler or not.
1648
      TNode<MaybeObject> maybe_context = Select<MaybeObject>(
1649
          IsSetWord32<LoadHandler::DoAccessCheckOnReceiverBits>(handler_word),
1650 1651 1652
          [=] { return LoadHandlerDataField(handler, 3); },
          [=] { return LoadHandlerDataField(handler, 2); });

1653 1654 1655 1656
      CSA_ASSERT(this, IsWeakOrCleared(maybe_context));
      TNode<Object> context = Select<Object>(
          IsCleared(maybe_context), [=] { return SmiConstant(0); },
          [=] { return GetHeapObjectAssumeWeak(maybe_context); });
1657

1658 1659 1660
      TNode<Foreign> foreign = LoadObjectField<Foreign>(
          call_handler_info, CallHandlerInfo::kJsCallbackOffset);
      TNode<RawPtrT> callback =
1661
          DecodeExternalPointer(LoadForeignForeignAddress(foreign));
1662
      TNode<Object> data =
1663 1664
          LoadObjectField(call_handler_info, CallHandlerInfo::kDataOffset);

1665
      TVARIABLE(Object, api_holder, p->receiver());
1666
      Label store(this);
1667
      GotoIf(Word32Equal(handler_kind, Int32Constant(StoreHandler::kApiSetter)),
1668 1669
             &store);

1670 1671 1672 1673
      CSA_ASSERT(this,
                 Word32Equal(
                     handler_kind,
                     Int32Constant(StoreHandler::kApiSetterHolderIsPrototype)));
1674

1675
      api_holder = LoadMapPrototype(LoadMap(CAST(p->receiver())));
1676 1677 1678
      Goto(&store);

      BIND(&store);
1679
      TNode<IntPtrT> argc = IntPtrConstant(1);
1680 1681
      Return(CallApiCallback(context, callback, argc, data, api_holder.value(),
                             p->receiver(), p->value()));
1682 1683
    }

1684 1685 1686
    BIND(&if_store_global_proxy);
    {
      ExitPoint direct_exit(this);
1687 1688
      StoreGlobalIC_PropertyCellCase(CAST(holder), p->value(), &direct_exit,
                                     miss);
1689
    }
1690 1691 1692
  }
}

1693
void AccessorAssembler::HandleStoreToProxy(const StoreICParameters* p,
1694
                                           TNode<JSProxy> proxy, Label* miss,
1695
                                           ElementSupport support_elements) {
1696 1697
  TVARIABLE(IntPtrT, var_index);
  TVARIABLE(Name, var_unique);
1698

1699
  Label if_index(this), if_unique_name(this),
1700
      to_name_failed(this, Label::kDeferred);
1701

1702
  if (support_elements == kSupportElements) {
1703
    TryToName(p->name(), &if_index, &var_index, &if_unique_name, &var_unique,
1704 1705 1706
              &to_name_failed);

    BIND(&if_unique_name);
1707 1708 1709
    CallBuiltin(Builtins::kProxySetProperty, p->context(), proxy,
                var_unique.value(), p->value(), p->receiver());
    Return(p->value());
1710 1711 1712 1713 1714 1715 1716 1717

    // The index case is handled earlier by the runtime.
    BIND(&if_index);
    // TODO(mslekova): introduce TryToName that doesn't try to compute
    // the intptr index value
    Goto(&to_name_failed);

    BIND(&to_name_failed);
1718 1719
    TailCallRuntime(Runtime::kSetPropertyWithReceiver, p->context(), proxy,
                    p->name(), p->value(), p->receiver());
1720
  } else {
1721 1722
    TNode<Object> name =
        CallBuiltin(Builtins::kToName, p->context(), p->name());
1723 1724
    TailCallBuiltin(Builtins::kProxySetProperty, p->context(), proxy, name,
                    p->value(), p->receiver());
1725 1726 1727
  }
}

1728 1729 1730 1731
void AccessorAssembler::HandleStoreICSmiHandlerCase(TNode<Word32T> handler_word,
                                                    TNode<JSObject> holder,
                                                    TNode<Object> value,
                                                    Label* miss) {
1732
  Comment("field store");
1733
#ifdef DEBUG
1734 1735
  TNode<Uint32T> handler_kind =
      DecodeWord32<StoreHandler::KindBits>(handler_word);
1736 1737 1738
  CSA_ASSERT(
      this,
      Word32Or(
1739 1740
          Word32Equal(handler_kind, Int32Constant(StoreHandler::kField)),
          Word32Equal(handler_kind, Int32Constant(StoreHandler::kConstField))));
1741 1742
#endif

1743 1744
  TNode<Uint32T> field_representation =
      DecodeWord32<StoreHandler::RepresentationBits>(handler_word);
1745 1746 1747 1748

  Label if_smi_field(this), if_double_field(this), if_heap_object_field(this),
      if_tagged_field(this);

1749 1750 1751 1752 1753 1754
  int32_t case_values[] = {Representation::kTagged, Representation::kHeapObject,
                           Representation::kSmi};
  Label* case_labels[] = {&if_tagged_field, &if_heap_object_field,
                          &if_smi_field};

  Switch(field_representation, &if_double_field, case_values, case_labels, 3);
1755

1756
  BIND(&if_tagged_field);
1757 1758
  {
    Comment("store tagged field");
1759 1760
    HandleStoreFieldAndReturn(handler_word, holder, value, base::nullopt,
                              Representation::Tagged(), miss);
1761 1762
  }

1763
  BIND(&if_heap_object_field);
1764
  {
1765 1766 1767
    Comment("heap object field checks");
    CheckHeapObjectTypeMatchesDescriptor(handler_word, holder, value, miss);

1768
    Comment("store heap object field");
1769 1770
    HandleStoreFieldAndReturn(handler_word, holder, value, base::nullopt,
                              Representation::HeapObject(), miss);
1771 1772
  }

1773
  BIND(&if_smi_field);
1774
  {
1775 1776 1777
    Comment("smi field checks");
    GotoIfNot(TaggedIsSmi(value), miss);

1778
    Comment("store smi field");
1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793
    HandleStoreFieldAndReturn(handler_word, holder, value, base::nullopt,
                              Representation::Smi(), miss);
  }

  BIND(&if_double_field);
  {
    CSA_ASSERT(this, Word32Equal(field_representation,
                                 Int32Constant(Representation::kDouble)));
    Comment("double field checks");
    TNode<Float64T> double_value = TryTaggedToFloat64(value, miss);
    CheckDescriptorConsidersNumbersMutable(handler_word, holder, miss);

    Comment("store double field");
    HandleStoreFieldAndReturn(handler_word, holder, value, double_value,
                              Representation::Double(), miss);
1794 1795 1796
  }
}

1797 1798 1799 1800
void AccessorAssembler::CheckHeapObjectTypeMatchesDescriptor(
    TNode<Word32T> handler_word, TNode<JSObject> holder, TNode<Object> value,
    Label* bailout) {
  GotoIf(TaggedIsSmi(value), bailout);
1801

1802 1803 1804 1805 1806 1807 1808 1809 1810 1811
  Label done(this);
  // Skip field type check in favor of constant value check when storing
  // to constant field.
  GotoIf(Word32Equal(DecodeWord32<StoreHandler::KindBits>(handler_word),
                     Int32Constant(StoreHandler::kConstField)),
         &done);
  TNode<IntPtrT> descriptor =
      Signed(DecodeWordFromWord32<StoreHandler::DescriptorBits>(handler_word));
  TNode<MaybeObject> maybe_field_type =
      LoadDescriptorValueOrFieldType(LoadMap(holder), descriptor);
1812

1813 1814
  GotoIf(TaggedIsSmi(maybe_field_type), &done);
  // Check that value type matches the field type.
1815
  {
1816 1817 1818
    TNode<HeapObject> field_type =
        GetHeapObjectAssumeWeak(maybe_field_type, bailout);
    Branch(TaggedEqual(LoadMap(CAST(value)), field_type), &done, bailout);
1819
  }
1820 1821
  BIND(&done);
}
1822

1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842
void AccessorAssembler::CheckDescriptorConsidersNumbersMutable(
    TNode<Word32T> handler_word, TNode<JSObject> holder, Label* bailout) {
  // We have to check that the representation is Double. Checking the value
  // (either in the field or being assigned) is not enough, as we could have
  // transitioned to Tagged but still be holding a HeapNumber, which would no
  // longer be allowed to be mutable.

  // TODO(leszeks): We could skip the representation check in favor of a
  // constant value check in HandleStoreFieldAndReturn here, but then
  // HandleStoreFieldAndReturn would need an IsHeapNumber check in case both the
  // representation changed and the value is no longer a HeapNumber.
  TNode<IntPtrT> descriptor_entry =
      Signed(DecodeWordFromWord32<StoreHandler::DescriptorBits>(handler_word));
  TNode<DescriptorArray> descriptors = LoadMapDescriptors(LoadMap(holder));
  TNode<Uint32T> details =
      LoadDetailsByDescriptorEntry(descriptors, descriptor_entry);

  GotoIfNot(IsEqualInWord32<PropertyDetails::RepresentationField>(
                details, Representation::kDouble),
            bailout);
1843 1844
}

1845 1846 1847 1848 1849
void AccessorAssembler::HandleStoreFieldAndReturn(
    TNode<Word32T> handler_word, TNode<JSObject> holder, TNode<Object> value,
    base::Optional<TNode<Float64T>> double_value, Representation representation,
    Label* miss) {
  Label done(this);
1850

1851
  bool store_value_as_double = representation.IsDouble();
1852

1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875
  TNode<BoolT> is_inobject =
      IsSetWord32<StoreHandler::IsInobjectBits>(handler_word);
  TNode<HeapObject> property_storage = Select<HeapObject>(
      is_inobject, [&]() { return holder; },
      [&]() { return LoadFastProperties(holder); });

  TNode<UintPtrT> index =
      DecodeWordFromWord32<StoreHandler::FieldIndexBits>(handler_word);
  TNode<IntPtrT> offset = Signed(TimesTaggedSize(index));

  // For Double fields, we want to mutate the current double-value
  // field rather than changing it to point at a new HeapNumber.
  if (store_value_as_double) {
    TVARIABLE(HeapObject, actual_property_storage, property_storage);
    TVARIABLE(IntPtrT, actual_offset, offset);

    Label property_and_offset_ready(this);

    // If we are unboxing double fields, and this is an in-object field, the
    // property_storage and offset are already pointing to the double-valued
    // field.
    if (FLAG_unbox_double_fields) {
      GotoIf(is_inobject, &property_and_offset_ready);
1876 1877
    }

1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908
    // Store the double value directly into the mutable HeapNumber.
    TNode<Object> field = LoadObjectField(property_storage, offset);
    CSA_ASSERT(this, IsHeapNumber(CAST(field)));
    actual_property_storage = CAST(field);
    actual_offset = IntPtrConstant(HeapNumber::kValueOffset);
    Goto(&property_and_offset_ready);

    BIND(&property_and_offset_ready);
    property_storage = actual_property_storage.value();
    offset = actual_offset.value();
  }

  // Do constant value check if necessary.
  Label do_store(this);
  GotoIfNot(Word32Equal(DecodeWord32<StoreHandler::KindBits>(handler_word),
                        Int32Constant(StoreHandler::kConstField)),
            &do_store);
  {
    if (store_value_as_double) {
      Label done(this);
      TNode<Float64T> current_value =
          LoadObjectField<Float64T>(property_storage, offset);
      BranchIfSameNumberValue(current_value, *double_value, &done, miss);
      BIND(&done);
      Return(value);
    } else {
      TNode<Object> current_value = LoadObjectField(property_storage, offset);
      GotoIfNot(TaggedEqual(current_value, value), miss);
      Return(value);
    }
  }
1909

1910 1911 1912
  BIND(&do_store);
  // Do the store.
  if (store_value_as_double) {
1913
    StoreObjectFieldNoWriteBarrier(property_storage, offset, *double_value);
1914 1915 1916
  } else if (representation.IsSmi()) {
    TNode<Smi> value_smi = CAST(value);
    StoreObjectFieldNoWriteBarrier(property_storage, offset, value_smi);
1917
  } else {
1918
    StoreObjectField(property_storage, offset, value);
1919
  }
1920 1921

  Return(value);
1922 1923
}

1924 1925
TNode<PropertyArray> AccessorAssembler::ExtendPropertiesBackingStore(
    TNode<HeapObject> object, TNode<IntPtrT> index) {
1926 1927
  Comment("[ Extend storage");

1928 1929
  TVARIABLE(HeapObject, var_properties);
  TVARIABLE(Int32T, var_encoded_hash);
1930
  TVARIABLE(IntPtrT, var_length);
1931

1932 1933
  TNode<Object> properties =
      LoadObjectField(object, JSObject::kPropertiesOrHashOffset);
1934 1935 1936 1937 1938 1939

  Label if_smi_hash(this), if_property_array(this), extend_store(this);
  Branch(TaggedIsSmi(properties), &if_smi_hash, &if_property_array);

  BIND(&if_smi_hash);
  {
1940
    TNode<Int32T> hash = SmiToInt32(CAST(properties));
1941
    TNode<Int32T> encoded_hash =
1942
        Word32Shl(hash, Int32Constant(PropertyArray::HashField::kShift));
1943
    var_encoded_hash = encoded_hash;
1944
    var_length = IntPtrConstant(0);
1945
    var_properties = EmptyFixedArrayConstant();
1946 1947 1948 1949 1950
    Goto(&extend_store);
  }

  BIND(&if_property_array);
  {
1951
    var_properties = CAST(properties);
1952
    TNode<Int32T> length_and_hash_int32 = LoadAndUntagToWord32ObjectField(
1953
        var_properties.value(), PropertyArray::kLengthAndHashOffset);
1954 1955
    var_encoded_hash = Word32And(
        length_and_hash_int32, Int32Constant(PropertyArray::HashField::kMask));
1956
    var_length = ChangeInt32ToIntPtr(
1957 1958
        Word32And(length_and_hash_int32,
                  Int32Constant(PropertyArray::LengthField::kMask)));
1959 1960
    Goto(&extend_store);
  }
1961

1962 1963
  BIND(&extend_store);
  {
1964
    TVARIABLE(HeapObject, var_new_properties, var_properties.value());
1965
    Label done(this);
1966 1967 1968
    // Previous property deletion could have left behind unused backing store
    // capacity even for a map that think it doesn't have any unused fields.
    // Perform a bounds check to see if we actually have to grow the array.
1969
    GotoIf(UintPtrLessThan(index, ParameterToIntPtr(var_length.value())),
1970
           &done);
1971

1972 1973
    TNode<IntPtrT> delta = IntPtrConstant(JSObject::kFieldsAdded);
    TNode<IntPtrT> new_capacity = IntPtrAdd(var_length.value(), delta);
1974 1975 1976 1977 1978 1979

    // Grow properties array.
    DCHECK(kMaxNumberOfDescriptors + JSObject::kFieldsAdded <
           FixedArrayBase::GetMaxLengthForNewSpaceAllocation(PACKED_ELEMENTS));
    // The size of a new properties backing store is guaranteed to be small
    // enough that the new backing store will be allocated in new space.
1980 1981
    CSA_ASSERT(this, IntPtrLessThan(new_capacity,
                                    IntPtrConstant(kMaxNumberOfDescriptors +
1982
                                                   JSObject::kFieldsAdded)));
1983

1984
    TNode<PropertyArray> new_properties = AllocatePropertyArray(new_capacity);
1985
    var_new_properties = new_properties;
1986 1987

    FillPropertyArrayWithUndefined(new_properties, var_length.value(),
1988
                                   new_capacity);
1989 1990 1991 1992

    // |new_properties| is guaranteed to be in new space, so we can skip
    // the write barrier.
    CopyPropertyArrayValues(var_properties.value(), new_properties,
1993
                            var_length.value(), SKIP_WRITE_BARRIER,
1994
                            DestroySource::kYes);
1995

1996
    TNode<Int32T> new_capacity_int32 = TruncateIntPtrToInt32(new_capacity);
1997
    TNode<Int32T> new_length_and_hash_int32 =
1998
        Word32Or(var_encoded_hash.value(), new_capacity_int32);
1999
    StoreObjectField(new_properties, PropertyArray::kLengthAndHashOffset,
2000
                     SmiFromInt32(new_length_and_hash_int32));
2001 2002 2003
    StoreObjectField(object, JSObject::kPropertiesOrHashOffset, new_properties);
    Comment("] Extend storage");
    Goto(&done);
2004
    BIND(&done);
2005
    return CAST(var_new_properties.value());
2006
  }
2007 2008
}

2009 2010 2011 2012 2013
void AccessorAssembler::EmitFastElementsBoundsCheck(
    TNode<JSObject> object, TNode<FixedArrayBase> elements,
    TNode<IntPtrT> intptr_index, TNode<BoolT> is_jsarray_condition,
    Label* miss) {
  TVARIABLE(IntPtrT, var_length);
2014 2015 2016 2017
  Comment("Fast elements bounds check");
  Label if_array(this), length_loaded(this, &var_length);
  GotoIf(is_jsarray_condition, &if_array);
  {
2018
    var_length = SmiUntag(LoadFixedArrayBaseLength(elements));
2019 2020
    Goto(&length_loaded);
  }
2021
  BIND(&if_array);
2022
  {
2023
    var_length = SmiUntag(LoadFastJSArrayLength(CAST(object)));
2024 2025
    Goto(&length_loaded);
  }
2026
  BIND(&length_loaded);
2027
  GotoIfNot(UintPtrLessThan(intptr_index, var_length.value()), miss);
2028 2029
}

2030
void AccessorAssembler::EmitElementLoad(
2031 2032 2033
    TNode<HeapObject> object, TNode<Word32T> elements_kind,
    TNode<IntPtrT> intptr_index, TNode<BoolT> is_jsarray_condition,
    Label* if_hole, Label* rebox_double, TVariable<Float64T>* var_double_value,
2034 2035
    Label* unimplemented_elements_kind, Label* out_of_bounds, Label* miss,
    ExitPoint* exit_point, LoadAccessMode access_mode) {
2036 2037 2038
  Label if_typed_array(this), if_fast(this), if_fast_packed(this),
      if_fast_holey(this), if_fast_double(this), if_fast_holey_double(this),
      if_nonfast(this), if_dictionary(this);
2039 2040 2041
  Branch(Int32GreaterThan(elements_kind,
                          Int32Constant(LAST_ANY_NONEXTENSIBLE_ELEMENTS_KIND)),
         &if_nonfast, &if_fast);
2042

2043
  BIND(&if_fast);
2044
  {
2045
    TNode<FixedArrayBase> elements = LoadJSObjectElements(CAST(object));
2046
    EmitFastElementsBoundsCheck(CAST(object), elements, intptr_index,
2047
                                is_jsarray_condition, out_of_bounds);
2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
    int32_t kinds[] = {
        // Handled by if_fast_packed.
        PACKED_SMI_ELEMENTS, PACKED_ELEMENTS, PACKED_NONEXTENSIBLE_ELEMENTS,
        PACKED_SEALED_ELEMENTS, PACKED_FROZEN_ELEMENTS,
        // Handled by if_fast_holey.
        HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS, HOLEY_NONEXTENSIBLE_ELEMENTS,
        HOLEY_FROZEN_ELEMENTS, HOLEY_SEALED_ELEMENTS,
        // Handled by if_fast_double.
        PACKED_DOUBLE_ELEMENTS,
        // Handled by if_fast_holey_double.
        HOLEY_DOUBLE_ELEMENTS};
    Label* labels[] = {// FAST_{SMI,}_ELEMENTS
                       &if_fast_packed, &if_fast_packed, &if_fast_packed,
                       &if_fast_packed, &if_fast_packed,
                       // FAST_HOLEY_{SMI,}_ELEMENTS
                       &if_fast_holey, &if_fast_holey, &if_fast_holey,
                       &if_fast_holey, &if_fast_holey,
                       // PACKED_DOUBLE_ELEMENTS
                       &if_fast_double,
                       // HOLEY_DOUBLE_ELEMENTS
                       &if_fast_holey_double};
2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079
    Switch(elements_kind, unimplemented_elements_kind, kinds, labels,
           arraysize(kinds));

    BIND(&if_fast_packed);
    {
      Comment("fast packed elements");
      exit_point->Return(
          access_mode == LoadAccessMode::kHas
              ? TrueConstant()
              : UnsafeLoadFixedArrayElement(CAST(elements), intptr_index));
    }
2080

2081 2082 2083
    BIND(&if_fast_holey);
    {
      Comment("fast holey elements");
2084 2085 2086
      TNode<Object> element =
          UnsafeLoadFixedArrayElement(CAST(elements), intptr_index);
      GotoIf(TaggedEqual(element, TheHoleConstant()), if_hole);
2087 2088
      exit_point->Return(access_mode == LoadAccessMode::kHas ? TrueConstant()
                                                             : element);
2089 2090
    }

2091 2092 2093 2094 2095 2096
    BIND(&if_fast_double);
    {
      Comment("packed double elements");
      if (access_mode == LoadAccessMode::kHas) {
        exit_point->Return(TrueConstant());
      } else {
2097 2098
        *var_double_value =
            LoadFixedDoubleArrayElement(CAST(elements), intptr_index);
2099 2100 2101 2102 2103 2104 2105
        Goto(rebox_double);
      }
    }

    BIND(&if_fast_holey_double);
    {
      Comment("holey double elements");
2106 2107
      TNode<Float64T> value =
          LoadFixedDoubleArrayElement(CAST(elements), intptr_index, if_hole);
2108 2109 2110
      if (access_mode == LoadAccessMode::kHas) {
        exit_point->Return(TrueConstant());
      } else {
2111
        *var_double_value = value;
2112 2113
        Goto(rebox_double);
      }
2114
    }
2115 2116
  }

2117
  BIND(&if_nonfast);
2118 2119
  {
    STATIC_ASSERT(LAST_ELEMENTS_KIND == LAST_FIXED_TYPED_ARRAY_ELEMENTS_KIND);
2120
    GotoIf(Int32GreaterThanOrEqual(
2121
               elements_kind,
2122
               Int32Constant(FIRST_FIXED_TYPED_ARRAY_ELEMENTS_KIND)),
2123
           &if_typed_array);
2124
    GotoIf(Word32Equal(elements_kind, Int32Constant(DICTIONARY_ELEMENTS)),
2125 2126 2127
           &if_dictionary);
    Goto(unimplemented_elements_kind);

2128 2129 2130
    BIND(&if_dictionary);
    {
      Comment("dictionary elements");
2131 2132 2133 2134 2135 2136 2137
      if (Is64()) {
        GotoIf(UintPtrLessThan(IntPtrConstant(JSArray::kMaxArrayIndex),
                               intptr_index),
               out_of_bounds);
      } else {
        GotoIf(IntPtrLessThan(intptr_index, IntPtrConstant(0)), out_of_bounds);
      }
2138 2139 2140 2141 2142 2143 2144

      TNode<FixedArrayBase> elements = LoadJSObjectElements(CAST(object));
      TNode<Object> value = BasicLoadNumberDictionaryElement(
          CAST(elements), intptr_index, miss, if_hole);
      exit_point->Return(access_mode == LoadAccessMode::kHas ? TrueConstant()
                                                             : value);
    }
2145

2146 2147 2148 2149
    BIND(&if_typed_array);
    {
      Comment("typed elements");
      // Check if buffer has been detached.
2150
      TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(CAST(object));
2151 2152 2153 2154 2155 2156 2157 2158
      GotoIf(IsDetachedBuffer(buffer), miss);

      // Bounds check.
      TNode<UintPtrT> length = LoadJSTypedArrayLength(CAST(object));
      GotoIfNot(UintPtrLessThan(intptr_index, length), out_of_bounds);
      if (access_mode == LoadAccessMode::kHas) {
        exit_point->Return(TrueConstant());
      } else {
2159
        TNode<RawPtrT> data_ptr = LoadJSTypedArrayDataPtr(CAST(object));
2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184

        Label uint8_elements(this), int8_elements(this), uint16_elements(this),
            int16_elements(this), uint32_elements(this), int32_elements(this),
            float32_elements(this), float64_elements(this),
            bigint64_elements(this), biguint64_elements(this);
        Label* elements_kind_labels[] = {
            &uint8_elements,    &uint8_elements,    &int8_elements,
            &uint16_elements,   &int16_elements,    &uint32_elements,
            &int32_elements,    &float32_elements,  &float64_elements,
            &bigint64_elements, &biguint64_elements};
        int32_t elements_kinds[] = {
            UINT8_ELEMENTS,    UINT8_CLAMPED_ELEMENTS, INT8_ELEMENTS,
            UINT16_ELEMENTS,   INT16_ELEMENTS,         UINT32_ELEMENTS,
            INT32_ELEMENTS,    FLOAT32_ELEMENTS,       FLOAT64_ELEMENTS,
            BIGINT64_ELEMENTS, BIGUINT64_ELEMENTS};
        const size_t kTypedElementsKindCount =
            LAST_FIXED_TYPED_ARRAY_ELEMENTS_KIND -
            FIRST_FIXED_TYPED_ARRAY_ELEMENTS_KIND + 1;
        DCHECK_EQ(kTypedElementsKindCount, arraysize(elements_kinds));
        DCHECK_EQ(kTypedElementsKindCount, arraysize(elements_kind_labels));
        Switch(elements_kind, miss, elements_kinds, elements_kind_labels,
               kTypedElementsKindCount);
        BIND(&uint8_elements);
        {
          Comment("UINT8_ELEMENTS");  // Handles UINT8_CLAMPED_ELEMENTS too.
2185
          TNode<Int32T> element = Load<Uint8T>(data_ptr, intptr_index);
2186 2187 2188 2189 2190
          exit_point->Return(SmiFromInt32(element));
        }
        BIND(&int8_elements);
        {
          Comment("INT8_ELEMENTS");
2191
          TNode<Int32T> element = Load<Int8T>(data_ptr, intptr_index);
2192 2193 2194 2195 2196
          exit_point->Return(SmiFromInt32(element));
        }
        BIND(&uint16_elements);
        {
          Comment("UINT16_ELEMENTS");
2197
          TNode<IntPtrT> index = WordShl(intptr_index, IntPtrConstant(1));
2198
          TNode<Int32T> element = Load<Uint16T>(data_ptr, index);
2199 2200 2201 2202 2203
          exit_point->Return(SmiFromInt32(element));
        }
        BIND(&int16_elements);
        {
          Comment("INT16_ELEMENTS");
2204
          TNode<IntPtrT> index = WordShl(intptr_index, IntPtrConstant(1));
2205
          TNode<Int32T> element = Load<Int16T>(data_ptr, index);
2206 2207 2208 2209 2210
          exit_point->Return(SmiFromInt32(element));
        }
        BIND(&uint32_elements);
        {
          Comment("UINT32_ELEMENTS");
2211
          TNode<IntPtrT> index = WordShl(intptr_index, IntPtrConstant(2));
2212
          TNode<Uint32T> element = Load<Uint32T>(data_ptr, index);
2213 2214 2215 2216 2217
          exit_point->Return(ChangeUint32ToTagged(element));
        }
        BIND(&int32_elements);
        {
          Comment("INT32_ELEMENTS");
2218
          TNode<IntPtrT> index = WordShl(intptr_index, IntPtrConstant(2));
2219
          TNode<Int32T> element = Load<Int32T>(data_ptr, index);
2220 2221 2222 2223 2224
          exit_point->Return(ChangeInt32ToTagged(element));
        }
        BIND(&float32_elements);
        {
          Comment("FLOAT32_ELEMENTS");
2225
          TNode<IntPtrT> index = WordShl(intptr_index, IntPtrConstant(2));
2226 2227
          TNode<Float32T> element = Load<Float32T>(data_ptr, index);
          *var_double_value = ChangeFloat32ToFloat64(element);
2228 2229 2230 2231 2232
          Goto(rebox_double);
        }
        BIND(&float64_elements);
        {
          Comment("FLOAT64_ELEMENTS");
2233
          TNode<IntPtrT> index = WordShl(intptr_index, IntPtrConstant(3));
2234 2235
          TNode<Float64T> element = Load<Float64T>(data_ptr, index);
          *var_double_value = element;
2236 2237 2238 2239 2240 2241
          Goto(rebox_double);
        }
        BIND(&bigint64_elements);
        {
          Comment("BIGINT64_ELEMENTS");
          exit_point->Return(LoadFixedTypedArrayElementAsTagged(
2242
              data_ptr, Unsigned(intptr_index), BIGINT64_ELEMENTS));
2243 2244 2245 2246 2247
        }
        BIND(&biguint64_elements);
        {
          Comment("BIGUINT64_ELEMENTS");
          exit_point->Return(LoadFixedTypedArrayElementAsTagged(
2248
              data_ptr, Unsigned(intptr_index), BIGUINT64_ELEMENTS));
2249
        }
2250
      }
2251
    }
2252 2253 2254
  }
}

2255 2256
void AccessorAssembler::InvalidateValidityCellIfPrototype(
    TNode<Map> map, base::Optional<TNode<Uint32T>> maybe_bitfield3) {
2257
  Label is_prototype(this), cont(this);
2258 2259 2260 2261
  TNode<Uint32T> bitfield3;
  if (bitfield3) {
    bitfield3 = maybe_bitfield3.value();
  } else {
2262
    bitfield3 = LoadMapBitField3(map);
2263 2264
  }

2265 2266
  Branch(IsSetWord32(bitfield3, Map::Bits3::IsPrototypeMapBit::kMask),
         &is_prototype, &cont);
2267 2268

  BIND(&is_prototype);
2269
  {
2270
    TNode<Object> maybe_prototype_info =
2271 2272 2273 2274
        LoadObjectField(map, Map::kTransitionsOrPrototypeInfoOffset);
    // If there's no prototype info then there's nothing to invalidate.
    GotoIf(TaggedIsSmi(maybe_prototype_info), &cont);

2275
    TNode<ExternalReference> function = ExternalConstant(
2276
        ExternalReference::invalidate_prototype_chains_function());
2277 2278
    CallCFunction(function, MachineType::AnyTagged(),
                  std::make_pair(MachineType::AnyTagged(), map));
2279 2280
    Goto(&cont);
  }
2281 2282 2283
  BIND(&cont);
}

2284 2285 2286 2287
void AccessorAssembler::GenericElementLoad(
    TNode<HeapObject> lookup_start_object, TNode<Map> lookup_start_object_map,
    TNode<Int32T> lookup_start_object_instance_type, TNode<IntPtrT> index,
    Label* slow) {
2288
  Comment("integer index");
2289 2290 2291

  ExitPoint direct_exit(this);

2292
  Label if_custom(this), if_element_hole(this), if_oob(this);
2293
  Label return_undefined(this);
2294 2295
  // Receivers requiring non-standard element accesses (interceptors, access
  // checks, strings and string wrappers, proxies) are handled in the runtime.
2296 2297 2298 2299 2300 2301
  GotoIf(
      IsCustomElementsReceiverInstanceType(lookup_start_object_instance_type),
      &if_custom);
  TNode<Int32T> elements_kind = LoadMapElementsKind(lookup_start_object_map);
  TNode<BoolT> is_jsarray_condition =
      IsJSArrayInstanceType(lookup_start_object_instance_type);
2302
  TVARIABLE(Float64T, var_double_value);
2303 2304 2305 2306 2307
  Label rebox_double(this, &var_double_value);

  // Unimplemented elements kinds fall back to a runtime call.
  Label* unimplemented_elements_kind = slow;
  IncrementCounter(isolate()->counters()->ic_keyed_load_generic_smi(), 1);
2308 2309 2310 2311
  EmitElementLoad(lookup_start_object, elements_kind, index,
                  is_jsarray_condition, &if_element_hole, &rebox_double,
                  &var_double_value, unimplemented_elements_kind, &if_oob, slow,
                  &direct_exit);
2312

2313
  BIND(&rebox_double);
2314 2315
  Return(AllocateHeapNumberWithValue(var_double_value.value()));

2316
  BIND(&if_oob);
2317 2318
  {
    Comment("out of bounds");
2319 2320
    // On TypedArrays, all OOB loads (positive and negative) return undefined
    // without ever checking the prototype chain.
2321 2322
    GotoIf(IsJSTypedArrayInstanceType(lookup_start_object_instance_type),
           &return_undefined);
2323 2324 2325
    // Positive OOB indices within JSArray index range are effectively the same
    // as hole loads. Larger keys and negative keys are named loads.
    if (Is64()) {
2326 2327
      Branch(UintPtrLessThanOrEqual(index,
                                    IntPtrConstant(JSArray::kMaxArrayIndex)),
2328 2329 2330 2331
             &if_element_hole, slow);
    } else {
      Branch(IntPtrLessThan(index, IntPtrConstant(0)), slow, &if_element_hole);
    }
2332 2333
  }

2334
  BIND(&if_element_hole);
2335 2336
  {
    Comment("found the hole");
2337 2338
    BranchIfPrototypesHaveNoElements(lookup_start_object_map, &return_undefined,
                                     slow);
2339
  }
2340 2341 2342 2343

  BIND(&if_custom);
  {
    Comment("check if string");
2344
    GotoIfNot(IsStringInstanceType(lookup_start_object_instance_type), slow);
2345
    Comment("load string character");
2346
    TNode<IntPtrT> length = LoadStringLengthAsWord(CAST(lookup_start_object));
2347 2348
    GotoIfNot(UintPtrLessThan(index, length), slow);
    IncrementCounter(isolate()->counters()->ic_keyed_load_generic_smi(), 1);
2349 2350
    TailCallBuiltin(Builtins::kStringCharAt, NoContextConstant(),
                    lookup_start_object, index);
2351
  }
2352 2353 2354

  BIND(&return_undefined);
  Return(UndefinedConstant());
2355 2356
}

2357 2358 2359 2360 2361
void AccessorAssembler::GenericPropertyLoad(
    TNode<HeapObject> lookup_start_object, TNode<Map> lookup_start_object_map,
    TNode<Int32T> lookup_start_object_instance_type, const LoadICParameters* p,
    Label* slow, UseStubCache use_stub_cache) {
  DCHECK_EQ(lookup_start_object, p->receiver_and_lookup_start_object());
2362 2363
  ExitPoint direct_exit(this);

2364
  Comment("key is unique name");
2365
  Label if_found_on_lookup_start_object(this), if_property_dictionary(this),
2366
      lookup_prototype_chain(this), special_receiver(this);
2367 2368
  TVARIABLE(Uint32T, var_details);
  TVARIABLE(Object, var_value);
2369

2370 2371
  TNode<Name> name = CAST(p->name());

2372
  // Receivers requiring non-standard accesses (interceptors, access
2373
  // checks, strings and string wrappers) are handled in the runtime.
2374 2375
  GotoIf(IsSpecialReceiverInstanceType(lookup_start_object_instance_type),
         &special_receiver);
2376

2377 2378
  // Check if the lookup_start_object has fast or slow properties.
  TNode<Uint32T> bitfield3 = LoadMapBitField3(lookup_start_object_map);
2379
  GotoIf(IsSetWord32<Map::Bits3::IsDictionaryMapBit>(bitfield3),
2380
         &if_property_dictionary);
2381

2382 2383 2384 2385
  // Try looking up the property on the lookup_start_object; if unsuccessful,
  // look for a handler in the stub cache.
  TNode<DescriptorArray> descriptors =
      LoadMapDescriptors(lookup_start_object_map);
2386

2387
  Label if_descriptor_found(this), try_stub_cache(this);
2388
  TVARIABLE(IntPtrT, var_name_index);
2389 2390
  Label* notfound = use_stub_cache == kUseStubCache ? &try_stub_cache
                                                    : &lookup_prototype_chain;
2391
  DescriptorLookup(name, descriptors, bitfield3, &if_descriptor_found,
2392
                   &var_name_index, notfound);
2393

2394
  BIND(&if_descriptor_found);
2395
  {
2396 2397 2398 2399
    LoadPropertyFromFastObject(lookup_start_object, lookup_start_object_map,
                               descriptors, var_name_index.value(),
                               &var_details, &var_value);
    Goto(&if_found_on_lookup_start_object);
2400 2401
  }

2402
  if (use_stub_cache == kUseStubCache) {
2403 2404 2405
    Label stub_cache(this);
    BIND(&try_stub_cache);
    // When there is no feedback vector don't use stub cache.
2406
    GotoIfNot(IsUndefined(p->vector()), &stub_cache);
2407
    // Fall back to the slow path for private symbols.
2408
    Branch(IsPrivateSymbol(name), slow, &lookup_prototype_chain);
2409

2410
    BIND(&stub_cache);
2411
    Comment("stub cache probe for fast property load");
2412
    TVARIABLE(MaybeObject, var_handler);
2413
    Label found_handler(this, &var_handler), stub_cache_miss(this);
2414
    TryProbeStubCache(isolate()->load_stub_cache(), lookup_start_object, name,
2415
                      &found_handler, &var_handler, &stub_cache_miss);
2416
    BIND(&found_handler);
2417
    {
2418 2419 2420
      LazyLoadICParameters lazy_p(p);
      HandleLoadICHandlerCase(&lazy_p, CAST(var_handler.value()),
                              &stub_cache_miss, &direct_exit);
2421
    }
2422

2423
    BIND(&stub_cache_miss);
2424 2425 2426 2427
    {
      // TODO(jkummerow): Check if the property exists on the prototype
      // chain. If it doesn't, then there's no point in missing.
      Comment("KeyedLoadGeneric_miss");
2428 2429 2430
      TailCallRuntime(Runtime::kKeyedLoadIC_Miss, p->context(),
                      p->receiver_and_lookup_start_object(), name, p->slot(),
                      p->vector());
2431 2432 2433
    }
  }

2434
  BIND(&if_property_dictionary);
2435 2436 2437 2438 2439
  {
    Comment("dictionary property load");
    // We checked for LAST_CUSTOM_ELEMENTS_RECEIVER before, which rules out
    // seeing global objects here (which would need special handling).

2440
    TVARIABLE(IntPtrT, var_name_index);
2441
    Label dictionary_found(this, &var_name_index);
2442 2443
    TNode<NameDictionary> properties =
        CAST(LoadSlowProperties(CAST(lookup_start_object)));
2444 2445
    NameDictionaryLookup<NameDictionary>(properties, name, &dictionary_found,
                                         &var_name_index,
2446
                                         &lookup_prototype_chain);
2447
    BIND(&dictionary_found);
2448 2449 2450
    {
      LoadPropertyFromNameDictionary(properties, var_name_index.value(),
                                     &var_details, &var_value);
2451
      Goto(&if_found_on_lookup_start_object);
2452 2453 2454
    }
  }

2455
  BIND(&if_found_on_lookup_start_object);
2456
  {
2457
    TNode<Object> value = CallGetterIfAccessor(
2458 2459
        var_value.value(), lookup_start_object, var_details.value(),
        p->context(), p->receiver(), slow);
2460 2461 2462 2463
    IncrementCounter(isolate()->counters()->ic_keyed_load_generic_symbol(), 1);
    Return(value);
  }

2464
  BIND(&lookup_prototype_chain);
2465
  {
2466
    TVARIABLE(Map, var_holder_map);
2467
    TVARIABLE(Int32T, var_holder_instance_type);
2468
    Label return_undefined(this), is_private_symbol(this);
2469
    Label loop(this, {&var_holder_map, &var_holder_instance_type});
2470

2471 2472
    var_holder_map = lookup_start_object_map;
    var_holder_instance_type = lookup_start_object_instance_type;
2473
    GotoIf(IsPrivateSymbol(name), &is_private_symbol);
2474

2475
    Goto(&loop);
2476
    BIND(&loop);
2477 2478
    {
      // Bailout if it can be an integer indexed exotic case.
2479 2480
      GotoIf(InstanceTypeEqual(var_holder_instance_type.value(),
                               JS_TYPED_ARRAY_TYPE),
2481
             slow);
2482
      TNode<HeapObject> proto = LoadMapPrototype(var_holder_map.value());
2483
      GotoIf(TaggedEqual(proto, NullConstant()), &return_undefined);
2484 2485
      TNode<Map> proto_map = LoadMap(proto);
      TNode<Uint16T> proto_instance_type = LoadMapInstanceType(proto_map);
2486
      var_holder_map = proto_map;
2487
      var_holder_instance_type = proto_instance_type;
2488
      Label next_proto(this), return_value(this, &var_value), goto_slow(this);
2489 2490 2491
      TryGetOwnProperty(p->context(), CAST(p->receiver()), CAST(proto),
                        proto_map, proto_instance_type, name, &return_value,
                        &var_value, &next_proto, &goto_slow);
2492 2493 2494

      // This trampoline and the next are required to appease Turbofan's
      // variable merging.
2495
      BIND(&next_proto);
2496 2497
      Goto(&loop);

2498
      BIND(&goto_slow);
2499 2500
      Goto(slow);

2501
      BIND(&return_value);
2502 2503 2504
      Return(var_value.value());
    }

2505 2506
    BIND(&is_private_symbol);
    {
2507
      CSA_ASSERT(this, IsPrivateSymbol(name));
2508 2509 2510 2511

      // For private names that don't exist on the receiver, we bail
      // to the runtime to throw. For private symbols, we just return
      // undefined.
2512
      Branch(IsPrivateName(CAST(name)), slow, &return_undefined);
2513 2514
    }

2515
    BIND(&return_undefined);
2516 2517
    Return(UndefinedConstant());
  }
2518 2519 2520

  BIND(&special_receiver);
  {
2521
    // TODO(jkummerow): Consider supporting JSModuleNamespace.
2522 2523 2524
    GotoIfNot(
        InstanceTypeEqual(lookup_start_object_instance_type, JS_PROXY_TYPE),
        slow);
2525

2526
    // Private field/symbol lookup is not supported.
2527
    GotoIf(IsPrivateSymbol(name), slow);
2528

2529 2530
    direct_exit.ReturnCallStub(
        Builtins::CallableFor(isolate(), Builtins::kProxyGetProperty),
2531 2532
        p->context(), lookup_start_object, name, p->receiver(),
        SmiConstant(OnNonExistent::kReturnUndefined));
2533
  }
2534 2535
}

2536 2537
//////////////////// Stub cache access helpers.

2538
enum AccessorAssembler::StubCacheTable : int {
2539 2540 2541 2542
  kPrimary = static_cast<int>(StubCache::kPrimary),
  kSecondary = static_cast<int>(StubCache::kSecondary)
};

2543 2544
TNode<IntPtrT> AccessorAssembler::StubCachePrimaryOffset(TNode<Name> name,
                                                         TNode<Map> map) {
2545
  // Compute the hash of the name (use entire hash field).
2546
  TNode<Uint32T> hash_field = LoadNameHashField(name);
2547 2548 2549 2550 2551 2552 2553 2554
  CSA_ASSERT(this,
             Word32Equal(Word32And(hash_field,
                                   Int32Constant(Name::kHashNotComputedMask)),
                         Int32Constant(0)));

  // Using only the low bits in 64-bit mode is unlikely to increase the
  // risk of collision even if the heap is spread over an area larger than
  // 4Gb (and not at all if it isn't).
2555
  TNode<IntPtrT> map_word = BitcastTaggedToWord(map);
2556

2557
  TNode<Int32T> map32 = TruncateIntPtrToInt32(UncheckedCast<IntPtrT>(
2558
      WordXor(map_word, WordShr(map_word, StubCache::kMapKeyShift))));
2559
  // Base the offset on a simple combination of name and map.
2560
  TNode<Word32T> hash = Int32Add(hash_field, map32);
2561 2562
  uint32_t mask = (StubCache::kPrimaryTableSize - 1)
                  << StubCache::kCacheIndexShift;
2563 2564 2565
  TNode<UintPtrT> result =
      ChangeUint32ToWord(Word32And(hash, Int32Constant(mask)));
  return Signed(result);
2566 2567
}

2568 2569
TNode<IntPtrT> AccessorAssembler::StubCacheSecondaryOffset(
    TNode<Name> name, TNode<IntPtrT> seed) {
2570 2571 2572
  // See v8::internal::StubCache::SecondaryOffset().

  // Use the seed from the primary cache in the secondary cache.
2573
  TNode<Int32T> name32 = TruncateIntPtrToInt32(BitcastTaggedToWord(name));
2574
  TNode<Int32T> hash = Int32Sub(TruncateIntPtrToInt32(seed), name32);
2575 2576 2577
  hash = Int32Add(hash, Int32Constant(StubCache::kSecondaryMagic));
  int32_t mask = (StubCache::kSecondaryTableSize - 1)
                 << StubCache::kCacheIndexShift;
2578 2579 2580
  TNode<UintPtrT> result =
      ChangeUint32ToWord(Word32And(hash, Int32Constant(mask)));
  return Signed(result);
2581 2582
}

2583
void AccessorAssembler::TryProbeStubCacheTable(
2584
    StubCache* stub_cache, StubCacheTable table_id, TNode<IntPtrT> entry_offset,
2585
    TNode<Object> name, TNode<Map> map, Label* if_handler,
2586
    TVariable<MaybeObject>* var_handler, Label* if_miss) {
2587 2588 2589
  StubCache::Table table = static_cast<StubCache::Table>(table_id);
  // The {table_offset} holds the entry offset times four (due to masking
  // and shifting optimizations).
2590 2591
  const int kMultiplier =
      sizeof(StubCache::Entry) >> StubCache::kCacheIndexShift;
2592 2593
  entry_offset = IntPtrMul(entry_offset, IntPtrConstant(kMultiplier));

2594
  TNode<ExternalReference> key_base = ExternalConstant(
2595
      ExternalReference::Create(stub_cache->key_reference(table)));
2596 2597 2598

  // Check that the key in the entry matches the name.
  DCHECK_EQ(0, offsetof(StubCache::Entry, key));
2599 2600 2601
  TNode<HeapObject> cached_key =
      CAST(Load(MachineType::TaggedPointer(), key_base, entry_offset));
  GotoIf(TaggedNotEqual(name, cached_key), if_miss);
2602 2603

  // Check that the map in the entry matches.
2604 2605
  TNode<Object> cached_map = Load<Object>(
      key_base,
2606
      IntPtrAdd(entry_offset, IntPtrConstant(offsetof(StubCache::Entry, map))));
2607
  GotoIf(TaggedNotEqual(map, cached_map), if_miss);
2608

2609
  TNode<MaybeObject> handler = ReinterpretCast<MaybeObject>(
2610
      Load(MachineType::AnyTagged(), key_base,
2611 2612
           IntPtrAdd(entry_offset,
                     IntPtrConstant(offsetof(StubCache::Entry, value)))));
2613 2614

  // We found the handler.
2615
  *var_handler = handler;
2616 2617 2618
  Goto(if_handler);
}

2619
void AccessorAssembler::TryProbeStubCache(StubCache* stub_cache,
2620
                                          TNode<Object> lookup_start_object,
2621
                                          TNode<Name> name, Label* if_handler,
2622
                                          TVariable<MaybeObject>* var_handler,
2623
                                          Label* if_miss) {
2624 2625 2626 2627 2628
  Label try_secondary(this), miss(this);

  Counters* counters = isolate()->counters();
  IncrementCounter(counters->megamorphic_stub_cache_probes(), 1);

2629 2630
  // Check that the {lookup_start_object} isn't a smi.
  GotoIf(TaggedIsSmi(lookup_start_object), &miss);
2631

2632
  TNode<Map> lookup_start_object_map = LoadMap(CAST(lookup_start_object));
2633 2634

  // Probe the primary table.
2635 2636
  TNode<IntPtrT> primary_offset =
      StubCachePrimaryOffset(name, lookup_start_object_map);
2637
  TryProbeStubCacheTable(stub_cache, kPrimary, primary_offset, name,
2638 2639
                         lookup_start_object_map, if_handler, var_handler,
                         &try_secondary);
2640

2641
  BIND(&try_secondary);
2642 2643
  {
    // Probe the secondary table.
2644 2645
    TNode<IntPtrT> secondary_offset =
        StubCacheSecondaryOffset(name, primary_offset);
2646
    TryProbeStubCacheTable(stub_cache, kSecondary, secondary_offset, name,
2647 2648
                           lookup_start_object_map, if_handler, var_handler,
                           &miss);
2649 2650
  }

2651
  BIND(&miss);
2652 2653 2654 2655 2656 2657 2658 2659
  {
    IncrementCounter(counters->megamorphic_stub_cache_misses(), 1);
    Goto(if_miss);
  }
}

//////////////////// Entry points into private implementation (one per stub).

2660
void AccessorAssembler::LoadIC_BytecodeHandler(const LazyLoadICParameters* p,
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670
                                               ExitPoint* exit_point) {
  // Must be kept in sync with LoadIC.

  // This function is hand-tuned to omit frame construction for common cases,
  // e.g.: monomorphic field and constant loads through smi handlers.
  // Polymorphic ICs with a hit in the first two entries also omit frames.
  // TODO(jgruber): Frame omission is fragile and can be affected by minor
  // changes in control flow and logic. We currently have no way of ensuring
  // that no frame is constructed, so it's easy to break this optimization by
  // accident.
2671 2672
  Label stub_call(this, Label::kDeferred), miss(this, Label::kDeferred),
      no_feedback(this, Label::kDeferred);
2673

2674 2675
  GotoIf(IsUndefined(p->vector()), &no_feedback);

2676 2677 2678
  TNode<Map> lookup_start_object_map =
      LoadReceiverMap(p->receiver_and_lookup_start_object());
  GotoIf(IsDeprecatedMap(lookup_start_object_map), &miss);
2679

2680 2681 2682 2683
  // Inlined fast path.
  {
    Comment("LoadIC_BytecodeHandler_fast");

2684
    TVARIABLE(MaybeObject, var_handler);
2685 2686
    Label try_polymorphic(this), if_handler(this, &var_handler);

2687 2688 2689
    TNode<MaybeObject> feedback = TryMonomorphicCase(
        p->slot(), CAST(p->vector()), lookup_start_object_map, &if_handler,
        &var_handler, &try_polymorphic);
2690

2691
    BIND(&if_handler);
2692
    HandleLoadICHandlerCase(p, CAST(var_handler.value()), &miss, exit_point);
2693

2694
    BIND(&try_polymorphic);
2695
    {
2696 2697
      TNode<HeapObject> strong_feedback =
          GetHeapObjectIfStrong(feedback, &miss);
2698
      GotoIfNot(IsWeakFixedArrayMap(LoadMap(strong_feedback)), &stub_call);
2699 2700
      HandlePolymorphicCase(lookup_start_object_map, CAST(strong_feedback),
                            &if_handler, &var_handler, &miss);
2701 2702 2703
    }
  }

2704
  BIND(&stub_call);
2705 2706 2707 2708
  {
    Comment("LoadIC_BytecodeHandler_noninlined");

    // Call into the stub that implements the non-inlined parts of LoadIC.
2709 2710
    Callable ic =
        Builtins::CallableFor(isolate(), Builtins::kLoadIC_Noninlined);
2711
    TNode<Code> code_target = HeapConstant(ic.code());
2712
    exit_point->ReturnCallStub(ic.descriptor(), code_target, p->context(),
2713 2714
                               p->receiver_and_lookup_start_object(), p->name(),
                               p->slot(), p->vector());
2715 2716
  }

2717 2718 2719 2720 2721
  BIND(&no_feedback);
  {
    Comment("LoadIC_BytecodeHandler_nofeedback");
    // Call into the stub that implements the non-inlined parts of LoadIC.
    exit_point->ReturnCallStub(
2722
        Builtins::CallableFor(isolate(), Builtins::kLoadIC_NoFeedback),
2723 2724
        p->context(), p->receiver(), p->name(),
        SmiConstant(FeedbackSlotKind::kLoadProperty));
2725 2726
  }

2727
  BIND(&miss);
2728 2729 2730
  {
    Comment("LoadIC_BytecodeHandler_miss");

2731 2732 2733
    exit_point->ReturnCallRuntime(Runtime::kLoadIC_Miss, p->context(),
                                  p->receiver(), p->name(), p->slot(),
                                  p->vector());
2734 2735 2736
  }
}

2737
void AccessorAssembler::LoadIC(const LoadICParameters* p) {
2738 2739 2740 2741
  // Must be kept in sync with LoadIC_BytecodeHandler.

  ExitPoint direct_exit(this);

2742
  TVARIABLE(MaybeObject, var_handler);
2743 2744
  Label if_handler(this, &var_handler), non_inlined(this, Label::kDeferred),
      try_polymorphic(this), miss(this, Label::kDeferred);
2745

2746 2747 2748
  TNode<Map> lookup_start_object_map =
      LoadReceiverMap(p->receiver_and_lookup_start_object());
  GotoIf(IsDeprecatedMap(lookup_start_object_map), &miss);
2749 2750

  // Check monomorphic case.
2751
  TNode<MaybeObject> feedback =
2752
      TryMonomorphicCase(p->slot(), CAST(p->vector()), lookup_start_object_map,
2753
                         &if_handler, &var_handler, &try_polymorphic);
2754
  BIND(&if_handler);
2755
  {
2756 2757 2758
    LazyLoadICParameters lazy_p(p);
    HandleLoadICHandlerCase(&lazy_p, CAST(var_handler.value()), &miss,
                            &direct_exit);
2759
  }
2760

2761
  BIND(&try_polymorphic);
2762
  TNode<HeapObject> strong_feedback = GetHeapObjectIfStrong(feedback, &miss);
2763 2764 2765
  {
    // Check polymorphic case.
    Comment("LoadIC_try_polymorphic");
2766
    GotoIfNot(IsWeakFixedArrayMap(LoadMap(strong_feedback)), &non_inlined);
2767 2768
    HandlePolymorphicCase(lookup_start_object_map, CAST(strong_feedback),
                          &if_handler, &var_handler, &miss);
2769 2770
  }

2771
  BIND(&non_inlined);
2772
  {
2773
    LoadIC_Noninlined(p, lookup_start_object_map, strong_feedback, &var_handler,
2774 2775
                      &if_handler, &miss, &direct_exit);
  }
2776

2777
  BIND(&miss);
2778
  direct_exit.ReturnCallRuntime(Runtime::kLoadIC_Miss, p->context(),
2779 2780
                                p->receiver_and_lookup_start_object(),
                                p->name(), p->slot(), p->vector());
2781 2782 2783
}

void AccessorAssembler::LoadIC_Noninlined(const LoadICParameters* p,
2784
                                          TNode<Map> lookup_start_object_map,
2785
                                          TNode<HeapObject> feedback,
2786
                                          TVariable<MaybeObject>* var_handler,
2787 2788 2789 2790
                                          Label* if_handler, Label* miss,
                                          ExitPoint* exit_point) {
  // Neither deprecated map nor monomorphic. These cases are handled in the
  // bytecode handler.
2791 2792
  CSA_ASSERT(this, Word32BinaryNot(IsDeprecatedMap(lookup_start_object_map)));
  CSA_ASSERT(this, TaggedNotEqual(lookup_start_object_map, feedback));
2793
  CSA_ASSERT(this, Word32BinaryNot(IsWeakFixedArrayMap(LoadMap(feedback))));
2794 2795
  DCHECK_EQ(MachineRepresentation::kTagged, var_handler->rep());

2796 2797
  {
    // Check megamorphic case.
2798
    GotoIfNot(TaggedEqual(feedback, MegamorphicSymbolConstant()), miss);
2799

2800
    TryProbeStubCache(isolate()->load_stub_cache(), p->lookup_start_object(),
2801
                      CAST(p->name()), if_handler, var_handler, miss);
2802
  }
2803 2804
}

2805 2806
void AccessorAssembler::LoadIC_NoFeedback(const LoadICParameters* p,
                                          TNode<Smi> ic_kind) {
2807
  Label miss(this, Label::kDeferred);
2808 2809 2810 2811
  TNode<Object> lookup_start_object = p->receiver_and_lookup_start_object();
  GotoIf(TaggedIsSmi(lookup_start_object), &miss);
  TNode<Map> lookup_start_object_map = LoadMap(CAST(lookup_start_object));
  GotoIf(IsDeprecatedMap(lookup_start_object_map), &miss);
2812

2813
  TNode<Uint16T> instance_type = LoadMapInstanceType(lookup_start_object_map);
2814

2815 2816 2817
  {
    // Special case for Function.prototype load, because it's very common
    // for ICs that are only executed once (MyFunc.prototype.foo = ...).
2818 2819 2820
    Label not_function_prototype(this, Label::kDeferred);
    GotoIfNot(InstanceTypeEqual(instance_type, JS_FUNCTION_TYPE),
              &not_function_prototype);
2821
    GotoIfNot(IsPrototypeString(p->name()), &not_function_prototype);
2822

2823 2824
    GotoIfPrototypeRequiresRuntimeLookup(CAST(lookup_start_object),
                                         lookup_start_object_map,
2825
                                         &not_function_prototype);
2826
    Return(LoadJSFunctionPrototype(CAST(lookup_start_object), &miss));
2827 2828 2829
    BIND(&not_function_prototype);
  }

2830 2831
  GenericPropertyLoad(CAST(lookup_start_object), lookup_start_object_map,
                      instance_type, p, &miss, kDontUseStubCache);
2832

2833
  BIND(&miss);
2834
  {
2835 2836
    TailCallRuntime(Runtime::kLoadNoFeedbackIC_Miss, p->context(),
                    p->receiver(), p->name(), ic_kind);
2837 2838 2839
  }
}

2840
void AccessorAssembler::LoadGlobalIC(TNode<HeapObject> maybe_feedback_vector,
2841
                                     const LazyNode<TaggedIndex>& lazy_slot,
2842 2843
                                     const LazyNode<Context>& lazy_context,
                                     const LazyNode<Name>& lazy_name,
2844
                                     TypeofMode typeof_mode,
2845
                                     ExitPoint* exit_point) {
2846 2847 2848 2849
  Label try_handler(this, Label::kDeferred), miss(this, Label::kDeferred),
      no_feedback(this, Label::kDeferred);

  GotoIf(IsUndefined(maybe_feedback_vector), &no_feedback);
2850
  {
2851
    TNode<TaggedIndex> slot = lazy_slot();
2852

2853 2854 2855 2856
    {
      TNode<FeedbackVector> vector = CAST(maybe_feedback_vector);
      LoadGlobalIC_TryPropertyCellCase(vector, slot, lazy_context, exit_point,
                                       &try_handler, &miss);
2857

2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871
      BIND(&try_handler);
      LoadGlobalIC_TryHandlerCase(vector, slot, lazy_context, lazy_name,
                                  typeof_mode, exit_point, &miss);
    }

    BIND(&miss);
    {
      Comment("LoadGlobalIC_MissCase");
      TNode<Context> context = lazy_context();
      TNode<Name> name = lazy_name();
      exit_point->ReturnCallRuntime(Runtime::kLoadGlobalIC_Miss, context, name,
                                    slot, maybe_feedback_vector,
                                    SmiConstant(typeof_mode));
    }
2872
  }
2873 2874 2875 2876 2877 2878 2879 2880 2881

  BIND(&no_feedback);
  {
    int ic_kind =
        static_cast<int>((typeof_mode == INSIDE_TYPEOF)
                             ? FeedbackSlotKind::kLoadGlobalInsideTypeof
                             : FeedbackSlotKind::kLoadGlobalNotInsideTypeof);
    exit_point->ReturnCallStub(
        Builtins::CallableFor(isolate(), Builtins::kLoadGlobalIC_NoFeedback),
2882
        lazy_context(), lazy_name(), SmiConstant(ic_kind));
2883
  }
2884 2885
}

2886
void AccessorAssembler::LoadGlobalIC_TryPropertyCellCase(
2887
    TNode<FeedbackVector> vector, TNode<TaggedIndex> slot,
2888 2889
    const LazyNode<Context>& lazy_context, ExitPoint* exit_point,
    Label* try_handler, Label* miss) {
2890 2891
  Comment("LoadGlobalIC_TryPropertyCellCase");

2892
  Label if_lexical_var(this), if_property_cell(this);
2893
  TNode<MaybeObject> maybe_weak_ref = LoadFeedbackVectorSlot(vector, slot);
2894
  Branch(TaggedIsSmi(maybe_weak_ref), &if_lexical_var, &if_property_cell);
2895

2896 2897
  BIND(&if_property_cell);
  {
2898
    // Load value or try handler case if the weak reference is cleared.
2899
    CSA_ASSERT(this, IsWeakOrCleared(maybe_weak_ref));
2900
    TNode<PropertyCell> property_cell =
2901
        CAST(GetHeapObjectAssumeWeak(maybe_weak_ref, try_handler));
2902 2903
    TNode<Object> value =
        LoadObjectField(property_cell, PropertyCell::kValueOffset);
2904
    GotoIf(TaggedEqual(value, TheHoleConstant()), miss);
2905 2906 2907 2908 2909 2910
    exit_point->Return(value);
  }

  BIND(&if_lexical_var);
  {
    Comment("Load lexical variable");
2911
    TNode<IntPtrT> lexical_handler = SmiUntag(CAST(maybe_weak_ref));
2912
    TNode<IntPtrT> context_index =
2913
        Signed(DecodeWord<FeedbackNexus::ContextIndexBits>(lexical_handler));
2914
    TNode<IntPtrT> slot_index =
2915
        Signed(DecodeWord<FeedbackNexus::SlotIndexBits>(lexical_handler));
2916
    TNode<Context> context = lazy_context();
2917 2918 2919 2920
    TNode<Context> script_context = LoadScriptContext(context, context_index);
    TNode<Object> result = LoadContextElement(script_context, slot_index);
    exit_point->Return(result);
  }
2921
}
2922

2923
void AccessorAssembler::LoadGlobalIC_TryHandlerCase(
2924 2925 2926
    TNode<FeedbackVector> vector, TNode<TaggedIndex> slot,
    const LazyNode<Context>& lazy_context, const LazyNode<Name>& lazy_name,
    TypeofMode typeof_mode, ExitPoint* exit_point, Label* miss) {
2927
  Comment("LoadGlobalIC_TryHandlerCase");
2928

2929
  Label call_handler(this), non_smi(this);
2930

2931
  TNode<MaybeObject> feedback_element =
2932
      LoadFeedbackVectorSlot(vector, slot, kTaggedSize);
2933
  TNode<Object> handler = CAST(feedback_element);
2934
  GotoIf(TaggedEqual(handler, UninitializedSymbolConstant()), miss);
2935

2936 2937 2938
  OnNonExistent on_nonexistent = typeof_mode == NOT_INSIDE_TYPEOF
                                     ? OnNonExistent::kThrowReferenceError
                                     : OnNonExistent::kReturnUndefined;
2939

2940
  TNode<Context> context = lazy_context();
2941
  TNode<NativeContext> native_context = LoadNativeContext(context);
2942 2943
  TNode<JSGlobalProxy> receiver =
      CAST(LoadContextElement(native_context, Context::GLOBAL_PROXY_INDEX));
2944
  TNode<Object> global =
2945
      LoadContextElement(native_context, Context::EXTENSION_INDEX);
2946

2947
  LazyLoadICParameters p([=] { return context; }, receiver, lazy_name,
2948
                         [=] { return slot; }, vector, global);
2949

2950 2951
  HandleLoadICHandlerCase(&p, handler, miss, exit_point, ICMode::kGlobalIC,
                          on_nonexistent);
2952 2953
}

2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028
void AccessorAssembler::ScriptContextTableLookup(
    TNode<Name> name, TNode<NativeContext> native_context, Label* found_hole,
    Label* not_found) {
  TNode<ScriptContextTable> script_context_table = CAST(
      LoadContextElement(native_context, Context::SCRIPT_CONTEXT_TABLE_INDEX));
  TVARIABLE(IntPtrT, context_index, IntPtrConstant(-1));
  Label loop(this, &context_index);
  TNode<IntPtrT> num_script_contexts = SmiUntag(CAST(LoadFixedArrayElement(
      script_context_table, ScriptContextTable::kUsedSlotIndex)));
  Goto(&loop);

  BIND(&loop);
  {
    context_index = IntPtrAdd(context_index.value(), IntPtrConstant(1));
    GotoIf(IntPtrGreaterThanOrEqual(context_index.value(), num_script_contexts),
           not_found);

    TNode<Context> script_context = CAST(LoadFixedArrayElement(
        script_context_table, context_index.value(),
        ScriptContextTable::kFirstContextSlotIndex * kTaggedSize));
    TNode<ScopeInfo> scope_info =
        CAST(LoadContextElement(script_context, Context::SCOPE_INFO_INDEX));
    TNode<IntPtrT> length = LoadAndUntagFixedArrayBaseLength(scope_info);
    GotoIf(IntPtrLessThanOrEqual(length, IntPtrConstant(0)), &loop);

    TVARIABLE(IntPtrT, scope_var_index,
              IntPtrConstant(ScopeInfo::kVariablePartIndex - 1));
    TNode<IntPtrT> num_scope_vars = SmiUntag(CAST(LoadFixedArrayElement(
        scope_info, IntPtrConstant(ScopeInfo::Fields::kContextLocalCount))));
    TNode<IntPtrT> end_index = IntPtrAdd(
        num_scope_vars, IntPtrConstant(ScopeInfo::kVariablePartIndex));
    Label loop_scope_info(this, &scope_var_index);
    Goto(&loop_scope_info);

    BIND(&loop_scope_info);
    {
      scope_var_index = IntPtrAdd(scope_var_index.value(), IntPtrConstant(1));
      GotoIf(IntPtrGreaterThanOrEqual(scope_var_index.value(), end_index),
             &loop);

      TNode<Object> var_name =
          LoadFixedArrayElement(scope_info, scope_var_index.value(), 0);
      GotoIf(TaggedNotEqual(var_name, name), &loop_scope_info);

      TNode<IntPtrT> var_index =
          IntPtrAdd(IntPtrConstant(Context::MIN_CONTEXT_SLOTS),
                    IntPtrSub(scope_var_index.value(),
                              IntPtrConstant(ScopeInfo::kVariablePartIndex)));
      TNode<Object> result = LoadContextElement(script_context, var_index);
      GotoIf(IsTheHole(result), found_hole);
      Return(result);
    }
  }
}

void AccessorAssembler::LoadGlobalIC_NoFeedback(TNode<Context> context,
                                                TNode<Object> name,
                                                TNode<Smi> smi_typeof_mode) {
  TNode<NativeContext> native_context = LoadNativeContext(context);
  Label regular_load(this), throw_reference_error(this, Label::kDeferred);

  GotoIfNot(IsString(CAST(name)), &regular_load);
  ScriptContextTableLookup(CAST(name), native_context, &throw_reference_error,
                           &regular_load);

  BIND(&throw_reference_error);
  Return(CallRuntime(Runtime::kThrowReferenceError, context, name));

  BIND(&regular_load);
  TNode<JSGlobalObject> global_object =
      CAST(LoadContextElement(native_context, Context::EXTENSION_INDEX));
  TailCallStub(Builtins::CallableFor(isolate(), Builtins::kLoadIC_NoFeedback),
               context, global_object, name, smi_typeof_mode);
}

3029 3030
void AccessorAssembler::KeyedLoadIC(const LoadICParameters* p,
                                    LoadAccessMode access_mode) {
3031 3032
  ExitPoint direct_exit(this);

3033
  TVARIABLE(MaybeObject, var_handler);
jgruber's avatar
jgruber committed
3034 3035
  Label if_handler(this, &var_handler), try_polymorphic(this, Label::kDeferred),
      try_megamorphic(this, Label::kDeferred),
3036
      try_uninitialized(this, Label::kDeferred),
jgruber's avatar
jgruber committed
3037
      try_polymorphic_name(this, Label::kDeferred),
3038
      miss(this, Label::kDeferred), generic(this, Label::kDeferred);
3039

3040 3041 3042
  TNode<Map> lookup_start_object_map =
      LoadReceiverMap(p->receiver_and_lookup_start_object());
  GotoIf(IsDeprecatedMap(lookup_start_object_map), &miss);
3043

3044
  GotoIf(IsUndefined(p->vector()), &generic);
3045

3046
  // Check monomorphic case.
3047
  TNode<MaybeObject> feedback =
3048
      TryMonomorphicCase(p->slot(), CAST(p->vector()), lookup_start_object_map,
3049
                         &if_handler, &var_handler, &try_polymorphic);
3050
  BIND(&if_handler);
3051
  {
3052 3053 3054 3055 3056
    LazyLoadICParameters lazy_p(p);
    HandleLoadICHandlerCase(&lazy_p, CAST(var_handler.value()), &miss,
                            &direct_exit, ICMode::kNonGlobalIC,
                            OnNonExistent::kReturnUndefined, kSupportElements,
                            access_mode);
3057
  }
3058

3059
  BIND(&try_polymorphic);
3060
  TNode<HeapObject> strong_feedback = GetHeapObjectIfStrong(feedback, &miss);
3061 3062 3063
  {
    // Check polymorphic case.
    Comment("KeyedLoadIC_try_polymorphic");
3064
    GotoIfNot(IsWeakFixedArrayMap(LoadMap(strong_feedback)), &try_megamorphic);
3065 3066
    HandlePolymorphicCase(lookup_start_object_map, CAST(strong_feedback),
                          &if_handler, &var_handler, &miss);
3067 3068
  }

3069
  BIND(&try_megamorphic);
3070 3071 3072
  {
    // Check megamorphic case.
    Comment("KeyedLoadIC_try_megamorphic");
3073 3074
    Branch(TaggedEqual(strong_feedback, MegamorphicSymbolConstant()), &generic,
           &try_uninitialized);
3075 3076 3077 3078
  }

  BIND(&generic);
  {
3079
    // TODO(jkummerow): Inline this? Or some of it?
3080 3081 3082
    TailCallBuiltin(access_mode == LoadAccessMode::kLoad
                        ? Builtins::kKeyedLoadIC_Megamorphic
                        : Builtins::kKeyedHasIC_Megamorphic,
3083 3084
                    p->context(), p->receiver(), p->name(), p->slot(),
                    p->vector());
3085
  }
3086

3087 3088 3089 3090
  BIND(&try_uninitialized);
  {
    // Check uninitialized case.
    Comment("KeyedLoadIC_try_uninitialized");
3091 3092
    Branch(TaggedEqual(strong_feedback, UninitializedSymbolConstant()), &miss,
           &try_polymorphic_name);
3093 3094
  }

3095
  BIND(&try_polymorphic_name);
3096
  {
3097 3098
    // We might have a name in feedback, and a weak fixed array in the next
    // slot.
3099
    Comment("KeyedLoadIC_try_polymorphic_name");
3100
    TVARIABLE(Name, var_name);
3101
    TVARIABLE(IntPtrT, var_index);
3102 3103
    Label if_polymorphic_name(this), feedback_matches(this),
        if_internalized(this), if_notinternalized(this, Label::kDeferred);
3104 3105

    // Fast-case: The recorded {feedback} matches the {name}.
3106
    GotoIf(TaggedEqual(strong_feedback, p->name()), &feedback_matches);
3107 3108

    // Try to internalize the {name} if it isn't already.
3109
    TryToName(p->name(), &miss, &var_index, &if_internalized, &var_name, &miss,
3110 3111 3112 3113 3114
              &if_notinternalized);

    BIND(&if_internalized);
    {
      // The {var_name} now contains a unique name.
3115 3116
      Branch(TaggedEqual(strong_feedback, var_name.value()),
             &if_polymorphic_name, &miss);
3117 3118 3119 3120
    }

    BIND(&if_notinternalized);
    {
3121 3122 3123 3124 3125 3126 3127 3128 3129
      TVARIABLE(IntPtrT, var_index);
      TryInternalizeString(CAST(p->name()), &miss, &var_index, &if_internalized,
                           &var_name, &miss, &miss);
    }

    BIND(&feedback_matches);
    {
      var_name = CAST(p->name());
      Goto(&if_polymorphic_name);
3130 3131 3132 3133
    }

    BIND(&if_polymorphic_name);
    {
3134 3135
      // If the name comparison succeeded, we know we have a weak fixed array
      // with at least one map/handler pair.
3136 3137 3138
      TailCallBuiltin(access_mode == LoadAccessMode::kLoad
                          ? Builtins::kKeyedLoadIC_PolymorphicName
                          : Builtins::kKeyedHasIC_PolymorphicName,
3139
                      p->context(), p->receiver(), var_name.value(), p->slot(),
3140
                      p->vector());
3141
    }
3142
  }
3143

3144
  BIND(&miss);
3145 3146
  {
    Comment("KeyedLoadIC_miss");
3147 3148 3149 3150
    TailCallRuntime(
        access_mode == LoadAccessMode::kLoad ? Runtime::kKeyedLoadIC_Miss
                                             : Runtime::kKeyedHasIC_Miss,
        p->context(), p->receiver(), p->name(), p->slot(), p->vector());
3151 3152 3153
  }
}

3154
void AccessorAssembler::KeyedLoadICGeneric(const LoadICParameters* p) {
3155
  TVARIABLE(Object, var_name, p->name());
3156

3157
  Label if_runtime(this, Label::kDeferred);
3158 3159 3160
  TNode<Object> lookup_start_object = p->lookup_start_object();
  GotoIf(TaggedIsSmi(lookup_start_object), &if_runtime);
  GotoIf(IsNullOrUndefined(lookup_start_object), &if_runtime);
3161

3162
  {
3163 3164
    TVARIABLE(IntPtrT, var_index);
    TVARIABLE(Name, var_unique);
3165 3166
    Label if_index(this), if_unique_name(this, &var_name), if_notunique(this),
        if_other(this, Label::kDeferred);
3167

3168 3169
    TryToName(var_name.value(), &if_index, &var_index, &if_unique_name,
              &var_unique, &if_other, &if_notunique);
3170

3171 3172 3173
    BIND(&if_unique_name);
    {
      LoadICParameters pp(p, var_unique.value());
3174 3175 3176
      TNode<Map> lookup_start_object_map = LoadMap(CAST(lookup_start_object));
      GenericPropertyLoad(CAST(lookup_start_object), lookup_start_object_map,
                          LoadMapInstanceType(lookup_start_object_map), &pp,
3177 3178
                          &if_runtime);
    }
3179

3180
    BIND(&if_other);
3181
    {
3182 3183 3184 3185
      var_name = CallBuiltin(Builtins::kToName, p->context(), var_name.value());
      TryToName(var_name.value(), &if_index, &var_index, &if_unique_name,
                &var_unique, &if_runtime, &if_notunique);
    }
3186

3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206
    BIND(&if_notunique);
    {
      if (FLAG_internalize_on_the_fly) {
        // Ideally we could return undefined directly here if the name is not
        // found in the string table, i.e. it was never internalized, but that
        // invariant doesn't hold with named property interceptors (at this
        // point), so we take the {if_runtime} path instead.
        Label if_in_string_table(this);
        TryInternalizeString(CAST(var_name.value()), &if_index, &var_index,
                             &if_in_string_table, &var_unique, &if_runtime,
                             &if_runtime);

        BIND(&if_in_string_table);
        {
          // TODO(bmeurer): We currently use a version of GenericPropertyLoad
          // here, where we don't try to probe the megamorphic stub cache
          // after successfully internalizing the incoming string. Past
          // experiments with this have shown that it causes too much traffic
          // on the stub cache. We may want to re-evaluate that in the future.
          LoadICParameters pp(p, var_unique.value());
3207 3208 3209 3210 3211
          TNode<Map> lookup_start_object_map =
              LoadMap(CAST(lookup_start_object));
          GenericPropertyLoad(CAST(lookup_start_object),
                              lookup_start_object_map,
                              LoadMapInstanceType(lookup_start_object_map), &pp,
3212
                              &if_runtime, kDontUseStubCache);
3213
        }
3214 3215
      } else {
        Goto(&if_runtime);
3216 3217 3218 3219 3220
      }
    }

    BIND(&if_index);
    {
3221 3222 3223
      TNode<Map> lookup_start_object_map = LoadMap(CAST(lookup_start_object));
      GenericElementLoad(CAST(lookup_start_object), lookup_start_object_map,
                         LoadMapInstanceType(lookup_start_object_map),
3224
                         var_index.value(), &if_runtime);
3225 3226 3227
    }
  }

3228
  BIND(&if_runtime);
3229 3230 3231 3232
  {
    Comment("KeyedLoadGeneric_slow");
    IncrementCounter(isolate()->counters()->ic_keyed_load_generic_slow(), 1);
    // TODO(jkummerow): Should we use the GetProperty TF stub instead?
3233 3234
    TailCallRuntime(Runtime::kGetProperty, p->context(),
                    p->receiver_and_lookup_start_object(), var_name.value());
3235 3236 3237
  }
}

3238 3239
void AccessorAssembler::KeyedLoadICPolymorphicName(const LoadICParameters* p,
                                                   LoadAccessMode access_mode) {
3240
  TVARIABLE(MaybeObject, var_handler);
3241 3242
  Label if_handler(this, &var_handler), miss(this, Label::kDeferred);

3243 3244
  TNode<Object> lookup_start_object = p->lookup_start_object();
  TNode<Map> lookup_start_object_map = LoadReceiverMap(lookup_start_object);
3245
  TNode<Name> name = CAST(p->name());
3246
  TNode<FeedbackVector> vector = CAST(p->vector());
3247
  TNode<TaggedIndex> slot = p->slot();
3248
  TNode<Context> context = p->context();
3249 3250 3251 3252

  // When we get here, we know that the {name} matches the recorded
  // feedback name in the {vector} and can safely be used for the
  // LoadIC handler logic below.
3253
  CSA_ASSERT(this, Word32BinaryNot(IsDeprecatedMap(lookup_start_object_map)));
3254
  CSA_ASSERT(this, TaggedEqual(name, LoadFeedbackVectorSlot(vector, slot)),
3255
             name, vector);
3256

3257
  // Check if we have a matching handler for the {lookup_start_object_map}.
3258
  TNode<MaybeObject> feedback_element =
3259
      LoadFeedbackVectorSlot(vector, slot, kTaggedSize);
3260
  TNode<WeakFixedArray> array = CAST(feedback_element);
3261 3262
  HandlePolymorphicCase(lookup_start_object_map, array, &if_handler,
                        &var_handler, &miss);
3263 3264 3265 3266

  BIND(&if_handler);
  {
    ExitPoint direct_exit(this);
3267 3268 3269 3270 3271
    LazyLoadICParameters lazy_p(p);
    HandleLoadICHandlerCase(&lazy_p, CAST(var_handler.value()), &miss,
                            &direct_exit, ICMode::kNonGlobalIC,
                            OnNonExistent::kReturnUndefined, kOnlyProperties,
                            access_mode);
3272 3273 3274 3275 3276
  }

  BIND(&miss);
  {
    Comment("KeyedLoadIC_miss");
3277 3278 3279 3280
    TailCallRuntime(
        access_mode == LoadAccessMode::kLoad ? Runtime::kKeyedLoadIC_Miss
                                             : Runtime::kKeyedHasIC_Miss,
        context, p->receiver_and_lookup_start_object(), name, slot, vector);
3281 3282 3283
  }
}

3284
void AccessorAssembler::StoreIC(const StoreICParameters* p) {
3285 3286 3287 3288 3289 3290
  TVARIABLE(MaybeObject, var_handler,
            ReinterpretCast<MaybeObject>(SmiConstant(0)));

  Label if_handler(this, &var_handler),
      if_handler_from_stub_cache(this, &var_handler, Label::kDeferred),
      try_polymorphic(this, Label::kDeferred),
3291
      try_megamorphic(this, Label::kDeferred), miss(this, Label::kDeferred),
3292
      no_feedback(this, Label::kDeferred);
3293

3294
  TNode<Map> receiver_map = LoadReceiverMap(p->receiver());
3295
  GotoIf(IsDeprecatedMap(receiver_map), &miss);
3296

3297
  GotoIf(IsUndefined(p->vector()), &no_feedback);
3298

3299
  // Check monomorphic case.
3300
  TNode<MaybeObject> feedback =
3301 3302
      TryMonomorphicCase(p->slot(), CAST(p->vector()), receiver_map,
                         &if_handler, &var_handler, &try_polymorphic);
3303
  BIND(&if_handler);
3304 3305
  {
    Comment("StoreIC_if_handler");
3306 3307
    HandleStoreICHandlerCase(p, var_handler.value(), &miss,
                             ICMode::kNonGlobalIC);
3308 3309
  }

3310
  BIND(&try_polymorphic);
3311
  TNode<HeapObject> strong_feedback = GetHeapObjectIfStrong(feedback, &miss);
3312 3313 3314
  {
    // Check polymorphic case.
    Comment("StoreIC_try_polymorphic");
3315 3316
    GotoIfNot(IsWeakFixedArrayMap(LoadMap(strong_feedback)), &try_megamorphic);
    HandlePolymorphicCase(receiver_map, CAST(strong_feedback), &if_handler,
3317
                          &var_handler, &miss);
3318 3319
  }

3320
  BIND(&try_megamorphic);
3321 3322
  {
    // Check megamorphic case.
3323
    GotoIfNot(TaggedEqual(strong_feedback, MegamorphicSymbolConstant()), &miss);
3324

3325 3326
    TryProbeStubCache(isolate()->store_stub_cache(), p->receiver(),
                      CAST(p->name()), &if_handler, &var_handler, &miss);
3327
  }
3328 3329 3330

  BIND(&no_feedback);
  {
3331 3332
    TailCallBuiltin(Builtins::kStoreIC_NoFeedback, p->context(), p->receiver(),
                    p->name(), p->value(), p->slot());
3333
  }
3334

3335
  BIND(&miss);
3336
  {
3337 3338
    TailCallRuntime(Runtime::kStoreIC_Miss, p->context(), p->value(), p->slot(),
                    p->vector(), p->receiver(), p->name());
3339 3340 3341
  }
}

3342
void AccessorAssembler::StoreGlobalIC(const StoreICParameters* pp) {
3343
  Label if_lexical_var(this), if_heapobject(this);
3344
  TNode<MaybeObject> maybe_weak_ref =
3345
      LoadFeedbackVectorSlot(CAST(pp->vector()), pp->slot());
3346
  Branch(TaggedIsSmi(maybe_weak_ref), &if_lexical_var, &if_heapobject);
3347

3348
  BIND(&if_heapobject);
3349
  {
3350
    Label try_handler(this), miss(this, Label::kDeferred);
3351

3352
    CSA_ASSERT(this, IsWeakOrCleared(maybe_weak_ref));
3353
    TNode<PropertyCell> property_cell =
3354
        CAST(GetHeapObjectAssumeWeak(maybe_weak_ref, &try_handler));
3355 3356

    ExitPoint direct_exit(this);
3357
    StoreGlobalIC_PropertyCellCase(property_cell, pp->value(), &direct_exit,
3358
                                   &miss);
3359

3360 3361 3362
    BIND(&try_handler);
    {
      Comment("StoreGlobalIC_try_handler");
3363 3364
      TNode<MaybeObject> handler =
          LoadFeedbackVectorSlot(CAST(pp->vector()), pp->slot(), kTaggedSize);
3365

3366
      GotoIf(TaggedEqual(handler, UninitializedSymbolConstant()), &miss);
3367

3368
      DCHECK(pp->receiver_is_null());
3369
      TNode<NativeContext> native_context = LoadNativeContext(pp->context());
3370 3371 3372 3373
      StoreICParameters p(
          pp->context(),
          LoadContextElement(native_context, Context::GLOBAL_PROXY_INDEX),
          pp->name(), pp->value(), pp->slot(), pp->vector());
3374 3375 3376 3377 3378 3379

      HandleStoreICHandlerCase(&p, handler, &miss, ICMode::kGlobalIC);
    }

    BIND(&miss);
    {
3380 3381
      TailCallRuntime(Runtime::kStoreGlobalIC_Miss, pp->context(), pp->value(),
                      pp->slot(), pp->vector(), pp->name());
3382
    }
3383 3384
  }

3385
  BIND(&if_lexical_var);
3386
  {
3387
    Comment("Store lexical variable");
3388
    TNode<IntPtrT> lexical_handler = SmiUntag(CAST(maybe_weak_ref));
3389
    TNode<IntPtrT> context_index =
3390
        Signed(DecodeWord<FeedbackNexus::ContextIndexBits>(lexical_handler));
3391
    TNode<IntPtrT> slot_index =
3392
        Signed(DecodeWord<FeedbackNexus::SlotIndexBits>(lexical_handler));
3393
    TNode<Context> script_context =
3394 3395 3396
        LoadScriptContext(pp->context(), context_index);
    StoreContextElement(script_context, slot_index, pp->value());
    Return(pp->value());
3397 3398 3399
  }
}

3400 3401 3402
void AccessorAssembler::StoreGlobalIC_PropertyCellCase(
    TNode<PropertyCell> property_cell, TNode<Object> value,
    ExitPoint* exit_point, Label* miss) {
3403 3404 3405 3406 3407
  Comment("StoreGlobalIC_TryPropertyCellCase");

  // Load the payload of the global parameter cell. A hole indicates that
  // the cell has been invalidated and that the store must be handled by the
  // runtime.
3408
  TNode<Object> cell_contents =
3409
      LoadObjectField(property_cell, PropertyCell::kValueOffset);
3410
  TNode<Int32T> details = LoadAndUntagToWord32ObjectField(
3411
      property_cell, PropertyCell::kPropertyDetailsRawOffset);
3412 3413 3414 3415 3416
  GotoIf(IsSetWord32(details, PropertyDetails::kAttributesReadOnlyMask), miss);
  CSA_ASSERT(this,
             Word32Equal(DecodeWord32<PropertyDetails::KindField>(details),
                         Int32Constant(kData)));

3417
  TNode<Uint32T> type =
3418
      DecodeWord32<PropertyDetails::PropertyCellTypeField>(details);
3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443

  Label constant(this), store(this), not_smi(this);

  GotoIf(Word32Equal(type, Int32Constant(
                               static_cast<int>(PropertyCellType::kConstant))),
         &constant);

  GotoIf(IsTheHole(cell_contents), miss);

  GotoIf(Word32Equal(
             type, Int32Constant(static_cast<int>(PropertyCellType::kMutable))),
         &store);
  CSA_ASSERT(this,
             Word32Or(Word32Equal(type, Int32Constant(static_cast<int>(
                                            PropertyCellType::kConstantType))),
                      Word32Equal(type, Int32Constant(static_cast<int>(
                                            PropertyCellType::kUndefined)))));

  GotoIfNot(TaggedIsSmi(cell_contents), &not_smi);
  GotoIfNot(TaggedIsSmi(value), miss);
  Goto(&store);

  BIND(&not_smi);
  {
    GotoIf(TaggedIsSmi(value), miss);
3444 3445 3446
    TNode<Map> expected_map = LoadMap(CAST(cell_contents));
    TNode<Map> map = LoadMap(CAST(value));
    GotoIfNot(TaggedEqual(expected_map, map), miss);
3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457
    Goto(&store);
  }

  BIND(&store);
  {
    StoreObjectField(property_cell, PropertyCell::kValueOffset, value);
    exit_point->Return(value);
  }

  BIND(&constant);
  {
3458
    GotoIfNot(TaggedEqual(cell_contents, value), miss);
3459 3460 3461 3462
    exit_point->Return(value);
  }
}

3463
void AccessorAssembler::KeyedStoreIC(const StoreICParameters* p) {
jgruber's avatar
jgruber committed
3464
  Label miss(this, Label::kDeferred);
3465
  {
3466
    TVARIABLE(MaybeObject, var_handler);
3467

jgruber's avatar
jgruber committed
3468 3469 3470
    Label if_handler(this, &var_handler),
        try_polymorphic(this, Label::kDeferred),
        try_megamorphic(this, Label::kDeferred),
3471
        no_feedback(this, Label::kDeferred),
jgruber's avatar
jgruber committed
3472
        try_polymorphic_name(this, Label::kDeferred);
3473

3474
    TNode<Map> receiver_map = LoadReceiverMap(p->receiver());
3475
    GotoIf(IsDeprecatedMap(receiver_map), &miss);
3476

3477
    GotoIf(IsUndefined(p->vector()), &no_feedback);
3478

3479
    // Check monomorphic case.
3480
    TNode<MaybeObject> feedback =
3481 3482
        TryMonomorphicCase(p->slot(), CAST(p->vector()), receiver_map,
                           &if_handler, &var_handler, &try_polymorphic);
3483
    BIND(&if_handler);
3484 3485
    {
      Comment("KeyedStoreIC_if_handler");
3486 3487
      HandleStoreICHandlerCase(p, var_handler.value(), &miss,
                               ICMode::kNonGlobalIC, kSupportElements);
3488
    }
3489

3490
    BIND(&try_polymorphic);
3491
    TNode<HeapObject> strong_feedback = GetHeapObjectIfStrong(feedback, &miss);
3492 3493 3494
    {
      // CheckPolymorphic case.
      Comment("KeyedStoreIC_try_polymorphic");
3495 3496 3497
      GotoIfNot(IsWeakFixedArrayMap(LoadMap(strong_feedback)),
                &try_megamorphic);
      HandlePolymorphicCase(receiver_map, CAST(strong_feedback), &if_handler,
3498
                            &var_handler, &miss);
3499
    }
3500

3501
    BIND(&try_megamorphic);
3502 3503 3504
    {
      // Check megamorphic case.
      Comment("KeyedStoreIC_try_megamorphic");
3505
      Branch(TaggedEqual(strong_feedback, MegamorphicSymbolConstant()),
3506
             &no_feedback, &try_polymorphic_name);
3507 3508 3509 3510
    }

    BIND(&no_feedback);
    {
3511 3512
      TailCallBuiltin(Builtins::kKeyedStoreIC_Megamorphic, p->context(),
                      p->receiver(), p->name(), p->value(), p->slot());
3513
    }
3514

3515
    BIND(&try_polymorphic_name);
3516 3517 3518
    {
      // We might have a name in feedback, and a fixed array in the next slot.
      Comment("KeyedStoreIC_try_polymorphic_name");
3519
      GotoIfNot(TaggedEqual(strong_feedback, p->name()), &miss);
3520 3521
      // If the name comparison succeeded, we know we have a feedback vector
      // with at least one map/handler pair.
3522 3523
      TNode<MaybeObject> feedback_element =
          LoadFeedbackVectorSlot(CAST(p->vector()), p->slot(), kTaggedSize);
3524
      TNode<WeakFixedArray> array = CAST(feedback_element);
3525
      HandlePolymorphicCase(receiver_map, array, &if_handler, &var_handler,
3526
                            &miss);
3527
    }
3528
  }
3529
  BIND(&miss);
3530 3531
  {
    Comment("KeyedStoreIC_miss");
3532 3533
    TailCallRuntime(Runtime::kKeyedStoreIC_Miss, p->context(), p->value(),
                    p->slot(), p->vector(), p->receiver(), p->name());
3534 3535 3536
  }
}

3537
void AccessorAssembler::StoreInArrayLiteralIC(const StoreICParameters* p) {
3538
  Label miss(this, Label::kDeferred), no_feedback(this, Label::kDeferred);
3539
  {
3540
    TVARIABLE(MaybeObject, var_handler);
3541 3542 3543 3544 3545

    Label if_handler(this, &var_handler),
        try_polymorphic(this, Label::kDeferred),
        try_megamorphic(this, Label::kDeferred);

3546
    TNode<Map> array_map = LoadReceiverMap(p->receiver());
3547
    GotoIf(IsDeprecatedMap(array_map), &miss);
3548

3549
    GotoIf(IsUndefined(p->vector()), &no_feedback);
3550

3551
    TNode<MaybeObject> feedback =
3552
        TryMonomorphicCase(p->slot(), CAST(p->vector()), array_map, &if_handler,
3553 3554 3555 3556 3557 3558
                           &var_handler, &try_polymorphic);

    BIND(&if_handler);
    {
      Comment("StoreInArrayLiteralIC_if_handler");
      // This is a stripped-down version of HandleStoreICHandlerCase.
3559 3560 3561 3562 3563
      Label if_transitioning_element_store(this), if_smi_handler(this);

      // Check used to identify the Slow case.
      // Currently only the Slow case uses a Smi handler.
      GotoIf(TaggedIsSmi(var_handler.value()), &if_smi_handler);
3564

3565
      TNode<HeapObject> handler = CAST(var_handler.value());
3566
      GotoIfNot(IsCode(handler), &if_transitioning_element_store);
3567 3568 3569
      TailCallStub(StoreWithVectorDescriptor{}, CAST(handler), p->context(),
                   p->receiver(), p->name(), p->value(), p->slot(),
                   p->vector());
3570 3571 3572

      BIND(&if_transitioning_element_store);
      {
3573 3574 3575
        TNode<MaybeObject> maybe_transition_map =
            LoadHandlerDataField(CAST(handler), 1);
        TNode<Map> transition_map =
3576
            CAST(GetHeapObjectAssumeWeak(maybe_transition_map, &miss));
3577
        GotoIf(IsDeprecatedMap(transition_map), &miss);
3578 3579
        TNode<Code> code =
            CAST(LoadObjectField(handler, StoreHandler::kSmiHandlerOffset));
3580 3581 3582
        TailCallStub(StoreTransitionDescriptor{}, code, p->context(),
                     p->receiver(), p->name(), transition_map, p->value(),
                     p->slot(), p->vector());
3583
      }
3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599

      BIND(&if_smi_handler);
      {
#ifdef DEBUG
        // A check to ensure that no other Smi handler uses this path.
        TNode<Int32T> handler_word = SmiToInt32(CAST(var_handler.value()));
        TNode<Uint32T> handler_kind =
            DecodeWord32<StoreHandler::KindBits>(handler_word);
        CSA_ASSERT(this, Word32Equal(handler_kind,
                                     Int32Constant(StoreHandler::kSlow)));
#endif

        Comment("StoreInArrayLiteralIC_Slow");
        TailCallRuntime(Runtime::kStoreInArrayLiteralIC_Slow, p->context(),
                        p->value(), p->receiver(), p->name());
      }
3600 3601 3602
    }

    BIND(&try_polymorphic);
3603
    TNode<HeapObject> strong_feedback = GetHeapObjectIfStrong(feedback, &miss);
3604 3605
    {
      Comment("StoreInArrayLiteralIC_try_polymorphic");
3606 3607 3608
      GotoIfNot(IsWeakFixedArrayMap(LoadMap(strong_feedback)),
                &try_megamorphic);
      HandlePolymorphicCase(array_map, CAST(strong_feedback), &if_handler,
3609
                            &var_handler, &miss);
3610 3611 3612 3613 3614
    }

    BIND(&try_megamorphic);
    {
      Comment("StoreInArrayLiteralIC_try_megamorphic");
3615 3616
      CSA_ASSERT(
          this,
3617 3618 3619
          Word32Or(TaggedEqual(strong_feedback, UninitializedSymbolConstant()),
                   TaggedEqual(strong_feedback, MegamorphicSymbolConstant())));
      GotoIfNot(TaggedEqual(strong_feedback, MegamorphicSymbolConstant()),
3620
                &miss);
3621 3622
      TailCallRuntime(Runtime::kStoreInArrayLiteralIC_Slow, p->context(),
                      p->value(), p->receiver(), p->name());
3623 3624 3625
    }
  }

3626 3627 3628 3629 3630 3631 3632
  BIND(&no_feedback);
  {
    Comment("StoreInArrayLiteralIC_NoFeedback");
    TailCallBuiltin(Builtins::kSetPropertyInLiteral, p->context(),
                    p->receiver(), p->name(), p->value());
  }

3633 3634 3635
  BIND(&miss);
  {
    Comment("StoreInArrayLiteralIC_miss");
3636 3637 3638
    TailCallRuntime(Runtime::kStoreInArrayLiteralIC_Miss, p->context(),
                    p->value(), p->slot(), p->vector(), p->receiver(),
                    p->name());
3639 3640 3641
  }
}

3642 3643
//////////////////// Public methods.

3644
void AccessorAssembler::GenerateLoadIC() {
3645
  using Descriptor = LoadWithVectorDescriptor;
3646

3647
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3648
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3649
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3650
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
3651
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3652 3653 3654 3655 3656

  LoadICParameters p(context, receiver, name, slot, vector);
  LoadIC(&p);
}

3657
void AccessorAssembler::GenerateLoadIC_Megamorphic() {
3658
  using Descriptor = LoadWithVectorDescriptor;
3659

3660
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3661
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3662
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3663
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
3664
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3665 3666 3667 3668 3669

  ExitPoint direct_exit(this);
  TVARIABLE(MaybeObject, var_handler);
  Label if_handler(this, &var_handler), miss(this, Label::kDeferred);

3670 3671 3672
  CSA_ASSERT(this, TaggedEqual(LoadFeedbackVectorSlot(CAST(vector), slot),
                               MegamorphicSymbolConstant()));

3673 3674
  TryProbeStubCache(isolate()->load_stub_cache(), receiver, CAST(name),
                    &if_handler, &var_handler, &miss);
3675 3676

  BIND(&if_handler);
3677 3678 3679 3680 3681 3682 3683
  LazyLoadICParameters p(
      // lazy_context
      [=] { return context; }, receiver,
      // lazy_name
      [=] { return name; },
      // lazy_slot
      [=] { return slot; }, vector);
3684 3685 3686 3687 3688 3689 3690
  HandleLoadICHandlerCase(&p, CAST(var_handler.value()), &miss, &direct_exit);

  BIND(&miss);
  direct_exit.ReturnCallRuntime(Runtime::kLoadIC_Miss, context, receiver, name,
                                slot, vector);
}

3691
void AccessorAssembler::GenerateLoadIC_Noninlined() {
3692
  using Descriptor = LoadWithVectorDescriptor;
3693

3694
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3695
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3696
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3697
  TNode<FeedbackVector> vector = CAST(Parameter(Descriptor::kVector));
3698
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3699 3700

  ExitPoint direct_exit(this);
3701
  TVARIABLE(MaybeObject, var_handler);
3702 3703
  Label if_handler(this, &var_handler), miss(this, Label::kDeferred);

3704
  TNode<MaybeObject> feedback_element = LoadFeedbackVectorSlot(vector, slot);
3705
  TNode<HeapObject> feedback = CAST(feedback_element);
3706 3707

  LoadICParameters p(context, receiver, name, slot, vector);
3708 3709 3710
  TNode<Map> lookup_start_object_map = LoadReceiverMap(p.lookup_start_object());
  LoadIC_Noninlined(&p, lookup_start_object_map, feedback, &var_handler,
                    &if_handler, &miss, &direct_exit);
3711

3712
  BIND(&if_handler);
3713
  {
3714 3715 3716
    LazyLoadICParameters lazy_p(&p);
    HandleLoadICHandlerCase(&lazy_p, CAST(var_handler.value()), &miss,
                            &direct_exit);
3717
  }
3718

3719
  BIND(&miss);
3720 3721 3722 3723
  direct_exit.ReturnCallRuntime(Runtime::kLoadIC_Miss, context, receiver, name,
                                slot, vector);
}

3724
void AccessorAssembler::GenerateLoadIC_NoFeedback() {
3725
  using Descriptor = LoadNoFeedbackDescriptor;
3726

3727
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3728
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3729
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3730
  TNode<Smi> ic_kind = CAST(Parameter(Descriptor::kICKind));
3731

3732 3733 3734
  LoadICParameters p(context, receiver, name,
                     TaggedIndexConstant(FeedbackSlot::Invalid().ToInt()),
                     UndefinedConstant());
3735
  LoadIC_NoFeedback(&p, ic_kind);
3736 3737
}

3738
void AccessorAssembler::GenerateLoadICTrampoline() {
3739
  using Descriptor = LoadDescriptor;
3740

3741
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3742
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3743
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3744
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3745
  TNode<FeedbackVector> vector = LoadFeedbackVectorForStub();
3746

3747
  TailCallBuiltin(Builtins::kLoadIC, context, receiver, name, slot, vector);
3748 3749
}

3750
void AccessorAssembler::GenerateLoadICTrampoline_Megamorphic() {
3751
  using Descriptor = LoadDescriptor;
3752

3753
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3754
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3755
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3756
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3757
  TNode<FeedbackVector> vector = LoadFeedbackVectorForStub();
3758 3759 3760 3761 3762

  TailCallBuiltin(Builtins::kLoadIC_Megamorphic, context, receiver, name, slot,
                  vector);
}

3763 3764 3765 3766 3767 3768 3769 3770 3771 3772
void AccessorAssembler::GenerateLoadGlobalIC_NoFeedback() {
  using Descriptor = LoadGlobalNoFeedbackDescriptor;

  TNode<Object> name = CAST(Parameter(Descriptor::kName));
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
  TNode<Smi> ic_kind = CAST(Parameter(Descriptor::kICKind));

  LoadGlobalIC_NoFeedback(context, name, ic_kind);
}

3773
void AccessorAssembler::GenerateLoadGlobalIC(TypeofMode typeof_mode) {
3774
  using Descriptor = LoadGlobalWithVectorDescriptor;
3775

3776
  TNode<Name> name = CAST(Parameter(Descriptor::kName));
3777
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3778
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
3779
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3780

3781
  ExitPoint direct_exit(this);
3782 3783 3784
  LoadGlobalIC(
      vector,
      // lazy_slot
3785
      [=] { return slot; },
3786 3787 3788 3789
      // lazy_context
      [=] { return context; },
      // lazy_name
      [=] { return name; }, typeof_mode, &direct_exit);
3790 3791
}

3792
void AccessorAssembler::GenerateLoadGlobalICTrampoline(TypeofMode typeof_mode) {
3793
  using Descriptor = LoadGlobalDescriptor;
3794

3795
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3796
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3797
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3798
  TNode<FeedbackVector> vector = LoadFeedbackVectorForStub();
3799

3800 3801 3802
  Callable callable =
      CodeFactory::LoadGlobalICInOptimizedCode(isolate(), typeof_mode);
  TailCallStub(callable, context, name, slot, vector);
3803 3804
}

3805
void AccessorAssembler::GenerateKeyedLoadIC() {
3806
  using Descriptor = LoadWithVectorDescriptor;
3807

3808
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3809
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3810
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3811
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
3812
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3813 3814

  LoadICParameters p(context, receiver, name, slot, vector);
3815
  KeyedLoadIC(&p, LoadAccessMode::kLoad);
3816 3817
}

3818
void AccessorAssembler::GenerateKeyedLoadIC_Megamorphic() {
3819
  using Descriptor = LoadWithVectorDescriptor;
3820

3821
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3822
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3823
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3824
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
3825
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3826 3827 3828 3829 3830

  LoadICParameters p(context, receiver, name, slot, vector);
  KeyedLoadICGeneric(&p);
}

3831
void AccessorAssembler::GenerateKeyedLoadICTrampoline() {
3832
  using Descriptor = LoadDescriptor;
3833

3834
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3835
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3836
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3837
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3838
  TNode<FeedbackVector> vector = LoadFeedbackVectorForStub();
3839

3840 3841
  TailCallBuiltin(Builtins::kKeyedLoadIC, context, receiver, name, slot,
                  vector);
3842 3843
}

3844
void AccessorAssembler::GenerateKeyedLoadICTrampoline_Megamorphic() {
3845
  using Descriptor = LoadDescriptor;
3846

3847
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3848
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3849
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3850
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3851
  TNode<FeedbackVector> vector = LoadFeedbackVectorForStub();
3852

3853 3854
  TailCallBuiltin(Builtins::kKeyedLoadIC_Megamorphic, context, receiver, name,
                  slot, vector);
3855 3856
}

3857
void AccessorAssembler::GenerateKeyedLoadIC_PolymorphicName() {
3858
  using Descriptor = LoadWithVectorDescriptor;
3859

3860
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3861
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3862
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3863
  TNode<FeedbackVector> vector = CAST(Parameter(Descriptor::kVector));
3864
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3865 3866

  LoadICParameters p(context, receiver, name, slot, vector);
3867
  KeyedLoadICPolymorphicName(&p, LoadAccessMode::kLoad);
3868 3869
}

3870
void AccessorAssembler::GenerateStoreGlobalIC() {
3871
  using Descriptor = StoreGlobalWithVectorDescriptor;
3872

3873
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3874
  TNode<Object> value = CAST(Parameter(Descriptor::kValue));
3875
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3876
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
3877
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3878

3879
  StoreICParameters p(context, base::nullopt, name, value, slot, vector);
3880 3881 3882 3883
  StoreGlobalIC(&p);
}

void AccessorAssembler::GenerateStoreGlobalICTrampoline() {
3884
  using Descriptor = StoreGlobalDescriptor;
3885

3886
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3887
  TNode<Object> value = CAST(Parameter(Descriptor::kValue));
3888
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3889
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3890
  TNode<FeedbackVector> vector = LoadFeedbackVectorForStub();
3891

3892
  TailCallBuiltin(Builtins::kStoreGlobalIC, context, name, value, slot, vector);
3893 3894
}

3895
void AccessorAssembler::GenerateStoreIC() {
3896
  using Descriptor = StoreWithVectorDescriptor;
3897

3898
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3899
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3900
  TNode<Object> value = CAST(Parameter(Descriptor::kValue));
3901
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3902
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
3903
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3904 3905

  StoreICParameters p(context, receiver, name, value, slot, vector);
3906
  StoreIC(&p);
3907 3908
}

3909
void AccessorAssembler::GenerateStoreICTrampoline() {
3910
  using Descriptor = StoreDescriptor;
3911

3912
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3913
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3914
  TNode<Object> value = CAST(Parameter(Descriptor::kValue));
3915
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3916
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3917
  TNode<FeedbackVector> vector = LoadFeedbackVectorForStub();
3918

3919 3920
  TailCallBuiltin(Builtins::kStoreIC, context, receiver, name, value, slot,
                  vector);
3921 3922
}

3923
void AccessorAssembler::GenerateKeyedStoreIC() {
3924
  using Descriptor = StoreWithVectorDescriptor;
3925

3926
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3927
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3928
  TNode<Object> value = CAST(Parameter(Descriptor::kValue));
3929
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3930
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
3931
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3932 3933

  StoreICParameters p(context, receiver, name, value, slot, vector);
3934
  KeyedStoreIC(&p);
3935 3936
}

3937
void AccessorAssembler::GenerateKeyedStoreICTrampoline() {
3938
  using Descriptor = StoreDescriptor;
3939

3940
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
3941
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
3942
  TNode<Object> value = CAST(Parameter(Descriptor::kValue));
3943
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3944
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3945
  TNode<FeedbackVector> vector = LoadFeedbackVectorForStub();
3946

3947 3948
  TailCallBuiltin(Builtins::kKeyedStoreIC, context, receiver, name, value, slot,
                  vector);
3949 3950
}

3951
void AccessorAssembler::GenerateStoreInArrayLiteralIC() {
3952
  using Descriptor = StoreWithVectorDescriptor;
3953

3954
  TNode<Object> array = CAST(Parameter(Descriptor::kReceiver));
3955
  TNode<Object> index = CAST(Parameter(Descriptor::kName));
3956
  TNode<Object> value = CAST(Parameter(Descriptor::kValue));
3957
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
3958
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
3959
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
3960 3961 3962 3963 3964

  StoreICParameters p(context, array, index, value, slot, vector);
  StoreInArrayLiteralIC(&p);
}

3965
void AccessorAssembler::GenerateCloneObjectIC_Slow() {
3966
  using Descriptor = CloneObjectWithVectorDescriptor;
3967
  TNode<Object> source = CAST(Parameter(Descriptor::kSource));
3968 3969 3970 3971 3972 3973 3974
  TNode<Smi> flags = CAST(Parameter(Descriptor::kFlags));
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));

  // The Slow case uses the same call interface as CloneObjectIC, so that it
  // can be tail called from it. However, the feedback slot and vector are not
  // used.

3975
  TNode<NativeContext> native_context = LoadNativeContext(context);
3976 3977 3978 3979 3980 3981
  TNode<JSFunction> object_fn =
      CAST(LoadContextElement(native_context, Context::OBJECT_FUNCTION_INDEX));
  TNode<Map> initial_map = CAST(
      LoadObjectField(object_fn, JSFunction::kPrototypeOrInitialMapOffset));
  CSA_ASSERT(this, IsMap(initial_map));

3982
  TNode<JSObject> result = AllocateJSObjectFromMap(initial_map);
3983 3984 3985 3986 3987

  {
    Label did_set_proto_if_needed(this);
    TNode<BoolT> is_null_proto = SmiNotEqual(
        SmiAnd(flags, SmiConstant(ObjectLiteral::kHasNullPrototype)),
3988
        SmiConstant(Smi::zero()));
3989 3990 3991 3992 3993 3994 3995 3996 3997 3998
    GotoIfNot(is_null_proto, &did_set_proto_if_needed);

    CallRuntime(Runtime::kInternalSetPrototype, context, result,
                NullConstant());

    Goto(&did_set_proto_if_needed);
    BIND(&did_set_proto_if_needed);
  }

  ReturnIf(IsNullOrUndefined(source), result);
3999
  source = ToObject_Inline(context, source);
4000

4001
  Label call_runtime(this, Label::kDeferred), done(this);
4002

4003 4004
  TNode<Map> source_map = LoadMap(CAST(source));
  GotoIfNot(IsJSObjectMap(source_map), &call_runtime);
4005 4006
  GotoIfNot(IsEmptyFixedArray(LoadElements(CAST(source))), &call_runtime);

4007
  ForEachEnumerableOwnProperty(
4008
      context, source_map, CAST(source), kPropertyAdditionOrder,
4009 4010 4011 4012
      [=](TNode<Name> key, TNode<Object> value) {
        SetPropertyInLiteral(context, result, key, value);
      },
      &call_runtime);
4013 4014 4015 4016 4017 4018 4019 4020 4021 4022
  Goto(&done);

  BIND(&call_runtime);
  CallRuntime(Runtime::kCopyDataProperties, context, result, source);

  Goto(&done);
  BIND(&done);
  Return(result);
}

4023
void AccessorAssembler::GenerateCloneObjectIC() {
4024
  using Descriptor = CloneObjectWithVectorDescriptor;
4025 4026
  TNode<Object> source = CAST(Parameter(Descriptor::kSource));
  TNode<Smi> flags = CAST(Parameter(Descriptor::kFlags));
4027
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
4028
  TNode<HeapObject> maybe_vector = CAST(Parameter(Descriptor::kVector));
4029
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
4030
  TVARIABLE(MaybeObject, var_handler);
4031 4032
  Label if_handler(this, &var_handler), miss(this, Label::kDeferred),
      try_polymorphic(this, Label::kDeferred),
4033
      try_megamorphic(this, Label::kDeferred), slow(this, Label::kDeferred);
4034

4035
  TNode<Map> source_map = LoadReceiverMap(source);
4036
  GotoIf(IsDeprecatedMap(source_map), &miss);
4037

4038
  GotoIf(IsUndefined(maybe_vector), &slow);
4039

4040 4041 4042
  TNode<MaybeObject> feedback =
      TryMonomorphicCase(slot, CAST(maybe_vector), source_map, &if_handler,
                         &var_handler, &try_polymorphic);
4043 4044 4045 4046 4047 4048 4049 4050

  BIND(&if_handler);
  {
    Comment("CloneObjectIC_if_handler");

    // Handlers for the CloneObjectIC stub are weak references to the Map of
    // a result object.
    TNode<Map> result_map = CAST(var_handler.value());
4051 4052
    TVARIABLE(HeapObject, var_properties, EmptyFixedArrayConstant());
    TVARIABLE(FixedArray, var_elements, EmptyFixedArrayConstant());
4053 4054 4055

    Label allocate_object(this);
    GotoIf(IsNullOrUndefined(source), &allocate_object);
4056
    CSA_SLOW_ASSERT(this, IsJSObjectMap(source_map));
4057 4058 4059 4060
    CSA_SLOW_ASSERT(this, IsJSObjectMap(result_map));

    // The IC fast case should only be taken if the result map a compatible
    // elements kind with the source object.
4061
    TNode<FixedArrayBase> source_elements = LoadElements(CAST(source));
4062 4063 4064 4065 4066 4067 4068 4069

    auto flags = ExtractFixedArrayFlag::kAllFixedArraysDontCopyCOW;
    var_elements = CAST(CloneFixedArray(source_elements, flags));

    // Copy the PropertyArray backing store. The source PropertyArray must be
    // either an Smi, or a PropertyArray.
    // FIXME: Make a CSA macro for this
    TNode<Object> source_properties =
4070
        LoadObjectField(CAST(source), JSObject::kPropertiesOrHashOffset);
4071 4072 4073 4074
    {
      GotoIf(TaggedIsSmi(source_properties), &allocate_object);
      GotoIf(IsEmptyFixedArray(source_properties), &allocate_object);

4075
      // This IC requires that the source object has fast properties.
4076 4077 4078
      TNode<PropertyArray> source_property_array = CAST(source_properties);

      TNode<IntPtrT> length = LoadPropertyArrayLength(source_property_array);
4079 4080
      GotoIf(IntPtrEqual(length, IntPtrConstant(0)), &allocate_object);

4081
      TNode<PropertyArray> property_array = AllocatePropertyArray(length);
4082
      FillPropertyArrayWithUndefined(property_array, IntPtrConstant(0), length);
4083
      CopyPropertyArrayValues(source_property_array, property_array, length,
4084
                              SKIP_WRITE_BARRIER, DestroySource::kNo);
4085
      var_properties = property_array;
4086 4087 4088 4089 4090 4091 4092 4093 4094
    }

    Goto(&allocate_object);
    BIND(&allocate_object);
    TNode<JSObject> object = UncheckedCast<JSObject>(AllocateJSObjectFromMap(
        result_map, var_properties.value(), var_elements.value()));
    ReturnIf(IsNullOrUndefined(source), object);

    // Lastly, clone any in-object properties.
4095 4096 4097 4098 4099 4100
    TNode<IntPtrT> source_start =
        LoadMapInobjectPropertiesStartInWords(source_map);
    TNode<IntPtrT> source_size = LoadMapInstanceSizeInWords(source_map);
    TNode<IntPtrT> result_start =
        LoadMapInobjectPropertiesStartInWords(result_map);
    TNode<IntPtrT> field_offset_difference =
4101
        TimesTaggedSize(IntPtrSub(result_start, source_start));
4102

4103 4104
    // Just copy the fields as raw data (pretending that there are no mutable
    // HeapNumbers). This doesn't need write barriers.
4105
    BuildFastLoop<IntPtrT>(
4106
        source_start, source_size,
4107 4108
        [=](TNode<IntPtrT> field_index) {
          TNode<IntPtrT> field_offset = TimesTaggedSize(field_index);
4109 4110
          TNode<TaggedT> field =
              LoadObjectField<TaggedT>(CAST(source), field_offset);
4111 4112 4113
          TNode<IntPtrT> result_offset =
              IntPtrAdd(field_offset, field_offset_difference);
          StoreObjectFieldNoWriteBarrier(object, result_offset, field);
4114
        },
4115
        1, IndexAdvanceMode::kPost);
4116

4117
    // If mutable HeapNumbers can occur, we need to go through the {object}
4118 4119 4120 4121
    // again here and properly clone them. We use a second loop here to
    // ensure that the GC (and heap verifier) always sees properly initialized
    // objects, i.e. never hits undefined values in double fields.
    if (!FLAG_unbox_double_fields) {
4122
      BuildFastLoop<IntPtrT>(
4123
          source_start, source_size,
4124 4125 4126
          [=](TNode<IntPtrT> field_index) {
            TNode<IntPtrT> result_offset = IntPtrAdd(
                TimesTaggedSize(field_index), field_offset_difference);
4127 4128 4129
            TNode<Object> field = LoadObjectField(object, result_offset);
            Label if_done(this), if_mutableheapnumber(this, Label::kDeferred);
            GotoIf(TaggedIsSmi(field), &if_done);
4130
            Branch(IsHeapNumber(CAST(field)), &if_mutableheapnumber, &if_done);
4131 4132
            BIND(&if_mutableheapnumber);
            {
4133
              TNode<HeapNumber> value = AllocateHeapNumberWithValue(
4134 4135 4136 4137 4138 4139
                  LoadHeapNumberValue(UncheckedCast<HeapNumber>(field)));
              StoreObjectField(object, result_offset, value);
              Goto(&if_done);
            }
            BIND(&if_done);
          },
4140
          1, IndexAdvanceMode::kPost);
4141 4142
    }

4143 4144 4145 4146
    Return(object);
  }

  BIND(&try_polymorphic);
4147
  TNode<HeapObject> strong_feedback = GetHeapObjectIfStrong(feedback, &miss);
4148 4149 4150 4151
  {
    Comment("CloneObjectIC_try_polymorphic");
    GotoIfNot(IsWeakFixedArrayMap(LoadMap(strong_feedback)), &try_megamorphic);
    HandlePolymorphicCase(source_map, CAST(strong_feedback), &if_handler,
4152
                          &var_handler, &miss);
4153 4154 4155 4156 4157
  }

  BIND(&try_megamorphic);
  {
    Comment("CloneObjectIC_try_megamorphic");
4158 4159 4160 4161 4162
    CSA_ASSERT(
        this,
        Word32Or(TaggedEqual(strong_feedback, UninitializedSymbolConstant()),
                 TaggedEqual(strong_feedback, MegamorphicSymbolConstant())));
    GotoIfNot(TaggedEqual(strong_feedback, MegamorphicSymbolConstant()), &miss);
4163 4164 4165 4166 4167
    Goto(&slow);
  }

  BIND(&slow);
  {
4168
    TailCallBuiltin(Builtins::kCloneObjectIC_Slow, context, source, flags, slot,
4169
                    maybe_vector);
4170 4171 4172 4173 4174
  }

  BIND(&miss);
  {
    Comment("CloneObjectIC_miss");
4175 4176 4177
    TNode<HeapObject> map_or_result =
        CAST(CallRuntime(Runtime::kCloneObjectIC_Miss, context, source, flags,
                         slot, maybe_vector));
4178 4179 4180 4181 4182 4183 4184
    var_handler = UncheckedCast<MaybeObject>(map_or_result);
    GotoIf(IsMap(map_or_result), &if_handler);
    CSA_ASSERT(this, IsJSObject(map_or_result));
    Return(map_or_result);
  }
}

4185
void AccessorAssembler::GenerateKeyedHasIC() {
4186
  using Descriptor = LoadWithVectorDescriptor;
4187

4188
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
4189
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
4190
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
4191
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
4192
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
4193 4194 4195 4196 4197 4198

  LoadICParameters p(context, receiver, name, slot, vector);
  KeyedLoadIC(&p, LoadAccessMode::kHas);
}

void AccessorAssembler::GenerateKeyedHasIC_Megamorphic() {
4199
  using Descriptor = LoadWithVectorDescriptor;
4200

4201
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
4202
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
4203
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
4204 4205 4206 4207 4208 4209
  // TODO(magardn): implement HasProperty handling in KeyedLoadICGeneric
  Return(HasProperty(context, receiver, name,
                     HasPropertyLookupMode::kHasProperty));
}

void AccessorAssembler::GenerateKeyedHasIC_PolymorphicName() {
4210
  using Descriptor = LoadWithVectorDescriptor;
4211

4212
  TNode<Object> receiver = CAST(Parameter(Descriptor::kReceiver));
4213
  TNode<Object> name = CAST(Parameter(Descriptor::kName));
4214
  TNode<TaggedIndex> slot = CAST(Parameter(Descriptor::kSlot));
4215
  TNode<HeapObject> vector = CAST(Parameter(Descriptor::kVector));
4216
  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
4217 4218 4219 4220 4221

  LoadICParameters p(context, receiver, name, slot, vector);
  KeyedLoadICPolymorphicName(&p, LoadAccessMode::kHas);
}

4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270
void AccessorAssembler::BranchIfPrototypesHaveNoElements(
    TNode<Map> receiver_map, Label* definitely_no_elements,
    Label* possibly_elements) {
  TVARIABLE(Map, var_map, receiver_map);
  Label loop_body(this, &var_map);
  TNode<FixedArray> empty_fixed_array = EmptyFixedArrayConstant();
  TNode<NumberDictionary> empty_slow_element_dictionary =
      EmptySlowElementDictionaryConstant();
  Goto(&loop_body);

  BIND(&loop_body);
  {
    TNode<Map> map = var_map.value();
    TNode<HeapObject> prototype = LoadMapPrototype(map);
    GotoIf(IsNull(prototype), definitely_no_elements);
    TNode<Map> prototype_map = LoadMap(prototype);
    TNode<Uint16T> prototype_instance_type = LoadMapInstanceType(prototype_map);

    // Pessimistically assume elements if a Proxy, Special API Object,
    // or JSPrimitiveWrapper wrapper is found on the prototype chain. After this
    // instance type check, it's not necessary to check for interceptors or
    // access checks.
    Label if_custom(this, Label::kDeferred), if_notcustom(this);
    Branch(IsCustomElementsReceiverInstanceType(prototype_instance_type),
           &if_custom, &if_notcustom);

    BIND(&if_custom);
    {
      // For string JSPrimitiveWrapper wrappers we still support the checks as
      // long as they wrap the empty string.
      GotoIfNot(
          InstanceTypeEqual(prototype_instance_type, JS_PRIMITIVE_WRAPPER_TYPE),
          possibly_elements);
      TNode<Object> prototype_value =
          LoadJSPrimitiveWrapperValue(CAST(prototype));
      Branch(IsEmptyString(prototype_value), &if_notcustom, possibly_elements);
    }

    BIND(&if_notcustom);
    {
      TNode<FixedArrayBase> prototype_elements = LoadElements(CAST(prototype));
      var_map = prototype_map;
      GotoIf(TaggedEqual(prototype_elements, empty_fixed_array), &loop_body);
      Branch(TaggedEqual(prototype_elements, empty_slow_element_dictionary),
             &loop_body, possibly_elements);
    }
  }
}

4271 4272
}  // namespace internal
}  // namespace v8