[csa][codegen] Optimize IsStrong/IsWeakOrCleared

The way to determine whether a MaybeObject is a strong or weak
reference to the heap object is to check its lowest two bits.
However, if the MaybeObject is known to not be a smi, that is, the
lowest bit is known to be 1, we can check one bit instead. This
allows Turbofan to select better instructions:

x64:

  Before:
    movl r9,r11
    andl r9,0x3
    cmpb r9l,0x1

  After:
    testb r11,0x2

arm64:

  Before:
    and w8, w7, #0x3
    cmp w8, #0x1 (1)
    b.ne #+0x320

  After:
    tbnz w7, #1, #+0x320

Change-Id: I03623183406ad7d920c96a752651e0116a22832e
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3861310Reviewed-by: Nico Hartmann <nicohartmann@chromium.org>
Commit-Queue: Hao A Xu <hao.a.xu@intel.com>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/main@{#83153}

include/v8-internal.h

@@ -52,6 +52,7 @@ const int kHeapObjectTag = 1;
 const int kWeakHeapObjectTag = 3;
 const int kHeapObjectTagSize = 2;
 const intptr_t kHeapObjectTagMask = (1 << kHeapObjectTagSize) - 1;
+const intptr_t kHeapObjectReferenceTagMask = 1 << (kHeapObjectTagSize - 1);
 
 // Tag information for fowarding pointers stored in object headers.
 // 0b00 at the lowest 2 bits in the header indicates that the map word is a

src/codegen/code-stub-assembler.cc

@@ -2067,12 +2067,24 @@ TNode<BoolT> CodeStubAssembler::IsStrong(TNode<MaybeObject> value) {
                      Int32Constant(kHeapObjectTag));
 }
 
+TNode<BoolT> CodeStubAssembler::IsStrong(TNode<HeapObjectReference> value) {
+  return IsNotSetWord32(
+      TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(value)),
+      kHeapObjectReferenceTagMask);
+}
+
 TNode<HeapObject> CodeStubAssembler::GetHeapObjectIfStrong(
     TNode<MaybeObject> value, Label* if_not_strong) {
   GotoIfNot(IsStrong(value), if_not_strong);
   return CAST(value);
 }
 
+TNode<HeapObject> CodeStubAssembler::GetHeapObjectIfStrong(
+    TNode<HeapObjectReference> value, Label* if_not_strong) {
+  GotoIfNot(IsStrong(value), if_not_strong);
+  return ReinterpretCast<HeapObject>(value);
+}
+
 TNode<BoolT> CodeStubAssembler::IsWeakOrCleared(TNode<MaybeObject> value) {
   return Word32Equal(Word32And(TruncateIntPtrToInt32(
                                    BitcastTaggedToWordForTagAndSmiBits(value)),
@@ -2080,6 +2092,13 @@ TNode<BoolT> CodeStubAssembler::IsWeakOrCleared(TNode<MaybeObject> value) {
                      Int32Constant(kWeakHeapObjectTag));
 }
 
+TNode<BoolT> CodeStubAssembler::IsWeakOrCleared(
+    TNode<HeapObjectReference> value) {
+  return IsSetWord32(
+      TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(value)),
+      kHeapObjectReferenceTagMask);
+}
+
 TNode<BoolT> CodeStubAssembler::IsCleared(TNode<MaybeObject> value) {
   return Word32Equal(TruncateIntPtrToInt32(BitcastMaybeObjectToWord(value)),
                      Int32Constant(kClearedWeakHeapObjectLower32));

src/codegen/code-stub-assembler.h

@@ -1473,10 +1473,14 @@ class V8_EXPORT_PRIVATE CodeStubAssembler
                            TVariable<Object>* extracted);
   // See MaybeObject for semantics of these functions.
   TNode<BoolT> IsStrong(TNode<MaybeObject> value);
+  TNode<BoolT> IsStrong(TNode<HeapObjectReference> value);
   TNode<HeapObject> GetHeapObjectIfStrong(TNode<MaybeObject> value,
                                           Label* if_not_strong);
+  TNode<HeapObject> GetHeapObjectIfStrong(TNode<HeapObjectReference> value,
+                                          Label* if_not_strong);
 
   TNode<BoolT> IsWeakOrCleared(TNode<MaybeObject> value);
+  TNode<BoolT> IsWeakOrCleared(TNode<HeapObjectReference> value);
   TNode<BoolT> IsCleared(TNode<MaybeObject> value);
   TNode<BoolT> IsNotCleared(TNode<MaybeObject> value) {
     return Word32BinaryNot(IsCleared(value));

src/compiler/code-assembler.cc

@@ -230,25 +230,6 @@ bool CodeAssembler::IsWord64CtzSupported() const {
   return raw_assembler()->machine()->Word64Ctz().IsSupported();
 }
 
-#ifdef DEBUG
-void CodeAssembler::GenerateCheckMaybeObjectIsObject(TNode<MaybeObject> node,
-                                                     const char* location) {
-  Label ok(this);
-  GotoIf(WordNotEqual(WordAnd(BitcastMaybeObjectToWord(node),
-                              IntPtrConstant(kHeapObjectTagMask)),
-                      IntPtrConstant(kWeakHeapObjectTag)),
-         &ok);
-  base::EmbeddedVector<char, 1024> message;
-  SNPrintF(message, "no Object: %s", location);
-  TNode<String> message_node = StringConstant(message.begin());
-  // This somewhat misuses the AbortCSADcheck runtime function. This will print
-  // "abort: CSA_DCHECK failed: <message>", which is good enough.
-  AbortCSADcheck(message_node);
-  Unreachable();
-  Bind(&ok);
-}
-#endif
-
 TNode<Int32T> CodeAssembler::Int32Constant(int32_t value) {
   return UncheckedCast<Int32T>(jsgraph()->Int32Constant(value));
 }

src/compiler/code-assembler.h

@@ -147,6 +147,7 @@ OBJECT_TYPE_CASE(Object)
 OBJECT_TYPE_CASE(Smi)
 OBJECT_TYPE_CASE(TaggedIndex)
 OBJECT_TYPE_CASE(HeapObject)
+OBJECT_TYPE_CASE(HeapObjectReference)
 OBJECT_TYPE_LIST(OBJECT_TYPE_CASE)
 HEAP_OBJECT_ORDINARY_TYPE_LIST(OBJECT_TYPE_CASE)
 STRUCT_LIST(OBJECT_TYPE_STRUCT_CASE)
@@ -425,7 +426,8 @@ class V8_EXPORT_PRIVATE CodeAssembler {
 
       static_assert(types_have_common_values<A, PreviousType>::value,
                     "Incompatible types: this cast can never succeed.");
-      static_assert(std::is_convertible<TNode<A>, TNode<Object>>::value,
+      static_assert(std::is_convertible<TNode<A>, TNode<MaybeObject>>::value ||
+                        std::is_convertible<TNode<A>, TNode<Object>>::value,
                     "Coercion to untagged values cannot be "
                     "checked.");
       static_assert(
@@ -434,10 +436,6 @@ class V8_EXPORT_PRIVATE CodeAssembler {
           "Unnecessary CAST: types are convertible.");
 #ifdef DEBUG
       if (FLAG_debug_code) {
-        if (std::is_same<PreviousType, MaybeObject>::value) {
-          code_assembler_->GenerateCheckMaybeObjectIsObject(
-              TNode<MaybeObject>::UncheckedCast(node_), location_);
-        }
         TNode<ExternalReference> function = code_assembler_->ExternalConstant(
             ExternalReference::check_object_type());
         code_assembler_->CallCFunction(
@@ -502,11 +500,6 @@ class V8_EXPORT_PRIVATE CodeAssembler {
 #define TORQUE_CAST(x) ca_.Cast(x)
 #endif
 
-#ifdef DEBUG
-  void GenerateCheckMaybeObjectIsObject(TNode<MaybeObject> node,
-                                        const char* location);
-#endif
-
   // Constants.
   TNode<Int32T> Int32Constant(int32_t value);
   TNode<Int64T> Int64Constant(int64_t value);

src/diagnostics/objects-printer.cc

@@ -2599,31 +2599,6 @@ void PreparseData::PreparseDataPrint(std::ostream& os) {
   os << "\n";
 }
 
-template <HeapObjectReferenceType kRefType, typename StorageType>
-void TaggedImpl<kRefType, StorageType>::Print() {
-  StdoutStream os;
-  this->Print(os);
-  os << std::flush;
-}
-
-template <HeapObjectReferenceType kRefType, typename StorageType>
-void TaggedImpl<kRefType, StorageType>::Print(std::ostream& os) {
-  Smi smi;
-  HeapObject heap_object;
-  if (ToSmi(&smi)) {
-    smi.SmiPrint(os);
-  } else if (IsCleared()) {
-    os << "[cleared]";
-  } else if (GetHeapObjectIfWeak(&heap_object)) {
-    os << "[weak] ";
-    heap_object.HeapObjectPrint(os);
-  } else if (GetHeapObjectIfStrong(&heap_object)) {
-    heap_object.HeapObjectPrint(os);
-  } else {
-    UNREACHABLE();
-  }
-}
-
 void HeapNumber::HeapNumberPrint(std::ostream& os) {
   HeapNumberShortPrint(os);
   os << "\n";

src/ic/accessor-assembler.cc

@@ -69,7 +69,7 @@ TNode<MaybeObject> AccessorAssembler::LoadHandlerDataField(
   return LoadMaybeWeakObjectField(handler, offset);
 }
 
-TNode<MaybeObject> AccessorAssembler::TryMonomorphicCase(
+TNode<HeapObjectReference> AccessorAssembler::TryMonomorphicCase(
     TNode<TaggedIndex> slot, TNode<FeedbackVector> vector,
     TNode<Map> lookup_start_object_map, Label* if_handler,
     TVariable<MaybeObject>* var_handler, Label* if_miss) {
@@ -84,9 +84,8 @@ TNode<MaybeObject> AccessorAssembler::TryMonomorphicCase(
   // into ElementOffsetFromIndex() allows it to be folded into a single
   // [base, index, offset] indirect memory access on x64.
   TNode<IntPtrT> offset = ElementOffsetFromIndex(slot, HOLEY_ELEMENTS);
-  TNode<MaybeObject> feedback = ReinterpretCast<MaybeObject>(
-      Load(MachineType::AnyTagged(), vector,
-           IntPtrAdd(offset, IntPtrConstant(header_size))));
+  TNode<HeapObjectReference> feedback = CAST(Load<MaybeObject>(
+      vector, IntPtrAdd(offset, IntPtrConstant(header_size))));
 
   // Try to quickly handle the monomorphic case without knowing for sure
   // if we have a weak reference in feedback.
@@ -1383,7 +1382,8 @@ void AccessorAssembler::HandleStoreICHandlerCase(
 
   BIND(&if_nonsmi_handler);
   {
-    GotoIf(IsWeakOrCleared(handler), &store_transition_or_global);
+    TNode<HeapObjectReference> ref_handler = CAST(handler);
+    GotoIf(IsWeakOrCleared(ref_handler), &store_transition_or_global);
     TNode<HeapObject> strong_handler = CAST(handler);
     TNode<Map> handler_map = LoadMap(strong_handler);
     Branch(IsCodeTMap(handler_map), &call_handler, &if_proto_handler);
@@ -3047,7 +3047,7 @@ void AccessorAssembler::LoadIC_BytecodeHandler(const LazyLoadICParameters* p,
     TVARIABLE(MaybeObject, var_handler);
     Label try_polymorphic(this), if_handler(this, &var_handler);
 
-    TNode<MaybeObject> feedback = TryMonomorphicCase(
+    TNode<HeapObjectReference> feedback = TryMonomorphicCase(
         p->slot(), CAST(p->vector()), lookup_start_object_map, &if_handler,
         &var_handler, &try_polymorphic);
 
@@ -3113,7 +3113,7 @@ void AccessorAssembler::LoadIC(const LoadICParameters* p) {
   GotoIf(IsUndefined(p->vector()), &no_feedback);
 
   // Check monomorphic case.
-  TNode<MaybeObject> feedback =
+  TNode<HeapObjectReference> feedback =
       TryMonomorphicCase(p->slot(), CAST(p->vector()), lookup_start_object_map,
                          &if_handler, &var_handler, &try_polymorphic);
   BIND(&if_handler);
@@ -3169,7 +3169,7 @@ void AccessorAssembler::LoadSuperIC(const LoadICParameters* p) {
   TNode<Map> lookup_start_object_map = LoadMap(CAST(p->lookup_start_object()));
   GotoIf(IsDeprecatedMap(lookup_start_object_map), &miss);
 
-  TNode<MaybeObject> feedback =
+  TNode<HeapObjectReference> feedback =
       TryMonomorphicCase(p->slot(), CAST(p->vector()), lookup_start_object_map,
                          &if_handler, &var_handler, &try_polymorphic);
 
@@ -3480,7 +3480,7 @@ void AccessorAssembler::KeyedLoadIC(const LoadICParameters* p,
   GotoIf(IsUndefined(p->vector()), &generic);
 
   // Check monomorphic case.
-  TNode<MaybeObject> feedback =
+  TNode<HeapObjectReference> feedback =
       TryMonomorphicCase(p->slot(), CAST(p->vector()), lookup_start_object_map,
                          &if_handler, &var_handler, &try_polymorphic);
   BIND(&if_handler);
@@ -3731,7 +3731,7 @@ void AccessorAssembler::StoreIC(const StoreICParameters* p) {
   GotoIf(IsUndefined(p->vector()), &no_feedback);
 
   // Check monomorphic case.
-  TNode<MaybeObject> feedback =
+  TNode<HeapObjectReference> feedback =
       TryMonomorphicCase(p->slot(), CAST(p->vector()), receiver_map,
                          &if_handler, &var_handler, &try_polymorphic);
   BIND(&if_handler);
@@ -3929,7 +3929,7 @@ void AccessorAssembler::KeyedStoreIC(const StoreICParameters* p) {
     GotoIf(IsUndefined(p->vector()), &no_feedback);
 
     // Check monomorphic case.
-    TNode<MaybeObject> feedback =
+    TNode<HeapObjectReference> feedback =
         TryMonomorphicCase(p->slot(), CAST(p->vector()), receiver_map,
                            &if_handler, &var_handler, &try_polymorphic);
     BIND(&if_handler);
@@ -4003,7 +4003,7 @@ void AccessorAssembler::DefineKeyedOwnIC(const StoreICParameters* p) {
     GotoIf(IsUndefined(p->vector()), &no_feedback);
 
     // Check monomorphic case.
-    TNode<MaybeObject> feedback =
+    TNode<HeapObjectReference> feedback =
         TryMonomorphicCase(p->slot(), CAST(p->vector()), receiver_map,
                            &if_handler, &var_handler, &try_polymorphic);
     BIND(&if_handler);
@@ -4074,7 +4074,7 @@ void AccessorAssembler::StoreInArrayLiteralIC(const StoreICParameters* p) {
 
     GotoIf(IsUndefined(p->vector()), &no_feedback);
 
-    TNode<MaybeObject> feedback =
+    TNode<HeapObjectReference> feedback =
         TryMonomorphicCase(p->slot(), CAST(p->vector()), array_map, &if_handler,
                            &var_handler, &try_polymorphic);
 
@@ -4879,7 +4879,7 @@ void AccessorAssembler::GenerateCloneObjectIC() {
 
   GotoIf(IsUndefined(maybe_vector), &slow);
 
-  TNode<MaybeObject> feedback =
+  TNode<HeapObjectReference> feedback =
       TryMonomorphicCase(slot, CAST(maybe_vector), source_map, &if_handler,
                          &var_handler, &try_polymorphic);
 

src/ic/accessor-assembler.h

@@ -346,12 +346,10 @@ class V8_EXPORT_PRIVATE AccessorAssembler : public CodeStubAssembler {
   // IC dispatcher behavior.
 
   // Checks monomorphic case. Returns {feedback} entry of the vector.
-  TNode<MaybeObject> TryMonomorphicCase(TNode<TaggedIndex> slot,
-                                        TNode<FeedbackVector> vector,
-                                        TNode<Map> lookup_start_object_map,
-                                        Label* if_handler,
-                                        TVariable<MaybeObject>* var_handler,
-                                        Label* if_miss);
+  TNode<HeapObjectReference> TryMonomorphicCase(
+      TNode<TaggedIndex> slot, TNode<FeedbackVector> vector,
+      TNode<Map> lookup_start_object_map, Label* if_handler,
+      TVariable<MaybeObject>* var_handler, Label* if_miss);
   void HandlePolymorphicCase(TNode<Map> lookup_start_object_map,
                              TNode<WeakFixedArray> feedback, Label* if_handler,
                              TVariable<MaybeObject>* var_handler,

src/interpreter/interpreter-assembler.cc

@@ -868,8 +868,8 @@ TNode<Object> InterpreterAssembler::ConstructWithSpread(
   IncrementCallCount(feedback_vector, slot_id);
 
   // Check if we have monomorphic {new_target} feedback already.
-  TNode<MaybeObject> feedback =
-      LoadFeedbackVectorSlot(feedback_vector, slot_id);
+  TNode<HeapObjectReference> feedback =
+      CAST(LoadFeedbackVectorSlot(feedback_vector, slot_id));
   Branch(IsWeakReferenceToObject(feedback, new_target), &construct,
          &extra_checks);
 

src/objects/object-type.cc

@@ -14,11 +14,42 @@ namespace internal {
 Address CheckObjectType(Address raw_value, Address raw_type,
                         Address raw_location) {
 #ifdef DEBUG
-  Object value(raw_value);
-  Smi type(raw_type);
+  ObjectType type = static_cast<ObjectType>(Smi(raw_type).value());
   String location = String::cast(Object(raw_location));
   const char* expected;
-  switch (static_cast<ObjectType>(type.value())) {
+
+  if (HAS_WEAK_HEAP_OBJECT_TAG(raw_value)) {
+    if (type == ObjectType::kHeapObjectReference) return Smi::FromInt(0).ptr();
+    // Casts of weak references are not allowed, one should use
+    // GetHeapObjectIfStrong / GetHeapObjectAssumeWeak first.
+    switch (type) {
+#define TYPE_CASE(Name)     \
+  case ObjectType::k##Name: \
+    expected = #Name;       \
+    break;
+#define TYPE_STRUCT_CASE(NAME, Name, name) \
+  case ObjectType::k##Name:                \
+    expected = #Name;                      \
+    break;
+
+      TYPE_CASE(Object)
+      TYPE_CASE(Smi)
+      TYPE_CASE(TaggedIndex)
+      TYPE_CASE(HeapObject)
+      TYPE_CASE(HeapObjectReference)
+      OBJECT_TYPE_LIST(TYPE_CASE)
+      HEAP_OBJECT_TYPE_LIST(TYPE_CASE)
+      STRUCT_LIST(TYPE_STRUCT_CASE)
+#undef TYPE_CASE
+#undef TYPE_STRUCT_CASE
+    }
+  } else {
+    Object value(raw_value);
+    switch (type) {
+      case ObjectType::kHeapObjectReference:
+        if (!value.IsSmi()) return Smi::FromInt(0).ptr();
+        expected = "HeapObjectReference";
+        break;
 #define TYPE_CASE(Name)                                 \
   case ObjectType::k##Name:                             \
     if (value.Is##Name()) return Smi::FromInt(0).ptr(); \
@@ -39,9 +70,11 @@ Address CheckObjectType(Address raw_value, Address raw_type,
     STRUCT_LIST(TYPE_STRUCT_CASE)
 #undef TYPE_CASE
 #undef TYPE_STRUCT_CASE
+    }
   }
+  MaybeObject maybe_value(raw_value);
   std::stringstream value_description;
-  value.Print(value_description);
+  maybe_value.Print(value_description);
   FATAL(
       "Type cast failed in %s\n"
       "  Expected %s but found %s",

src/objects/object-type.h

@@ -20,6 +20,7 @@ enum class ObjectType {
   ENUM_ELEMENT(Smi)                    //
   ENUM_ELEMENT(TaggedIndex)            //
   ENUM_ELEMENT(HeapObject)             //
+  ENUM_ELEMENT(HeapObjectReference)    //
   OBJECT_TYPE_LIST(ENUM_ELEMENT)       //
   HEAP_OBJECT_TYPE_LIST(ENUM_ELEMENT)  //
   STRUCT_LIST(ENUM_STRUCT_ELEMENT)     //

src/objects/tagged-impl.cc

@@ -7,6 +7,8 @@
 #include <sstream>
 
 #include "src/objects/objects.h"
+#include "src/objects/smi.h"
+#include "src/objects/tagged-impl-inl.h"
 #include "src/strings/string-stream.h"
 #include "src/utils/ostreams.h"
 
@@ -52,6 +54,34 @@ void TaggedImpl<kRefType, StorageType>::ShortPrint(std::ostream& os) {
   os << Brief(*this);
 }
 
+#ifdef OBJECT_PRINT
+template <HeapObjectReferenceType kRefType, typename StorageType>
+void TaggedImpl<kRefType, StorageType>::Print() {
+  StdoutStream os;
+  this->Print(os);
+  os << std::flush;
+}
+
+template <HeapObjectReferenceType kRefType, typename StorageType>
+void TaggedImpl<kRefType, StorageType>::Print(std::ostream& os) {
+  Smi smi(0);
+  HeapObject heap_object;
+  if (ToSmi(&smi)) {
+    os << "Smi: " << std::hex << "0x" << smi.value();
+    os << std::dec << " (" << smi.value() << ")\n";
+  } else if (IsCleared()) {
+    os << "[cleared]";
+  } else if (GetHeapObjectIfWeak(&heap_object)) {
+    os << "[weak] ";
+    heap_object.HeapObjectPrint(os);
+  } else if (GetHeapObjectIfStrong(&heap_object)) {
+    heap_object.HeapObjectPrint(os);
+  } else {
+    UNREACHABLE();
+  }
+}
+#endif  // OBJECT_PRINT
+
 // Explicit instantiation declarations.
 template class TaggedImpl<HeapObjectReferenceType::STRONG, Address>;
 template class TaggedImpl<HeapObjectReferenceType::WEAK, Address>;