[string] Micro-optimize String::GetFlatContent

- Add an inline fast path for flat strings.
- Efficient shape queries.
- Add constants for seq/external one/two-byte string tags.

Bug: v8:12195
Change-Id: Ia2c3d3c9207ab56bc4f0b425d21c7c93ad98aef4
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3259530
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Camillo Bruni <cbruni@chromium.org>
Cr-Commit-Position: refs/heads/main@{#77706}

src/codegen/code-stub-assembler.cc

@@ -6362,8 +6362,8 @@ TNode<BoolT> CodeStubAssembler::IsSeqOneByteStringInstanceType(
   CSA_DCHECK(this, IsStringInstanceType(instance_type));
   return Word32Equal(
       Word32And(instance_type,
-                Int32Constant(kStringRepresentationMask | kStringEncodingMask)),
-      Int32Constant(kSeqStringTag | kOneByteStringTag));
+                Int32Constant(kStringRepresentationAndEncodingMask)),
+      Int32Constant(kSeqOneByteStringTag));
 }
 
 TNode<BoolT> CodeStubAssembler::IsConsStringInstanceType(

src/objects/instance-type.h

@@ -46,6 +46,16 @@ const uint32_t kStringEncodingMask = 1 << 3;
 const uint32_t kTwoByteStringTag = 0;
 const uint32_t kOneByteStringTag = 1 << 3;
 
+// Combined tags for convenience (add more if needed).
+constexpr uint32_t kStringRepresentationAndEncodingMask =
+    kStringRepresentationMask | kStringEncodingMask;
+constexpr uint32_t kSeqOneByteStringTag = kSeqStringTag | kOneByteStringTag;
+constexpr uint32_t kSeqTwoByteStringTag = kSeqStringTag | kTwoByteStringTag;
+constexpr uint32_t kExternalOneByteStringTag =
+    kExternalStringTag | kOneByteStringTag;
+constexpr uint32_t kExternalTwoByteStringTag =
+    kExternalStringTag | kTwoByteStringTag;
+
 // For strings, bit 4 indicates whether the data pointer of an external string
 // is cached. Note that the string representation is expected to be
 // kExternalStringTag.

src/objects/string-inl.h

@@ -182,37 +182,37 @@ uint32_t StringShape::encoding_tag() const {
 }
 
 uint32_t StringShape::full_representation_tag() const {
-  return (type_ & (kStringRepresentationMask | kStringEncodingMask));
+  return (type_ & (kStringRepresentationAndEncodingMask));
 }
 
-STATIC_ASSERT((kStringRepresentationMask | kStringEncodingMask) ==
+STATIC_ASSERT((kStringRepresentationAndEncodingMask) ==
               Internals::kFullStringRepresentationMask);
 
 STATIC_ASSERT(static_cast<uint32_t>(kStringEncodingMask) ==
               Internals::kStringEncodingMask);
 
 bool StringShape::IsSequentialOneByte() const {
-  return full_representation_tag() == (kSeqStringTag | kOneByteStringTag);
+  return full_representation_tag() == kSeqOneByteStringTag;
 }
 
 bool StringShape::IsSequentialTwoByte() const {
-  return full_representation_tag() == (kSeqStringTag | kTwoByteStringTag);
+  return full_representation_tag() == kSeqTwoByteStringTag;
 }
 
 bool StringShape::IsExternalOneByte() const {
-  return full_representation_tag() == (kExternalStringTag | kOneByteStringTag);
+  return full_representation_tag() == kExternalOneByteStringTag;
 }
 
-STATIC_ASSERT((kExternalStringTag | kOneByteStringTag) ==
+STATIC_ASSERT(kExternalOneByteStringTag ==
               Internals::kExternalOneByteRepresentationTag);
 
 STATIC_ASSERT(v8::String::ONE_BYTE_ENCODING == kOneByteStringTag);
 
 bool StringShape::IsExternalTwoByte() const {
-  return full_representation_tag() == (kExternalStringTag | kTwoByteStringTag);
+  return full_representation_tag() == kExternalTwoByteStringTag;
 }
 
-STATIC_ASSERT((kExternalStringTag | kTwoByteStringTag) ==
+STATIC_ASSERT(kExternalTwoByteStringTag ==
               Internals::kExternalTwoByteRepresentationTag);
 
 STATIC_ASSERT(v8::String::TWO_BYTE_ENCODING == kTwoByteStringTag);
@@ -501,23 +501,23 @@ bool String::IsEqualToImpl(
   const Char* data = str.data();
   while (true) {
     int32_t type = string.map(cage_base).instance_type();
-    switch (type & (kStringRepresentationMask | kStringEncodingMask)) {
-      case kSeqStringTag | kOneByteStringTag:
+    switch (type & kStringRepresentationAndEncodingMask) {
+      case kSeqOneByteStringTag:
         return CompareCharsEqual(
             SeqOneByteString::cast(string).GetChars(no_gc, access_guard) +
                 slice_offset,
             data, len);
-      case kSeqStringTag | kTwoByteStringTag:
+      case kSeqTwoByteStringTag:
         return CompareCharsEqual(
             SeqTwoByteString::cast(string).GetChars(no_gc, access_guard) +
                 slice_offset,
             data, len);
-      case kExternalStringTag | kOneByteStringTag:
+      case kExternalOneByteStringTag:
         return CompareCharsEqual(
             ExternalOneByteString::cast(string).GetChars(cage_base) +
                 slice_offset,
             data, len);
-      case kExternalStringTag | kTwoByteStringTag:
+      case kExternalTwoByteStringTag:
         return CompareCharsEqual(
             ExternalTwoByteString::cast(string).GetChars(cage_base) +
                 slice_offset,
@@ -640,6 +640,45 @@ Handle<String> String::Flatten(LocalIsolate* isolate, Handle<String> string,
   return string;
 }
 
+// static
+base::Optional<String::FlatContent> String::TryGetFlatContentFromDirectString(
+    PtrComprCageBase cage_base, const DisallowGarbageCollection& no_gc,
+    String string, int offset, int length) {
+  DCHECK_GE(offset, 0);
+  DCHECK_GE(length, 0);
+  DCHECK_LE(offset + length, string.length());
+  switch (StringShape{string, cage_base}.full_representation_tag()) {
+    case kSeqOneByteStringTag:
+      return FlatContent(
+          SeqOneByteString::cast(string).GetChars(no_gc) + offset, length,
+          no_gc);
+    case kSeqTwoByteStringTag:
+      return FlatContent(
+          SeqTwoByteString::cast(string).GetChars(no_gc) + offset, length,
+          no_gc);
+    case kExternalOneByteStringTag:
+      return FlatContent(
+          ExternalOneByteString::cast(string).GetChars(cage_base) + offset,
+          length, no_gc);
+    case kExternalTwoByteStringTag:
+      return FlatContent(
+          ExternalTwoByteString::cast(string).GetChars(cage_base) + offset,
+          length, no_gc);
+    default:
+      return {};
+  }
+  UNREACHABLE();
+}
+
+String::FlatContent String::GetFlatContent(
+    const DisallowGarbageCollection& no_gc) {
+  PtrComprCageBase cage_base = GetPtrComprCageBase(*this);
+  base::Optional<FlatContent> flat_content =
+      TryGetFlatContentFromDirectString(cage_base, no_gc, *this, 0, length());
+  if (flat_content.has_value()) return flat_content.value();
+  return SlowGetFlatContent(no_gc);
+}
+
 uint16_t String::Get(int index) const {
   DCHECK(!SharedStringAccessGuardIfNeeded::IsNeeded(*this));
   return GetImpl(index, GetPtrComprCageBase(*this),
@@ -738,28 +777,28 @@ ConsString String::VisitFlat(
   while (true) {
     int32_t tag = StringShape(string, cage_base).full_representation_tag();
     switch (tag) {
-      case kSeqStringTag | kOneByteStringTag:
+      case kSeqOneByteStringTag:
         visitor->VisitOneByteString(
             SeqOneByteString::cast(string).GetChars(no_gc, access_guard) +
                 slice_offset,
             length - offset);
         return ConsString();
 
-      case kSeqStringTag | kTwoByteStringTag:
+      case kSeqTwoByteStringTag:
         visitor->VisitTwoByteString(
             SeqTwoByteString::cast(string).GetChars(no_gc, access_guard) +
                 slice_offset,
             length - offset);
         return ConsString();
 
-      case kExternalStringTag | kOneByteStringTag:
+      case kExternalOneByteStringTag:
         visitor->VisitOneByteString(
             ExternalOneByteString::cast(string).GetChars(cage_base) +
                 slice_offset,
             length - offset);
         return ConsString();
 
-      case kExternalStringTag | kTwoByteStringTag:
+      case kExternalTwoByteStringTag:
         visitor->VisitTwoByteString(
             ExternalTwoByteString::cast(string).GetChars(cage_base) +
                 slice_offset,

src/objects/string.cc

@@ -568,7 +568,7 @@ Handle<Object> String::ToNumber(Isolate* isolate, Handle<String> subject) {
   return isolate->factory()->NewNumber(StringToDouble(isolate, subject, flags));
 }
 
-String::FlatContent String::GetFlatContent(
+String::FlatContent String::SlowGetFlatContent(
     const DisallowGarbageCollection& no_gc) {
 #if DEBUG
   // Check that this method is called only from the main thread.
@@ -579,52 +579,39 @@ String::FlatContent String::GetFlatContent(
                    ThreadId::Current() == isolate->thread_id());
   }
 #endif
-  USE(no_gc);
+
   PtrComprCageBase cage_base = GetPtrComprCageBase(*this);
-  int length = this->length();
-  StringShape shape(*this, cage_base);
   String string = *this;
+  StringShape shape(string, cage_base);
   int offset = 0;
-  if (shape.representation_tag() == kConsStringTag) {
+
+  // Extract cons- and sliced strings.
+  if (shape.IsCons()) {
     ConsString cons = ConsString::cast(string);
-    if (cons.second(cage_base).length() != 0) {
-      return FlatContent(no_gc);
-    }
+    if (!cons.IsFlat(cage_base)) return FlatContent(no_gc);
     string = cons.first(cage_base);
     shape = StringShape(string, cage_base);
-  } else if (shape.representation_tag() == kSlicedStringTag) {
+  } else if (shape.IsSliced()) {
     SlicedString slice = SlicedString::cast(string);
     offset = slice.offset();
     string = slice.parent(cage_base);
     shape = StringShape(string, cage_base);
-    DCHECK(shape.representation_tag() != kConsStringTag &&
-           shape.representation_tag() != kSlicedStringTag);
   }
-  if (shape.representation_tag() == kThinStringTag) {
+
+  DCHECK(!shape.IsCons());
+  DCHECK(!shape.IsSliced());
+
+  // Extract thin strings.
+  if (shape.IsThin()) {
     ThinString thin = ThinString::cast(string);
     string = thin.actual(cage_base);
     shape = StringShape(string, cage_base);
-    DCHECK(!shape.IsCons());
-    DCHECK(!shape.IsSliced());
-  }
-  if (shape.encoding_tag() == kOneByteStringTag) {
-    const uint8_t* start;
-    if (shape.representation_tag() == kSeqStringTag) {
-      start = SeqOneByteString::cast(string).GetChars(no_gc);
-    } else {
-      start = ExternalOneByteString::cast(string).GetChars(cage_base);
-    }
-    return FlatContent(start + offset, length, no_gc);
-  } else {
-    DCHECK_EQ(shape.encoding_tag(), kTwoByteStringTag);
-    const base::uc16* start;
-    if (shape.representation_tag() == kSeqStringTag) {
-      start = SeqTwoByteString::cast(string).GetChars(no_gc);
-    } else {
-      start = ExternalTwoByteString::cast(string).GetChars(cage_base);
-    }
-    return FlatContent(start + offset, length, no_gc);
   }
+
+  DCHECK(shape.IsDirect());
+  return TryGetFlatContentFromDirectString(cage_base, no_gc, string, offset,
+                                           length())
+      .value();
 }
 
 std::unique_ptr<char[]> String::ToCString(AllowNullsFlag allow_nulls,
@@ -919,17 +906,19 @@ bool String::SlowEquals(
 bool String::SlowEquals(Isolate* isolate, Handle<String> one,
                         Handle<String> two) {
   // Fast check: negative check with lengths.
-  int one_length = one->length();
+  const int one_length = one->length();
   if (one_length != two->length()) return false;
   if (one_length == 0) return true;
 
   // Fast check: if at least one ThinString is involved, dereference it/them
   // and restart.
   if (one->IsThinString() || two->IsThinString()) {
-    if (one->IsThinString())
+    if (one->IsThinString()) {
       one = handle(ThinString::cast(*one).actual(), isolate);
-    if (two->IsThinString())
+    }
+    if (two->IsThinString()) {
       two = handle(ThinString::cast(*two).actual(), isolate);
+    }
     return String::Equals(isolate, one, two);
   }
 
@@ -967,12 +956,17 @@ bool String::SlowEquals(Isolate* isolate, Handle<String> one,
   if (flat1.IsOneByte() && flat2.IsOneByte()) {
     return CompareCharsEqual(flat1.ToOneByteVector().begin(),
                              flat2.ToOneByteVector().begin(), one_length);
-  } else {
-    for (int i = 0; i < one_length; i++) {
-      if (flat1.Get(i) != flat2.Get(i)) return false;
-    }
-    return true;
+  } else if (flat1.IsTwoByte() && flat2.IsTwoByte()) {
+    return CompareCharsEqual(flat1.ToUC16Vector().begin(),
+                             flat2.ToUC16Vector().begin(), one_length);
+  } else if (flat1.IsOneByte() && flat2.IsTwoByte()) {
+    return CompareCharsEqual(flat1.ToOneByteVector().begin(),
+                             flat2.ToUC16Vector().begin(), one_length);
+  } else if (flat1.IsTwoByte() && flat2.IsOneByte()) {
+    return CompareCharsEqual(flat1.ToUC16Vector().begin(),
+                             flat2.ToOneByteVector().begin(), one_length);
   }
+  UNREACHABLE();
 }
 
 // static

src/objects/string.h

@@ -263,7 +263,7 @@ class String : public TorqueGeneratedString<String, Name> {
   // If the string isn't flat, and therefore doesn't have flat content, the
   // returned structure will report so, and can't provide a vector of either
   // kind.
-  V8_EXPORT_PRIVATE FlatContent
+  V8_EXPORT_PRIVATE V8_INLINE FlatContent
   GetFlatContent(const DisallowGarbageCollection& no_gc);
 
   // Returns the parent of a sliced string or first part of a flat cons string.
@@ -586,6 +586,13 @@ class String : public TorqueGeneratedString<String, Name> {
   V8_EXPORT_PRIVATE static Handle<String> SlowFlatten(
       Isolate* isolate, Handle<ConsString> cons, AllocationType allocation);
 
+  V8_EXPORT_PRIVATE V8_INLINE static base::Optional<FlatContent>
+  TryGetFlatContentFromDirectString(PtrComprCageBase cage_base,
+                                    const DisallowGarbageCollection& no_gc,
+                                    String string, int offset, int length);
+  V8_EXPORT_PRIVATE FlatContent
+  SlowGetFlatContent(const DisallowGarbageCollection& no_gc);
+
   static Handle<String> SlowCopy(Isolate* isolate, Handle<SeqString> source,
                                  AllocationType allocation);