[torque] Support 'real' internal classes

Rework the implementation of non-external Torque classes to use
Struct machinery rather than FixedArray machinery. This allows
Torque-only defined 'internal' classes to the automatically generate
class verifiers and printers.

As part of this change, generate C++ boilerplate accessors for
internal Torque classes, since this is a pre-requisite for the
verifiers, printers and other Struct-based functionality.

Moreover, augment the header-generating functionality in Torque
to create separate header files for field offset definitions,
internal class C++ definitions and instance types.

Bug: v8:7793
Change-Id: I47d5f1570040c2b44d378f23b6cf95d3d132dacc
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1607645
Commit-Queue: Sigurd Schneider <sigurds@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#62317}

BUILD.gn

@@ -1024,6 +1024,7 @@ action("run_torque") {
     "$target_gen_dir/torque-generated/exported-macros-assembler-tq.cc",
     "$target_gen_dir/torque-generated/exported-macros-assembler-tq.h",
     "$target_gen_dir/torque-generated/csa-types-tq.h",
+    "$target_gen_dir/torque-generated/instance-types-tq.h",
   ]
   foreach(file, torque_files) {
     filetq = string_replace(file, ".tq", "-tq-csa")

src/builtins/base.tq

@@ -247,8 +247,7 @@ extern class DescriptorArray extends HeapObject {
 // than building the definition from C++.
 intrinsic %GetAllocationBaseSize<Class: type>(map: Map): intptr;
 intrinsic %Allocate<Class: type>(size: intptr): Class;
-intrinsic %AllocateInternalClass<Class: type>(slotCount: constexpr intptr):
-    Class;
+intrinsic %GetStructMap(instanceKind: constexpr InstanceType): Map;
 
 intrinsic %AddIndexedFieldSizeToObjectSize<T: type>(
     baseSize: intptr, indexSize: T, fieldSize: int32): intptr {

src/codegen/code-stub-assembler.cc

@@ -1473,6 +1473,12 @@ TNode<Float64T> CodeStubAssembler::LoadHeapNumberValue(
       object, HeapNumber::kValueOffset, MachineType::Float64()));
 }
 
+TNode<Map> CodeStubAssembler::GetStructMap(InstanceType instance_type) {
+  Handle<Map> map_handle(Map::GetStructMap(isolate(), instance_type),
+                         isolate());
+  return HeapConstant(map_handle);
+}
+
 TNode<Map> CodeStubAssembler::LoadMap(SloppyTNode<HeapObject> object) {
   return UncheckedCast<Map>(LoadObjectField(object, HeapObject::kMapOffset,
                                             MachineType::TaggedPointer()));

src/codegen/code-stub-assembler.h

@@ -1717,6 +1717,8 @@ class V8_EXPORT_PRIVATE CodeStubAssembler
                               fixed_array_map);
   }
 
+  TNode<Map> GetStructMap(InstanceType instance_type);
+
   TNode<FixedArray> AllocateUninitializedFixedArray(intptr_t capacity) {
     return UncheckedCast<FixedArray>(AllocateFixedArray(
         PACKED_ELEMENTS, IntPtrConstant(capacity), AllocationFlag::kNone));

src/compiler/code-assembler.h

@@ -73,6 +73,9 @@ class PromiseReactionJobTask;
 class PromiseRejectReactionJobTask;
 class WasmDebugInfo;
 class Zone;
+#define MAKE_FORWARD_DECLARATION(V, NAME, Name, name) class Name;
+TORQUE_STRUCT_LIST_GENERATOR(MAKE_FORWARD_DECLARATION, UNUSED)
+#undef MAKE_FORWARD_DECLARATION
 
 template <typename T>
 class Signature;

src/compiler/types.cc

@@ -363,6 +363,9 @@ Type::bitset BitsetType::Lub(const MapRefLike& map) {
     case PROMISE_REJECT_REACTION_JOB_TASK_TYPE:
     case PROMISE_RESOLVE_THENABLE_JOB_TASK_TYPE:
     case FINALIZATION_GROUP_CLEANUP_JOB_TASK_TYPE:
+#define MAKE_TORQUE_CLASS_TYPE(V) case V:
+      TORQUE_DEFINED_INSTANCE_TYPES(MAKE_TORQUE_CLASS_TYPE)
+#undef MAKE_TORQUE_CLASS_TYPE
       UNREACHABLE();
   }
   UNREACHABLE();

src/diagnostics/objects-debug.cc

@@ -67,6 +67,7 @@
 #include "src/utils/ostreams.h"
 #include "src/wasm/wasm-objects-inl.h"
 #include "torque-generated/class-verifiers-tq.h"
+#include "torque-generated/internal-class-definitions-tq-inl.h"
 
 namespace v8 {
 namespace internal {

src/diagnostics/objects-printer.cc

@@ -65,6 +65,8 @@
 #include "src/wasm/wasm-code-manager.h"
 #include "src/wasm/wasm-engine.h"
 #include "src/wasm/wasm-objects-inl.h"
+#include "torque-generated/class-definitions-tq-inl.h"
+#include "torque-generated/internal-class-definitions-tq-inl.h"
 
 namespace v8 {
 namespace internal {

src/heap/factory.cc

@@ -1611,17 +1611,7 @@ Handle<Context> Factory::NewBuiltinContext(Handle<NativeContext> native_context,
 
 Handle<Struct> Factory::NewStruct(InstanceType type,
                                   AllocationType allocation) {
-  Map map;
-  switch (type) {
-#define MAKE_CASE(TYPE, Name, name) \
-  case TYPE:                        \
-    map = *name##_map();            \
-    break;
-    STRUCT_LIST(MAKE_CASE)
-#undef MAKE_CASE
-    default:
-      UNREACHABLE();
-  }
+  Map map = Map::GetStructMap(isolate(), type);
   int size = map.instance_size();
   HeapObject result = AllocateRawWithImmortalMap(size, allocation, map);
   Handle<Struct> str(Struct::cast(result), isolate());

src/heap/setup-heap-internal.cc

@@ -42,6 +42,8 @@
 #include "src/objects/template-objects-inl.h"
 #include "src/regexp/regexp.h"
 #include "src/wasm/wasm-objects.h"
+#include "torque-generated/class-definitions-tq.h"
+#include "torque-generated/internal-class-definitions-tq-inl.h"
 
 namespace v8 {
 namespace internal {

src/objects/instance-type.h

@@ -11,6 +11,8 @@
 // Has to be the last include (doesn't have include guards):
 #include "src/objects/object-macros.h"
 
+#include "torque-generated/instance-types-tq.h"
+
 namespace v8 {
 namespace internal {
 
@@ -176,7 +178,11 @@ enum InstanceType : uint16_t {
   PROMISE_RESOLVE_THENABLE_JOB_TASK_TYPE,
   FINALIZATION_GROUP_CLEANUP_JOB_TASK_TYPE,  // LAST_MICROTASK_TYPE
 
-  ALLOCATION_SITE_TYPE,
+#define MAKE_TORQUE_INSTANCE_TYPE(V) V,
+  TORQUE_DEFINED_INSTANCE_TYPES(MAKE_TORQUE_INSTANCE_TYPE)
+#undef MAKE_TORQUE_INSTANCE_TYPE
+
+      ALLOCATION_SITE_TYPE,
   EMBEDDER_DATA_ARRAY_TYPE,
   // FixedArrays.
   FIXED_ARRAY_TYPE,  // FIRST_FIXED_ARRAY_TYPE

src/objects/map.cc

@@ -85,6 +85,21 @@ void Map::PrintReconfiguration(Isolate* isolate, FILE* file, int modify_index,
   os << "]\n";
 }
 
+Map Map::GetStructMap(Isolate* isolate, InstanceType type) {
+  Map map;
+  switch (type) {
+#define MAKE_CASE(TYPE, Name, name)            \
+  case TYPE:                                   \
+    map = ReadOnlyRoots(isolate).name##_map(); \
+    break;
+    STRUCT_LIST(MAKE_CASE)
+#undef MAKE_CASE
+    default:
+      UNREACHABLE();
+  }
+  return map;
+}
+
 VisitorId Map::GetVisitorId(Map map) {
   STATIC_ASSERT(kVisitorIdCount <= 256);
 

src/objects/map.h

@@ -797,6 +797,8 @@ class Map : public HeapObject {
 
   inline bool CanTransition() const;
 
+  static Map GetStructMap(Isolate* isolate, InstanceType type);
+
 #define DECL_TESTER(Type, ...) inline bool Is##Type##Map() const;
   INSTANCE_TYPE_CHECKERS(DECL_TESTER)
 #undef DECL_TESTER

src/objects/objects-definitions.h

@@ -7,6 +7,8 @@
 
 #include "src/init/heap-symbols.h"
 
+#include "torque-generated/instance-types-tq.h"
+
 namespace v8 {
 
 namespace internal {
@@ -115,6 +117,8 @@ namespace internal {
   V(PROMISE_RESOLVE_THENABLE_JOB_TASK_TYPE)              \
   V(FINALIZATION_GROUP_CLEANUP_JOB_TASK_TYPE)            \
                                                          \
+  TORQUE_DEFINED_INSTANCE_TYPES(V)                       \
+                                                         \
   V(ALLOCATION_SITE_TYPE)                                \
   V(EMBEDDER_DATA_ARRAY_TYPE)                            \
                                                          \
@@ -347,14 +351,18 @@ namespace internal {
 #define STRUCT_LIST_ADAPTER(V, NAME, Name, name) V(NAME, Name, name)
 
 // Produces (NAME, Name, name) entries.
-#define STRUCT_LIST(V) STRUCT_LIST_GENERATOR(STRUCT_LIST_ADAPTER, V)
+#define STRUCT_LIST(V)                          \
+  STRUCT_LIST_GENERATOR(STRUCT_LIST_ADAPTER, V) \
+  TORQUE_STRUCT_LIST_GENERATOR(STRUCT_LIST_ADAPTER, V)
 
 // Adapts one STRUCT_LIST_GENERATOR entry to the STRUCT_MAPS_LIST entry
 #define STRUCT_MAPS_LIST_ADAPTER(V, NAME, Name, name) \
   V(Map, name##_map, Name##Map)
 
 // Produces (Map, struct_name_map, StructNameMap) entries
-#define STRUCT_MAPS_LIST(V) STRUCT_LIST_GENERATOR(STRUCT_MAPS_LIST_ADAPTER, V)
+#define STRUCT_MAPS_LIST(V)                          \
+  STRUCT_LIST_GENERATOR(STRUCT_MAPS_LIST_ADAPTER, V) \
+  TORQUE_STRUCT_LIST_GENERATOR(STRUCT_MAPS_LIST_ADAPTER, V)
 
 //
 // The following macros define list of allocation size objects and list of

src/objects/objects.cc

@@ -116,6 +116,9 @@
 #include "src/wasm/wasm-objects.h"
 #include "src/zone/zone.h"
 
+#include "torque-generated/class-definitions-tq-inl.h"
+#include "torque-generated/internal-class-definitions-tq-inl.h"
+
 namespace v8 {
 namespace internal {
 

src/torque/constants.h

@@ -18,6 +18,8 @@ static const char* const CONSTEXPR_TYPE_PREFIX = "constexpr ";
 static const char* const NEVER_TYPE_STRING = "never";
 static const char* const CONSTEXPR_BOOL_TYPE_STRING = "constexpr bool";
 static const char* const CONSTEXPR_INTPTR_TYPE_STRING = "constexpr intptr";
+static const char* const CONSTEXPR_INSTANCE_TYPE_TYPE_STRING =
+    "constexpr InstanceType";
 static const char* const BOOL_TYPE_STRING = "bool";
 static const char* const VOID_TYPE_STRING = "void";
 static const char* const ARGUMENTS_TYPE_STRING = "Arguments";

src/torque/csa-generator.cc

@@ -256,9 +256,8 @@ void CSAGenerator::EmitInstruction(const CallIntrinsicInstruction& instruction,
   } else if (instruction.intrinsic->ExternalName() == "%Allocate") {
     out_ << "ca_.UncheckedCast<" << return_type->GetGeneratedTNodeTypeName()
          << ">(CodeStubAssembler(state_).Allocate";
-  } else if (instruction.intrinsic->ExternalName() ==
-             "%AllocateInternalClass") {
-    out_ << "CodeStubAssembler(state_).AllocateUninitializedFixedArray";
+  } else if (instruction.intrinsic->ExternalName() == "%GetStructMap") {
+    out_ << "CodeStubAssembler(state_).GetStructMap";
   } else {
     ReportError("no built in intrinsic with name " +
                 instruction.intrinsic->ExternalName());
@@ -717,12 +716,8 @@ void CSAGenerator::EmitInstruction(
 
   out_ << "    compiler::TNode<IntPtrT> " << offset_name
        << " = ca_.IntPtrConstant(";
-  if (instruction.class_type->IsExtern()) {
     out_ << field.aggregate->GetGeneratedTNodeTypeName() << "::k"
          << CamelifyString(field.name_and_type.name) << "Offset";
-  } else {
-    out_ << "FixedArray::kHeaderSize + " << field.offset;
-  }
   out_ << ");\n"
        << "    USE(" << stack->Top() << ");\n";
 }

src/torque/implementation-visitor.h

@@ -174,7 +174,7 @@ class LocationReference {
 
 struct InitializerResults {
   std::vector<Identifier*> names;
-  NameValueMap field_value_map;
+  std::map<std::string, VisitResult> field_value_map;
 };
 
 template <class T>
@@ -352,9 +352,11 @@ class ImplementationVisitor {
   void GenerateClassFieldOffsets(const std::string& output_directory);
   void GeneratePrintDefinitions(const std::string& output_directory);
   void GenerateClassDefinitions(const std::string& output_directory);
+  void GenerateInstanceTypes(const std::string& output_directory);
   void GenerateClassVerifiers(const std::string& output_directory);
   void GenerateExportedMacrosAssembler(const std::string& output_directory);
   void GenerateCSATypes(const std::string& output_directory);
+  void GenerateCppForInternalClasses(const std::string& output_directory);
 
   VisitResult Visit(Expression* expr);
   const Type* Visit(Statement* stmt);

src/torque/torque-compiler.cc

@@ -85,6 +85,8 @@ void CompileCurrentAst(TorqueCompilerOptions options) {
   implementation_visitor.GenerateClassVerifiers(output_directory);
   implementation_visitor.GenerateExportedMacrosAssembler(output_directory);
   implementation_visitor.GenerateCSATypes(output_directory);
+  implementation_visitor.GenerateInstanceTypes(output_directory);
+  implementation_visitor.GenerateCppForInternalClasses(output_directory);
 
   implementation_visitor.EndCSAFiles();
   implementation_visitor.GenerateImplementation(output_directory);

src/torque/type-oracle.h

@@ -107,6 +107,10 @@ class TypeOracle : public ContextualClass<TypeOracle> {
     return Get().GetBuiltinType(CONSTEXPR_INTPTR_TYPE_STRING);
   }
 
+  static const Type* GetConstexprInstanceTypeType() {
+    return Get().GetBuiltinType(CONSTEXPR_INSTANCE_TYPE_TYPE_STRING);
+  }
+
   static const Type* GetVoidType() {
     return Get().GetBuiltinType(VOID_TYPE_STRING);
   }

src/torque/type-visitor.cc

@@ -162,15 +162,24 @@ const ClassType* TypeVisitor::ComputeType(ClassDeclaration* decl) {
     new_class = TypeOracle::GetClassType(super_type, decl->name->value,
                                          decl->flags, generates, decl, alias);
   } else {
-    if (decl->super) {
-      ReportError("Only extern classes can inherit.");
+    if (!decl->super) {
+      ReportError("Intern class ", decl->name->value,
+                  " must extend class Struct.");
+    }
+    const Type* super_type = TypeVisitor::ComputeType(*decl->super);
+    const ClassType* super_class = ClassType::DynamicCast(super_type);
+    const Type* struct_type = Declarations::LookupGlobalType("Struct");
+    if (!super_class || super_class != struct_type) {
+      ReportError("Intern class ", decl->name->value,
+                  " must extend class Struct.");
     }
     if (decl->generates) {
       ReportError("Only extern classes can specify a generated type.");
     }
-    new_class =
-        TypeOracle::GetClassType(TypeOracle::GetTaggedType(), decl->name->value,
-                                 decl->flags, "FixedArray", decl, alias);
+    new_class = TypeOracle::GetClassType(
+        super_type, decl->name->value,
+        decl->flags | ClassFlag::kGeneratePrint | ClassFlag::kGenerateVerify,
+        decl->name->value, decl, alias);
   }
   return new_class;
 }

src/torque/types.cc

@@ -349,7 +349,7 @@ void ClassType::Finalize() const {
   TypeVisitor::VisitClassFieldsAndMethods(const_cast<ClassType*>(this),
                                           this->decl_);
   is_finalized_ = true;
-  if (GenerateCppClassDefinitions()) {
+  if (GenerateCppClassDefinitions() || !IsExtern()) {
     for (const Field& f : fields()) {
       if (f.is_weak) {
         Error("Generation of C++ class for Torque class ", name(),

src/torque/types.h

@@ -282,6 +282,8 @@ class V8_EXPORT_PRIVATE BuiltinPointerType final : public Type {
   }
   size_t function_pointer_type_id() const { return function_pointer_type_id_; }
 
+  std::vector<std::string> GetRuntimeTypes() const override { return {"Smi"}; }
+
  private:
   friend class TypeOracle;
   BuiltinPointerType(const Type* parent, TypeVector parameter_types,
@@ -526,10 +528,10 @@ class ClassType final : public AggregateType {
   std::string GetGeneratedTNodeTypeNameImpl() const override;
   bool IsExtern() const { return flags_ & ClassFlag::kExtern; }
   bool ShouldGeneratePrint() const {
-    return flags_ & ClassFlag::kGeneratePrint;
+    return flags_ & ClassFlag::kGeneratePrint || !IsExtern();
   }
   bool ShouldGenerateVerify() const {
-    return flags_ & ClassFlag::kGenerateVerify;
+    return flags_ & ClassFlag::kGenerateVerify || !IsExtern();
   }
   bool IsTransient() const override { return flags_ & ClassFlag::kTransient; }
   bool IsAbstract() const { return flags_ & ClassFlag::kAbstract; }
@@ -540,7 +542,7 @@ class ClassType final : public AggregateType {
     return flags_ & ClassFlag::kHasSameInstanceTypeAsParent;
   }
   bool GenerateCppClassDefinitions() const {
-    return flags_ & ClassFlag::kGenerateCppClassDefinitions;
+    return flags_ & ClassFlag::kGenerateCppClassDefinitions || !IsExtern();
   }
   bool HasIndexedField() const override;
   size_t size() const { return size_; }
@@ -606,8 +608,6 @@ class VisitResult {
   base::Optional<StackRange> stack_range_;
 };
 
-using NameValueMap = std::map<std::string, VisitResult>;
-
 VisitResult ProjectStructField(VisitResult structure,
                                const std::string& fieldname);
 

src/torque/utils.cc

@@ -246,6 +246,19 @@ std::string CamelifyString(const std::string& underscore_string) {
   return result;
 }
 
+std::string SnakeifyString(const std::string& camel_string) {
+  std::string result;
+  bool previousWasLower = false;
+  for (auto current : camel_string) {
+    if (previousWasLower && isupper(current)) {
+      result += "_";
+    }
+    result += tolower(current);
+    previousWasLower = (islower(current));
+  }
+  return result;
+}
+
 std::string DashifyString(const std::string& underscore_string) {
   std::string result = underscore_string;
   std::replace(result.begin(), result.end(), '_', '-');

src/torque/utils.h

@@ -94,6 +94,7 @@ template <class... Args>
 
 std::string CapifyStringWithUnderscores(const std::string& camellified_string);
 std::string CamelifyString(const std::string& underscore_string);
+std::string SnakeifyString(const std::string& camel_string);
 std::string DashifyString(const std::string& underscore_string);
 std::string UnderlinifyPath(std::string path);
 

test/torque/test-torque.tq

@@ -820,7 +820,7 @@ namespace test {
     check(a.b.GetX() == 2);
   }
 
-  class InternalClass {
+  class InternalClass extends Struct {
     Flip() labels NotASmi {
       const tmp = Cast<Smi>(this.b) otherwise NotASmi;
       this.b = this.a;
@@ -878,7 +878,7 @@ namespace test {
     return new FixedArray{map: kFixedArrayMap, length: 5, objects: ...i};
   }
 
-  class SmiPair {
+  class SmiPair extends Struct {
     GetA():&Smi {
       return & this.a;
     }
@@ -908,7 +908,7 @@ namespace test {
     StaticAssert(1 + 2 == 3);
   }
 
-  class SmiBox {
+  class SmiBox extends Struct {
     value: Smi;
     unrelated: Smi;
   }

third_party/v8/builtins/array-sort.tq

@@ -14,7 +14,7 @@
 // https://github.com/python/cpython/blob/master/Objects/listsort.txt
 
 namespace array {
-  class SortState {
+  class SortState extends Struct {
     Compare(implicit context: Context)(x: Object, y: Object): Number {
       const sortCompare: CompareBuiltinFn = this.sortComparePtr;
       return sortCompare(context, this.userCmpFn, x, y);