[torque] generate files per Torque file instead of per namespace

TBR: yangguo@chromium.org

Bug: v8:7793
Change-Id: Ie680d72a1da23038d3136f82532496eabbd9c363
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1632227
Commit-Queue: Tobias Tebbi <tebbi@chromium.org>
Reviewed-by: Yang Guo <yangguo@chromium.org>
Reviewed-by: Sigurd Schneider <sigurds@chromium.org>
Cr-Commit-Position: refs/heads/master@{#62004}

BUILD.gn

@@ -990,57 +990,6 @@ if (!v8_enable_i18n_support) {
   torque_files -= [ "src/objects/intl-objects.tq" ]
 }
 
-torque_namespaces = [
-  "arguments",
-  "array",
-  "array-copywithin",
-  "array-filter",
-  "array-find",
-  "array-findindex",
-  "array-foreach",
-  "array-join",
-  "array-map",
-  "array-of",
-  "array-reverse",
-  "array-shift",
-  "array-slice",
-  "array-splice",
-  "array-unshift",
-  "array-lastindexof",
-  "base",
-  "boolean",
-  "collections",
-  "data-view",
-  "extras-utils",
-  "growable-fixed-array",
-  "internal-coverage",
-  "iterator",
-  "math",
-  "object",
-  "proxy",
-  "regexp",
-  "regexp-replace",
-  "string",
-  "string-html",
-  "string-iterator",
-  "string-repeat",
-  "string-slice",
-  "string-substring",
-  "test",
-  "typed-array",
-  "typed-array-createtypedarray",
-  "typed-array-every",
-  "typed-array-filter",
-  "typed-array-find",
-  "typed-array-findindex",
-  "typed-array-foreach",
-  "typed-array-reduce",
-  "typed-array-reduceright",
-  "typed-array-slice",
-  "typed-array-some",
-  "typed-array-subarray",
-]
-
 action("run_torque") {
   visibility = [
     ":*",
@@ -1069,10 +1018,11 @@ action("run_torque") {
     "$target_gen_dir/torque-generated/exported-macros-assembler-tq.h",
     "$target_gen_dir/torque-generated/csa-types-tq.h",
   ]
-  foreach(namespace, torque_namespaces) {
+  foreach(file, torque_files) {
+    filetq = string_replace(file, ".tq", "-tq-csa")
     outputs += [
-      "$target_gen_dir/torque-generated/builtins-$namespace-gen-tq.cc",
-      "$target_gen_dir/torque-generated/builtins-$namespace-gen-tq.h",
+      "$target_gen_dir/torque-generated/$filetq.cc",
+      "$target_gen_dir/torque-generated/$filetq.h",
     ]
   }
 
@@ -1082,11 +1032,10 @@ action("run_torque") {
                        root_build_dir),
     "-o",
     rebase_path("$target_gen_dir/torque-generated", root_build_dir),
+    "-v8-root",
+    rebase_path(".", root_build_dir),
   ]
-
-  foreach(file, torque_files) {
-    args += [ rebase_path(file, root_build_dir) ]
-  }
+  args += torque_files
 }
 
 group("v8_maybe_icu") {
@@ -1114,10 +1063,11 @@ v8_source_set("torque_generated_initializers") {
     "$target_gen_dir/torque-generated/exported-macros-assembler-tq.cc",
     "$target_gen_dir/torque-generated/exported-macros-assembler-tq.h",
   ]
-  foreach(namespace, torque_namespaces) {
+  foreach(file, torque_files) {
+    filetq = string_replace(file, ".tq", "-tq-csa")
     sources += [
-      "$target_gen_dir/torque-generated/builtins-$namespace-gen-tq.cc",
-      "$target_gen_dir/torque-generated/builtins-$namespace-gen-tq.h",
+      "$target_gen_dir/torque-generated/$filetq.cc",
+      "$target_gen_dir/torque-generated/$filetq.h",
     ]
   }
 

src/torque/contextual.h

@@ -14,6 +14,9 @@ namespace v8 {
 namespace internal {
 namespace torque {
 
+template <class Variable>
+V8_EXPORT_PRIVATE typename Variable::VariableType*& ContextualVariableTop();
+
 // {ContextualVariable} provides a clean alternative to a global variable.
 // The contextual variable is mutable, and supports managing the value of
 // a variable in a well-nested fashion via the {Scope} class.
@@ -66,7 +69,9 @@ class ContextualVariable {
   }
 
  private:
-  V8_EXPORT_PRIVATE static VarType*& Top();
+  template <class T>
+  friend typename T::VariableType*& ContextualVariableTop();
+  static VarType*& Top() { return ContextualVariableTop<Derived>(); }
 
   static bool HasScope() { return Top() != nullptr; }
   friend class MessageBuilder;
@@ -77,12 +82,11 @@ class ContextualVariable {
   struct VarName                                  \
       : v8::internal::torque::ContextualVariable<VarName, __VA_ARGS__> {}
 
-#define DEFINE_CONTEXTUAL_VARIABLE(VarName)                   \
-  template <>                                                 \
-  V8_EXPORT_PRIVATE VarName::VariableType*&                   \
-  ContextualVariable<VarName, VarName::VariableType>::Top() { \
-    static thread_local VarName::VariableType* top = nullptr; \
-    return top;                                               \
+#define DEFINE_CONTEXTUAL_VARIABLE(VarName)                                    \
+  template <>                                                                  \
+  V8_EXPORT_PRIVATE VarName::VariableType*& ContextualVariableTop<VarName>() { \
+    static thread_local VarName::VariableType* top = nullptr;                  \
+    return top;                                                                \
   }
 
 // By inheriting from {ContextualClass} a class can become a contextual variable

src/torque/csa-generator.cc

@@ -56,7 +56,7 @@ Stack<std::string> CSAGenerator::EmitBlock(const Block* block) {
 }
 
 void CSAGenerator::EmitSourcePosition(SourcePosition pos, bool always_emit) {
-  const std::string& file = SourceFileMap::GetSource(pos.source);
+  const std::string& file = SourceFileMap::AbsolutePath(pos.source);
   if (always_emit || !previous_position_.CompareStartIgnoreColumn(pos)) {
     // Lines in Torque SourcePositions are zero-based, while the
     // CodeStubAssembler and downwind systems are one-based.
@@ -691,8 +691,8 @@ void CSAGenerator::EmitInstruction(const AbortInstruction& instruction,
       out_ << "    CodeStubAssembler(state_).DebugBreak();\n";
       break;
     case AbortInstruction::Kind::kAssertionFailure: {
-      std::string file =
-          StringLiteralQuote(SourceFileMap::GetSource(instruction.pos.source));
+      std::string file = StringLiteralQuote(
+          SourceFileMap::PathFromV8Root(instruction.pos.source));
       out_ << "    CodeStubAssembler(state_).FailAssert("
            << StringLiteralQuote(instruction.message) << ", " << file << ", "
            << instruction.pos.start.line + 1 << ");\n";

src/torque/declarable.h

@@ -183,15 +183,9 @@ class Namespace : public Scope {
   const std::string& name() const { return name_; }
   bool IsDefaultNamespace() const;
   bool IsTestNamespace() const;
-  std::ostream& source_stream() { return source_stream_; }
-  std::ostream& header_stream() { return header_stream_; }
-  std::string source() { return source_stream_.str(); }
-  std::string header() { return header_stream_.str(); }
 
  private:
   std::string name_;
-  std::stringstream header_stream_;
-  std::stringstream source_stream_;
 };
 
 inline Namespace* CurrentNamespace() {

src/torque/global-context.h

@@ -82,6 +82,14 @@ class GlobalContext : public ContextualClass<GlobalContext> {
   static Ast* ast() { return &Get().ast_; }
   static size_t FreshId() { return Get().fresh_id_++; }
 
+  struct PerFileStreams {
+    std::stringstream csa_headerfile;
+    std::stringstream csa_ccfile;
+  };
+  static PerFileStreams& GeneratedPerFile(SourceId file) {
+    return Get().generated_per_file_[file];
+  }
+
  private:
   bool collect_language_server_data_;
   bool force_assert_statements_;
@@ -89,6 +97,7 @@ class GlobalContext : public ContextualClass<GlobalContext> {
   Ast ast_;
   std::vector<std::unique_ptr<Declarable>> declarables_;
   std::vector<std::string> cpp_includes_;
+  std::map<SourceId, PerFileStreams> generated_per_file_;
   GlobalClassList classes_;
   size_t fresh_id_ = 0;
 

src/torque/implementation-visitor.cc

@@ -48,61 +48,62 @@ const Type* ImplementationVisitor::Visit(Statement* stmt) {
   return result;
 }
 
-void ImplementationVisitor::BeginNamespaceFile(Namespace* nspace) {
-  std::ostream& source = nspace->source_stream();
-  std::ostream& header = nspace->header_stream();
+void ImplementationVisitor::BeginCSAFiles() {
+  for (SourceId file : SourceFileMap::AllSources()) {
+    std::ostream& source = GlobalContext::GeneratedPerFile(file).csa_ccfile;
+    std::ostream& header = GlobalContext::GeneratedPerFile(file).csa_headerfile;
 
-  for (const std::string& include_path : GlobalContext::CppIncludes()) {
-    source << "#include " << StringLiteralQuote(include_path) << "\n";
-  }
+    for (const std::string& include_path : GlobalContext::CppIncludes()) {
+      source << "#include " << StringLiteralQuote(include_path) << "\n";
+    }
 
-  for (Namespace* n : GlobalContext::Get().GetNamespaces()) {
-    source << "#include \"torque-generated/builtins-" +
-                  DashifyString(n->name()) + "-gen-tq.h\"\n";
-  }
-  source << "\n";
+    for (SourceId file : SourceFileMap::AllSources()) {
+      source << "#include \"torque-generated/" +
+                    SourceFileMap::PathFromV8RootWithoutExtension(file) +
+                    "-tq-csa.h\"\n";
+    }
+    source << "\n";
 
-  source << "namespace v8 {\n"
-         << "namespace internal {\n"
-         << "\n";
+    source << "namespace v8 {\n"
+           << "namespace internal {\n"
+           << "\n";
 
-  std::string upper_name(nspace->name());
-  transform(upper_name.begin(), upper_name.end(), upper_name.begin(),
-            ::toupper);
-  std::string headerDefine =
-      "V8_GEN_TORQUE_GENERATED_" + upper_name + "_NAMESPACE_TQ_H_";
-  header << "#ifndef " << headerDefine << "\n";
-  header << "#define " << headerDefine << "\n\n";
-  header << "#include \"src/compiler/code-assembler.h\"\n";
-  header << "#include \"src/codegen/code-stub-assembler.h\"\n";
-  header << "#include \"src/utils/utils.h\"\n";
-  header << "#include \"torque-generated/field-offsets-tq.h\"\n";
-  header << "#include \"torque-generated/csa-types-tq.h\"\n";
-  header << "\n";
+    std::string headerDefine =
+        "V8_GEN_TORQUE_GENERATED_" +
+        UnderlinifyPath(SourceFileMap::PathFromV8Root(file)) + "_H_";
+    header << "#ifndef " << headerDefine << "\n";
+    header << "#define " << headerDefine << "\n\n";
+    header << "#include \"src/compiler/code-assembler.h\"\n";
+    header << "#include \"src/codegen/code-stub-assembler.h\"\n";
+    header << "#include \"src/utils/utils.h\"\n";
+    header << "#include \"torque-generated/field-offsets-tq.h\"\n";
+    header << "#include \"torque-generated/csa-types-tq.h\"\n";
+    header << "\n";
 
-  header << "namespace v8 {\n"
-         << "namespace internal {\n"
-         << "\n";
+    header << "namespace v8 {\n"
+           << "namespace internal {\n"
+           << "\n";
+  }
 }
 
-void ImplementationVisitor::EndNamespaceFile(Namespace* nspace) {
-  std::ostream& source = nspace->source_stream();
-  std::ostream& header = nspace->header_stream();
+void ImplementationVisitor::EndCSAFiles() {
+  for (SourceId file : SourceFileMap::AllSources()) {
+    std::ostream& source = GlobalContext::GeneratedPerFile(file).csa_ccfile;
+    std::ostream& header = GlobalContext::GeneratedPerFile(file).csa_headerfile;
 
-  std::string upper_name(nspace->name());
-  transform(upper_name.begin(), upper_name.end(), upper_name.begin(),
-            ::toupper);
-  std::string headerDefine =
-      "V8_GEN_TORQUE_GENERATED_" + upper_name + "_NAMESPACE_V8_H_";
+    std::string headerDefine =
+        "V8_GEN_TORQUE_GENERATED_" +
+        UnderlinifyPath(SourceFileMap::PathFromV8Root(file)) + "_H_";
 
-  source << "}  // namespace internal\n"
-         << "}  // namespace v8\n"
-         << "\n";
+    source << "}  // namespace internal\n"
+           << "}  // namespace v8\n"
+           << "\n";
 
-  header << "}  // namespace internal\n"
-         << "}  // namespace v8\n"
-         << "\n";
-  header << "#endif  // " << headerDefine << "\n";
+    header << "}  // namespace internal\n"
+           << "}  // namespace v8\n"
+           << "\n";
+    header << "#endif  // " << headerDefine << "\n";
+  }
 }
 
 void ImplementationVisitor::Visit(NamespaceConstant* decl) {
@@ -395,15 +396,15 @@ std::string AddParameter(size_t i, Builtin* builtin,
 void ImplementationVisitor::Visit(Builtin* builtin) {
   if (builtin->IsExternal()) return;
   CurrentScope::Scope current_scope(builtin);
+  CurrentCallable::Scope current_callable(builtin);
+  CurrentReturnValue::Scope current_return_value;
+
   const std::string& name = builtin->ExternalName();
   const Signature& signature = builtin->signature();
   source_out() << "TF_BUILTIN(" << name << ", CodeStubAssembler) {\n"
                << "  compiler::CodeAssemblerState* state_ = state();"
                << "  compiler::CodeAssembler ca_(state());\n";
 
-  CurrentCallable::Scope current_callable(builtin);
-  CurrentReturnValue::Scope current_return_value;
-
   Stack<const Type*> parameter_types;
   Stack<std::string> parameters;
 
@@ -1391,17 +1392,21 @@ VisitResult ImplementationVisitor::Visit(SpreadExpression* expr) {
       "initialization expressions");
 }
 
-void ImplementationVisitor::GenerateImplementation(const std::string& dir,
-                                                   Namespace* nspace) {
-  std::string new_source(nspace->source());
-  std::string base_file_name =
-      "builtins-" + DashifyString(nspace->name()) + "-gen-tq";
+void ImplementationVisitor::GenerateImplementation(const std::string& dir) {
+  for (SourceId file : SourceFileMap::AllSources()) {
+    std::string path_from_root =
+        SourceFileMap::PathFromV8RootWithoutExtension(file);
 
-  std::string source_file_name = dir + "/" + base_file_name + ".cc";
-  WriteFile(source_file_name, new_source);
-  std::string new_header(nspace->header());
-  std::string header_file_name = dir + "/" + base_file_name + ".h";
-  WriteFile(header_file_name, new_header);
+    std::string new_source(
+        GlobalContext::GeneratedPerFile(file).csa_ccfile.str());
+
+    std::string source_file_name = dir + "/" + path_from_root + "-tq-csa.cc";
+    WriteFile(source_file_name, new_source);
+    std::string new_header(
+        GlobalContext::GeneratedPerFile(file).csa_headerfile.str());
+    std::string header_file_name = dir + "/" + path_from_root + "-tq-csa.h";
+    WriteFile(header_file_name, new_header);
+  }
 }
 
 void ImplementationVisitor::GenerateMacroFunctionDeclaration(
@@ -2459,6 +2464,7 @@ std::string ImplementationVisitor::ExternalParameterName(
 DEFINE_CONTEXTUAL_VARIABLE(ImplementationVisitor::ValueBindingsManager)
 DEFINE_CONTEXTUAL_VARIABLE(ImplementationVisitor::LabelBindingsManager)
 DEFINE_CONTEXTUAL_VARIABLE(ImplementationVisitor::CurrentCallable)
+DEFINE_CONTEXTUAL_VARIABLE(ImplementationVisitor::CurrentFileStreams)
 DEFINE_CONTEXTUAL_VARIABLE(ImplementationVisitor::CurrentReturnValue)
 
 bool IsCompatibleSignature(const Signature& sig, const TypeVector& types,
@@ -2519,6 +2525,12 @@ void ImplementationVisitor::VisitAllDeclarables() {
 void ImplementationVisitor::Visit(Declarable* declarable) {
   CurrentScope::Scope current_scope(declarable->ParentScope());
   CurrentSourcePosition::Scope current_source_position(declarable->Position());
+  CurrentFileStreams::Scope current_file_streams(
+      &GlobalContext::GeneratedPerFile(declarable->Position().source));
+  if (Callable* callable = Callable::DynamicCast(declarable)) {
+    if (!callable->ShouldGenerateExternalCode())
+      CurrentFileStreams::Get() = nullptr;
+  }
   switch (declarable->kind()) {
     case Declarable::kExternMacro:
       return Visit(ExternMacro::cast(declarable));
@@ -3409,9 +3421,11 @@ void ImplementationVisitor::GenerateExportedMacrosAssembler(
     h_contents << "#include \"src/execution/frames.h\"\n";
     h_contents << "#include \"torque-generated/csa-types-tq.h\"\n";
     cc_contents << "#include \"torque-generated/" << file_name << ".h\"\n";
-    for (Namespace* n : GlobalContext::Get().GetNamespaces()) {
-      cc_contents << "#include \"torque-generated/builtins-" +
-                         DashifyString(n->name()) + "-gen-tq.h\"\n";
+
+    for (SourceId file : SourceFileMap::AllSources()) {
+      cc_contents << "#include \"torque-generated/" +
+                         SourceFileMap::PathFromV8RootWithoutExtension(file) +
+                         "-tq-csa.h\"\n";
     }
 
     NamespaceScope h_namespaces(h_contents, {"v8", "internal"});

src/torque/implementation-visitor.h

@@ -370,16 +370,18 @@ class ImplementationVisitor {
   const Type* Visit(DebugStatement* stmt);
   const Type* Visit(AssertStatement* stmt);
 
-  void BeginNamespaceFile(Namespace* nspace);
-  void EndNamespaceFile(Namespace* nspace);
+  void BeginCSAFiles();
+  void EndCSAFiles();
 
-  void GenerateImplementation(const std::string& dir, Namespace* nspace);
+  void GenerateImplementation(const std::string& dir);
 
   DECLARE_CONTEXTUAL_VARIABLE(ValueBindingsManager,
                               BindingsManager<LocalValue>);
   DECLARE_CONTEXTUAL_VARIABLE(LabelBindingsManager,
                               BindingsManager<LocalLabel>);
   DECLARE_CONTEXTUAL_VARIABLE(CurrentCallable, Callable*);
+  DECLARE_CONTEXTUAL_VARIABLE(CurrentFileStreams,
+                              GlobalContext::PerFileStreams*);
   DECLARE_CONTEXTUAL_VARIABLE(CurrentReturnValue, base::Optional<VisitResult>);
 
   // A BindingsManagersScope has to be active for local bindings to be created.
@@ -566,20 +568,16 @@ class ImplementationVisitor {
   std::string ExternalParameterName(const std::string& name);
 
   std::ostream& source_out() {
-    Callable* callable = CurrentCallable::Get();
-    if (!callable || callable->ShouldGenerateExternalCode()) {
-      return CurrentNamespace()->source_stream();
-    } else {
-      return null_stream_;
+    if (auto* streams = CurrentFileStreams::Get()) {
+      return streams->csa_ccfile;
     }
+    return null_stream_;
   }
   std::ostream& header_out() {
-    Callable* callable = CurrentCallable::Get();
-    if (!callable || callable->ShouldGenerateExternalCode()) {
-      return CurrentNamespace()->header_stream();
-    } else {
-      return null_stream_;
+    if (auto* streams = CurrentFileStreams::Get()) {
+      return streams->csa_headerfile;
     }
+    return null_stream_;
   }
   CfgAssembler& assembler() { return *assembler_; }
 

src/torque/ls/json-parser.cc

@@ -183,7 +183,7 @@ class JsonGrammar : public Grammar {
 JsonParserResult ParseJson(const std::string& input) {
   // Torque needs a CurrentSourceFile scope during parsing.
   // As JSON lives in memory only, a unknown file scope is created.
-  SourceFileMap::Scope source_map_scope;
+  SourceFileMap::Scope source_map_scope("");
   TorqueMessages::Scope messages_scope;
   CurrentSourceFile::Scope unkown_file(SourceFileMap::AddSource("<json>"));
 

src/torque/ls/message-handler.cc

@@ -69,7 +69,7 @@ void ResetCompilationErrorDiagnostics(MessageWriter writer) {
     PublishDiagnosticsNotification notification;
     notification.set_method("textDocument/publishDiagnostics");
 
-    std::string error_file = SourceFileMap::GetSource(source);
+    std::string error_file = SourceFileMap::AbsolutePath(source);
     notification.params().set_uri(error_file);
     // Trigger empty array creation.
     USE(notification.params().diagnostics_size());
@@ -115,7 +115,7 @@ class DiagnosticCollector {
     notification.set_method("textDocument/publishDiagnostics");
 
     std::string file =
-        id.IsValid() ? SourceFileMap::GetSource(id) : "<unknown>";
+        id.IsValid() ? SourceFileMap::AbsolutePath(id) : "<unknown>";
     notification.params().set_uri(file);
     return notification;
   }
@@ -164,7 +164,7 @@ void SendCompilationDiagnostics(const TorqueCompilerResult& result,
 
 void CompilationFinished(TorqueCompilerResult result, MessageWriter writer) {
   LanguageServerData::Get() = std::move(result.language_server_data);
-  SourceFileMap::Get() = result.source_file_map;
+  SourceFileMap::Get() = *result.source_file_map;
 
   SendCompilationDiagnostics(result, writer);
 }

src/torque/ls/message.h

@@ -241,7 +241,7 @@ class Location : public NestedJsonAccessor {
   JSON_OBJECT_ACCESSORS(Range, range)
 
   void SetTo(SourcePosition position) {
-    set_uri(SourceFileMap::GetSource(position.source));
+    set_uri(SourceFileMap::AbsolutePath(position.source));
     range().start().set_line(position.start.line);
     range().start().set_character(position.start.column);
     range().end().set_line(position.end.line);

src/torque/ls/torque-language-server.cc

@@ -21,7 +21,7 @@ int WrappedMain(int argc, const char** argv) {
   Logger::Scope log_scope;
   TorqueFileList::Scope files_scope;
   LanguageServerData::Scope server_data_scope;
-  SourceFileMap::Scope source_file_map_scope;
+  SourceFileMap::Scope source_file_map_scope("");
   DiagnosticsFiles::Scope diagnostics_files_scope;
 
   for (int i = 1; i < argc; ++i) {

src/torque/source-positions.cc

@@ -4,6 +4,8 @@
 
 #include "src/torque/source-positions.h"
 
+#include "src/torque/utils.h"
+
 namespace v8 {
 namespace internal {
 namespace torque {
@@ -12,6 +14,55 @@ DEFINE_CONTEXTUAL_VARIABLE(CurrentSourceFile)
 DEFINE_CONTEXTUAL_VARIABLE(CurrentSourcePosition)
 DEFINE_CONTEXTUAL_VARIABLE(SourceFileMap)
 
+// static
+const std::string& SourceFileMap::PathFromV8Root(SourceId file) {
+  CHECK(file.IsValid());
+  return Get().sources_[file.id_];
+}
+
+// static
+std::string SourceFileMap::AbsolutePath(SourceId file) {
+  const std::string& root_path = PathFromV8Root(file);
+  if (StringStartsWith(root_path, "file://")) return root_path;
+  return Get().v8_root_ + "/" + PathFromV8Root(file);
+}
+
+// static
+std::string SourceFileMap::PathFromV8RootWithoutExtension(SourceId file) {
+  std::string path_from_root = PathFromV8Root(file);
+  if (!StringEndsWith(path_from_root, ".tq")) {
+    Error("Not a .tq file: ", path_from_root).Throw();
+  }
+  path_from_root.resize(path_from_root.size() - strlen(".tq"));
+  return path_from_root;
+}
+
+// static
+SourceId SourceFileMap::AddSource(std::string path) {
+  Get().sources_.push_back(std::move(path));
+  return SourceId(static_cast<int>(Get().sources_.size()) - 1);
+}
+
+// static
+SourceId SourceFileMap::GetSourceId(const std::string& path) {
+  for (size_t i = 0; i < Get().sources_.size(); ++i) {
+    if (Get().sources_[i] == path) {
+      return SourceId(static_cast<int>(i));
+    }
+  }
+  return SourceId::Invalid();
+}
+
+// static
+std::vector<SourceId> SourceFileMap::AllSources() {
+  SourceFileMap& self = Get();
+  std::vector<SourceId> result;
+  for (int i = 0; i < static_cast<int>(self.sources_.size()); ++i) {
+    result.push_back(SourceId(i));
+  }
+  return result;
+}
+
 }  // namespace torque
 }  // namespace internal
 }  // namespace v8

src/torque/source-positions.h

@@ -71,34 +71,23 @@ struct SourcePosition {
 DECLARE_CONTEXTUAL_VARIABLE(CurrentSourceFile, SourceId);
 DECLARE_CONTEXTUAL_VARIABLE(CurrentSourcePosition, SourcePosition);
 
-class SourceFileMap : public ContextualClass<SourceFileMap> {
+class V8_EXPORT_PRIVATE SourceFileMap : public ContextualClass<SourceFileMap> {
  public:
-  SourceFileMap() = default;
-  static const std::string& GetSource(SourceId source) {
-    CHECK(source.IsValid());
-    return Get().sources_[source.id_];
-  }
-
-  static SourceId AddSource(std::string path) {
-    Get().sources_.push_back(std::move(path));
-    return SourceId(static_cast<int>(Get().sources_.size()) - 1);
-  }
-
-  static SourceId GetSourceId(const std::string& path) {
-    for (size_t i = 0; i < Get().sources_.size(); ++i) {
-      if (Get().sources_[i] == path) {
-        return SourceId(static_cast<int>(i));
-      }
-    }
-    return SourceId::Invalid();
-  }
+  explicit SourceFileMap(std::string v8_root) : v8_root_(std::move(v8_root)) {}
+  static const std::string& PathFromV8Root(SourceId file);
+  static std::string PathFromV8RootWithoutExtension(SourceId file);
+  static std::string AbsolutePath(SourceId file);
+  static SourceId AddSource(std::string path);
+  static SourceId GetSourceId(const std::string& path);
+  static std::vector<SourceId> AllSources();
 
  private:
   std::vector<std::string> sources_;
+  std::string v8_root_;
 };
 
 inline std::string PositionAsString(SourcePosition pos) {
-  return SourceFileMap::GetSource(pos.source) + ":" +
+  return SourceFileMap::PathFromV8Root(pos.source) + ":" +
          std::to_string(pos.start.line + 1) + ":" +
          std::to_string(pos.start.column + 1);
 }

src/torque/torque-compiler.cc

@@ -31,7 +31,7 @@ void ReadAndParseTorqueFile(const std::string& path) {
   CurrentSourceFile::Scope source_id_scope(source_id);
 
   // path might be either a normal file path or an encoded URI.
-  auto maybe_content = ReadFile(path);
+  auto maybe_content = ReadFile(SourceFileMap::AbsolutePath(source_id));
   if (!maybe_content) {
     if (auto maybe_path = FileUriDecode(path)) {
       maybe_content = ReadFile(*maybe_path);
@@ -72,9 +72,7 @@ void CompileCurrentAst(TorqueCompilerOptions options) {
   ImplementationVisitor implementation_visitor;
   implementation_visitor.SetDryRun(output_directory.length() == 0);
 
-  for (Namespace* n : GlobalContext::Get().GetNamespaces()) {
-    implementation_visitor.BeginNamespaceFile(n);
-  }
+  implementation_visitor.BeginCSAFiles();
 
   implementation_visitor.VisitAllDeclarables();
 
@@ -86,10 +84,8 @@ void CompileCurrentAst(TorqueCompilerOptions options) {
   implementation_visitor.GenerateExportedMacrosAssembler(output_directory);
   implementation_visitor.GenerateCSATypes(output_directory);
 
-  for (Namespace* n : GlobalContext::Get().GetNamespaces()) {
-    implementation_visitor.EndNamespaceFile(n);
-    implementation_visitor.GenerateImplementation(output_directory, n);
-  }
+  implementation_visitor.EndCSAFiles();
+  implementation_visitor.GenerateImplementation(output_directory);
 
   if (GlobalContext::collect_language_server_data()) {
     LanguageServerData::SetGlobalContext(std::move(GlobalContext::Get()));
@@ -101,8 +97,9 @@ void CompileCurrentAst(TorqueCompilerOptions options) {
 
 TorqueCompilerResult CompileTorque(const std::string& source,
                                    TorqueCompilerOptions options) {
-  SourceFileMap::Scope source_map_scope;
-  CurrentSourceFile::Scope no_file_scope(SourceFileMap::AddSource("<torque>"));
+  SourceFileMap::Scope source_map_scope("");
+  CurrentSourceFile::Scope no_file_scope(
+      SourceFileMap::AddSource("dummy-filename.tq"));
   CurrentAst::Scope ast_scope;
   TorqueMessages::Scope messages_scope;
   LanguageServerData::Scope server_data_scope;
@@ -125,7 +122,7 @@ TorqueCompilerResult CompileTorque(const std::string& source,
 
 TorqueCompilerResult CompileTorque(std::vector<std::string> files,
                                    TorqueCompilerOptions options) {
-  SourceFileMap::Scope source_map_scope;
+  SourceFileMap::Scope source_map_scope(options.v8_root);
   CurrentSourceFile::Scope unknown_source_file_scope(SourceId::Invalid());
   CurrentAst::Scope ast_scope;
   TorqueMessages::Scope messages_scope;
@@ -133,7 +130,9 @@ TorqueCompilerResult CompileTorque(std::vector<std::string> files,
 
   TorqueCompilerResult result;
   try {
-    for (const auto& path : files) ReadAndParseTorqueFile(path);
+    for (const auto& path : files) {
+      ReadAndParseTorqueFile(path);
+    }
     CompileCurrentAst(options);
   } catch (TorqueAbortCompilation&) {
     // Do nothing. The relevant TorqueMessage is part of the

src/torque/torque-compiler.h

@@ -17,6 +17,7 @@ namespace torque {
 
 struct TorqueCompilerOptions {
   std::string output_directory = "";
+  std::string v8_root = "";
   bool collect_language_server_data = false;
 
   // assert(...) are only generated for debug builds. The provide
@@ -29,7 +30,7 @@ struct TorqueCompilerResult {
   // Map translating SourceIds to filenames. This field is
   // set on errors, so the SourcePosition of the error can be
   // resolved.
-  SourceFileMap source_file_map;
+  base::Optional<SourceFileMap> source_file_map;
 
   // Eagerly collected data needed for the LanguageServer.
   // Set the corresponding options flag to enable.

src/torque/torque.cc

@@ -20,21 +20,29 @@ std::string ErrorPrefixFor(TorqueMessage::Kind kind) {
 
 int WrappedMain(int argc, const char** argv) {
   std::string output_directory;
+  std::string v8_root;
   std::vector<std::string> files;
 
   for (int i = 1; i < argc; ++i) {
     // Check for options
-    if (!strcmp("-o", argv[i])) {
+    if (std::string(argv[i]) == "-o") {
       output_directory = argv[++i];
-      continue;
+    } else if (std::string(argv[i]) == "-v8-root") {
+      v8_root = std::string(argv[++i]);
+    } else {
+      // Otherwise it's a .tq file. Remember it for compilation.
+      files.emplace_back(argv[i]);
+      if (!StringEndsWith(files.back(), ".tq")) {
+        std::cerr << "Unexpected command-line argument \"" << files.back()
+                  << "\", expected a .tq file.\n";
+        base::OS::Abort();
+      }
     }
-
-    // Otherwise it's a .tq file. Remember it for compilation.
-    files.emplace_back(argv[i]);
   }
 
   TorqueCompilerOptions options;
-  options.output_directory = output_directory;
+  options.output_directory = std::move(output_directory);
+  options.v8_root = std::move(v8_root);
   options.collect_language_server_data = false;
   options.force_assert_statements = false;
 
@@ -42,7 +50,7 @@ int WrappedMain(int argc, const char** argv) {
 
   // PositionAsString requires the SourceFileMap to be set to
   // resolve the file name. Needed to report errors and lint warnings.
-  SourceFileMap::Scope source_file_map_scope(result.source_file_map);
+  SourceFileMap::Scope source_file_map_scope(*result.source_file_map);
 
   for (const TorqueMessage& message : result.messages) {
     if (message.position) {

src/torque/utils.cc

@@ -248,6 +248,15 @@ std::string DashifyString(const std::string& underscore_string) {
   return result;
 }
 
+std::string UnderlinifyPath(std::string path) {
+  std::replace(path.begin(), path.end(), '-', '_');
+  std::replace(path.begin(), path.end(), '/', '_');
+  std::replace(path.begin(), path.end(), '\\', '_');
+  std::replace(path.begin(), path.end(), '.', '_');
+  transform(path.begin(), path.end(), path.begin(), ::toupper);
+  return path;
+}
+
 void ReplaceFileContentsIfDifferent(const std::string& file_path,
                                     const std::string& contents) {
   std::ifstream old_contents_stream(file_path.c_str());

src/torque/utils.h

@@ -95,6 +95,7 @@ template <class... Args>
 std::string CapifyStringWithUnderscores(const std::string& camellified_string);
 std::string CamelifyString(const std::string& underscore_string);
 std::string DashifyString(const std::string& underscore_string);
+std::string UnderlinifyPath(std::string path);
 
 void ReplaceFileContentsIfDifferent(const std::string& file_path,
                                     const std::string& contents);
@@ -345,6 +346,15 @@ class NullOStream : public std::ostream {
   NullStreambuf buffer_;
 };
 
+inline bool StringStartsWith(const std::string& s, const std::string& prefix) {
+  if (s.size() < prefix.size()) return false;
+  return s.substr(0, prefix.size()) == prefix;
+}
+inline bool StringEndsWith(const std::string& s, const std::string& suffix) {
+  if (s.size() < suffix.size()) return false;
+  return s.substr(s.size() - suffix.size()) == suffix;
+}
+
 }  // namespace torque
 }  // namespace internal
 }  // namespace v8

test/cctest/torque/test-torque.cc

@@ -19,7 +19,6 @@
 #include "src/strings/char-predicates.h"
 #include "test/cctest/compiler/code-assembler-tester.h"
 #include "test/cctest/compiler/function-tester.h"
-#include "torque-generated/builtins-test-gen-tq.h"
 
 namespace v8 {
 namespace internal {

test/unittests/torque/earley-parser-unittest.cc

@@ -68,7 +68,7 @@ struct SimpleArithmeticGrammar : Grammar {
 
 TEST(EarleyParser, SimpleArithmetic) {
   SimpleArithmeticGrammar grammar;
-  SourceFileMap::Scope source_file_map;
+  SourceFileMap::Scope source_file_map("");
   CurrentSourceFile::Scope current_source_file{
       SourceFileMap::AddSource("dummy_filename")};
   std::string result1 =

test/unittests/torque/ls-message-unittest.cc

@@ -61,7 +61,7 @@ TEST(LanguageServerMessage,
 }
 
 TEST(LanguageServerMessage, GotoDefinitionUnkownFile) {
-  SourceFileMap::Scope source_file_map_scope;
+  SourceFileMap::Scope source_file_map_scope("");
 
   GotoDefinitionRequest request;
   request.set_id(42);
@@ -80,7 +80,7 @@ TEST(LanguageServerMessage, GotoDefinitionUnkownFile) {
 }
 
 TEST(LanguageServerMessage, GotoDefinition) {
-  SourceFileMap::Scope source_file_map_scope;
+  SourceFileMap::Scope source_file_map_scope("");
   SourceId test_id = SourceFileMap::AddSource("file://test.tq");
   SourceId definition_id = SourceFileMap::AddSource("file://base.tq");
 
@@ -136,7 +136,7 @@ TEST(LanguageServerMessage, CompilationErrorSendsDiagnostics) {
   DiagnosticsFiles::Scope diagnostic_files_scope;
   LanguageServerData::Scope server_data_scope;
   TorqueMessages::Scope messages_scope;
-  SourceFileMap::Scope source_file_map_scope;
+  SourceFileMap::Scope source_file_map_scope("");
 
   TorqueCompilerResult result;
   { Error("compilation failed somehow"); }
@@ -165,8 +165,8 @@ TEST(LanguageServerMessage, LintErrorSendsDiagnostics) {
   DiagnosticsFiles::Scope diagnostic_files_scope;
   TorqueMessages::Scope messages_scope;
   LanguageServerData::Scope server_data_scope;
-  SourceFileMap::Scope sourc_file_map_scope;
-  SourceId test_id = SourceFileMap::AddSource("test.tq");
+  SourceFileMap::Scope sourc_file_map_scope("");
+  SourceId test_id = SourceFileMap::AddSource("file://test.tq");
 
   // No compilation errors but two lint warnings.
   {
@@ -186,7 +186,7 @@ TEST(LanguageServerMessage, LintErrorSendsDiagnostics) {
 
     EXPECT_EQ(notification.method(), "textDocument/publishDiagnostics");
     ASSERT_FALSE(notification.IsNull("params"));
-    EXPECT_EQ(notification.params().uri(), "test.tq");
+    EXPECT_EQ(notification.params().uri(), "file://test.tq");
 
     ASSERT_EQ(notification.params().diagnostics_size(), static_cast<size_t>(2));
     Diagnostic diagnostic1 = notification.params().diagnostics(0);
@@ -204,7 +204,7 @@ TEST(LanguageServerMessage, LintErrorSendsDiagnostics) {
 
 TEST(LanguageServerMessage, CleanCompileSendsNoDiagnostics) {
   LanguageServerData::Scope server_data_scope;
-  SourceFileMap::Scope sourc_file_map_scope;
+  SourceFileMap::Scope sourc_file_map_scope("");
 
   TorqueCompilerResult result;
   result.source_file_map = SourceFileMap::Get();
@@ -216,12 +216,12 @@ TEST(LanguageServerMessage, CleanCompileSendsNoDiagnostics) {
 
 TEST(LanguageServerMessage, NoSymbolsSendsEmptyResponse) {
   LanguageServerData::Scope server_data_scope;
-  SourceFileMap::Scope sourc_file_map_scope;
+  SourceFileMap::Scope sourc_file_map_scope("");
 
   DocumentSymbolRequest request;
   request.set_id(42);
   request.set_method("textDocument/documentSymbol");
-  request.params().textDocument().set_uri("test.tq");
+  request.params().textDocument().set_uri("file://test.tq");
 
   bool writer_called = false;
   HandleMessage(request.GetJsonValue(), [&](JsonValue& raw_response) {

test/unittests/torque/ls-server-data-unittest.cc

@@ -13,7 +13,7 @@ namespace torque {
 namespace {
 
 struct TestCompiler {
-  SourceFileMap::Scope file_map_scope;
+  SourceFileMap::Scope file_map_scope{""};
   LanguageServerData::Scope server_data_scope;
 
   void Compile(const std::string& source) {
@@ -23,7 +23,7 @@ struct TestCompiler {
     options.force_assert_statements = true;
 
     TorqueCompilerResult result = CompileTorque(source, options);
-    SourceFileMap::Get() = result.source_file_map;
+    SourceFileMap::Get() = *result.source_file_map;
     LanguageServerData::Get() = std::move(result.language_server_data);
   }
 };
@@ -42,7 +42,7 @@ TEST(LanguageServer, GotoTypeDefinition) {
   compiler.Compile(source);
 
   // Find the definition for type 'T1' of argument 'a' on line 4.
-  const SourceId id = SourceFileMap::GetSourceId("<torque>");
+  const SourceId id = SourceFileMap::GetSourceId("dummy-filename.tq");
   auto maybe_position = LanguageServerData::FindDefinition(id, {4, 19});
   ASSERT_TRUE(maybe_position.has_value());
   EXPECT_EQ(*maybe_position, (SourcePosition{id, {2, 5}, {2, 7}}));
@@ -64,7 +64,7 @@ TEST(LanguageServer, GotoTypeDefinitionExtends) {
   compiler.Compile(source);
 
   // Find the definition for 'T1' of the extends clause on line 3.
-  const SourceId id = SourceFileMap::GetSourceId("<torque>");
+  const SourceId id = SourceFileMap::GetSourceId("dummy-filename.tq");
   auto maybe_position = LanguageServerData::FindDefinition(id, {3, 16});
   ASSERT_TRUE(maybe_position.has_value());
   EXPECT_EQ(*maybe_position, (SourcePosition{id, {2, 5}, {2, 7}}));
@@ -72,7 +72,7 @@ TEST(LanguageServer, GotoTypeDefinitionExtends) {
 
 TEST(LanguageServer, GotoTypeDefinitionNoDataForFile) {
   LanguageServerData::Scope server_data_scope;
-  SourceFileMap::Scope file_scope;
+  SourceFileMap::Scope file_scope("");
   SourceId test_id = SourceFileMap::AddSource("test.tq");
 
   // Regression test, this step should not crash.
@@ -94,7 +94,7 @@ TEST(LanguageServer, GotoLabelDefinitionInSignature) {
   compiler.Compile(source);
 
   // Find the definition for 'Bailout' of the otherwise clause on line 6.
-  const SourceId id = SourceFileMap::GetSourceId("<torque>");
+  const SourceId id = SourceFileMap::GetSourceId("dummy-filename.tq");
   auto maybe_position = LanguageServerData::FindDefinition(id, {6, 18});
   ASSERT_TRUE(maybe_position.has_value());
   EXPECT_EQ(*maybe_position, (SourcePosition{id, {5, 19}, {5, 26}}));
@@ -116,7 +116,7 @@ TEST(LanguageServer, GotoLabelDefinitionInTryBlock) {
   compiler.Compile(source);
 
   // Find the definition for 'Bailout' of the otherwise clause on line 6.
-  const SourceId id = SourceFileMap::GetSourceId("<torque>");
+  const SourceId id = SourceFileMap::GetSourceId("dummy-filename.tq");
   auto maybe_position = LanguageServerData::FindDefinition(id, {6, 25});
   ASSERT_TRUE(maybe_position.has_value());
   EXPECT_EQ(*maybe_position, (SourcePosition{id, {7, 8}, {7, 15}}));
@@ -133,7 +133,7 @@ TEST(LanguageServer, GotoDefinitionClassSuperType) {
   compiler.Compile(source);
 
   // Find the definition for 'Tagged' of the 'extends' on line 3.
-  const SourceId id = SourceFileMap::GetSourceId("<torque>");
+  const SourceId id = SourceFileMap::GetSourceId("dummy-filename.tq");
   auto maybe_position = LanguageServerData::FindDefinition(id, {3, 33});
   ASSERT_TRUE(maybe_position.has_value());
   EXPECT_EQ(*maybe_position, (SourcePosition{id, {2, 5}, {2, 11}}));
@@ -151,7 +151,7 @@ TEST(LanguageServer, GotoLabelDefinitionInSignatureGotoStmt) {
   compiler.Compile(source);
 
   // Find the definition for 'Fail' of the goto statement on line 3.
-  const SourceId id = SourceFileMap::GetSourceId("<torque>");
+  const SourceId id = SourceFileMap::GetSourceId("dummy-filename.tq");
   auto maybe_position = LanguageServerData::FindDefinition(id, {3, 7});
   ASSERT_TRUE(maybe_position.has_value());
   EXPECT_EQ(*maybe_position, (SourcePosition{id, {2, 26}, {2, 30}}));
@@ -170,7 +170,7 @@ TEST(LanguageServer, GotoLabelDefinitionInTryBlockGoto) {
   compiler.Compile(source);
 
   // Find the definition for 'Bailout' of the goto statement on line 3.
-  const SourceId id = SourceFileMap::GetSourceId("<torque>");
+  const SourceId id = SourceFileMap::GetSourceId("dummy-filename.tq");
   auto maybe_position = LanguageServerData::FindDefinition(id, {3, 13});
   ASSERT_TRUE(maybe_position.has_value());
   EXPECT_EQ(*maybe_position, (SourcePosition{id, {4, 8}, {4, 15}}));
@@ -192,7 +192,7 @@ TEST(LanguageServer, GotoLabelDefinitionGotoInOtherwise) {
   compiler.Compile(source);
 
   // Find the definition for 'Bailout' of the otherwise clause on line 6.
-  const SourceId id = SourceFileMap::GetSourceId("<torque>");
+  const SourceId id = SourceFileMap::GetSourceId("dummy-filename.tq");
   auto maybe_position = LanguageServerData::FindDefinition(id, {6, 30});
   ASSERT_TRUE(maybe_position.has_value());
   EXPECT_EQ(*maybe_position, (SourcePosition{id, {7, 8}, {7, 15}}));
@@ -214,7 +214,7 @@ TEST(LanguageServer, SymbolsArePopulated) {
   TestCompiler compiler;
   compiler.Compile(source);
 
-  const SourceId id = SourceFileMap::GetSourceId("<torque>");
+  const SourceId id = SourceFileMap::GetSourceId("dummy-filename.tq");
   const auto& symbols = LanguageServerData::SymbolsForSourceId(id);
   ASSERT_FALSE(symbols.empty());
 }