[Builtins] Infrastructure for source positions in stubs/builtins

Now, the CodeAssembler can annotate Nodes with SourcePositions.
SourcePositions themselves get a new mode "external," in which
they get a file_id, line and column. The file_id is currently
maintained in the isolate, mapping to strings for filenames.

Additionally, inlining information is ignored at this point,
but in the long run I'd like to recognize calls to different
CSA functions as manual inlinings.

At this point, if you want to see the results in tools like GDB,
you'll need to build without clang, and use the GCC toolchain.
GN flag is_clang=false will do the trick.

Bug: v8:8418
Change-Id: I123cdc041612285fa7d0ba532a625bceeda5d338
Reviewed-on: https://chromium-review.googlesource.com/c/1322954
Commit-Queue: Michael Stanton <mvstanton@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59009}

src/builtins/builtins-utils-gen.h

@@ -26,27 +26,27 @@ class CodeAssemblerState;
 //
 // In the body of the builtin function the arguments can be accessed
 // as "Parameter(n)".
-#define TF_BUILTIN(Name, AssemblerBase)                                  \
-  class Name##Assembler : public AssemblerBase {                         \
-   public:                                                               \
-    typedef Builtin_##Name##_InterfaceDescriptor Descriptor;             \
-                                                                         \
-    explicit Name##Assembler(compiler::CodeAssemblerState* state)        \
-        : AssemblerBase(state) {}                                        \
-    void Generate##Name##Impl();                                         \
-                                                                         \
-    Node* Parameter(Descriptor::ParameterIndices index) {                \
-      return CodeAssembler::Parameter(static_cast<int>(index));          \
-    }                                                                    \
-  };                                                                     \
-  void Builtins::Generate_##Name(compiler::CodeAssemblerState* state) {  \
-    Name##Assembler assembler(state);                                    \
-    state->SetInitialDebugInformation(#Name, __FILE__, __LINE__);        \
-    if (Builtins::KindOf(Builtins::k##Name) == Builtins::TFJ) {          \
-      assembler.PerformStackCheck(assembler.GetJSContextParameter());    \
-    }                                                                    \
-    assembler.Generate##Name##Impl();                                    \
-  }                                                                      \
+#define TF_BUILTIN(Name, AssemblerBase)                                 \
+  class Name##Assembler : public AssemblerBase {                        \
+   public:                                                              \
+    typedef Builtin_##Name##_InterfaceDescriptor Descriptor;            \
+                                                                        \
+    explicit Name##Assembler(compiler::CodeAssemblerState* state)       \
+        : AssemblerBase(state) {}                                       \
+    void Generate##Name##Impl();                                        \
+                                                                        \
+    Node* Parameter(Descriptor::ParameterIndices index) {               \
+      return CodeAssembler::Parameter(static_cast<int>(index));         \
+    }                                                                   \
+  };                                                                    \
+  void Builtins::Generate_##Name(compiler::CodeAssemblerState* state) { \
+    Name##Assembler assembler(state);                                   \
+    state->SetInitialDebugInformation(#Name, __FILE__, __LINE__);       \
+    if (Builtins::KindOf(Builtins::k##Name) == Builtins::TFJ) {         \
+      assembler.PerformStackCheck(assembler.GetJSContextParameter());   \
+    }                                                                   \
+    assembler.Generate##Name##Impl();                                   \
+  }                                                                     \
   void Name##Assembler::Generate##Name##Impl()
 
 }  // namespace internal

src/compiler/bytecode-graph-builder.cc

@@ -747,11 +747,11 @@ class BytecodeGraphBuilder::OsrIteratorState {
  private:
   struct IteratorsStates {
     int exception_handler_index_;
-    SourcePositionTableIterator::IndexAndPosition source_iterator_state_;
+    SourcePositionTableIterator::IndexAndPositionState source_iterator_state_;
 
-    IteratorsStates(
-        int exception_handler_index,
-        SourcePositionTableIterator::IndexAndPosition source_iterator_state)
+    IteratorsStates(int exception_handler_index,
+                    SourcePositionTableIterator::IndexAndPositionState
+                        source_iterator_state)
         : exception_handler_index_(exception_handler_index),
           source_iterator_state_(source_iterator_state) {}
   };

src/compiler/code-assembler.cc

@@ -105,6 +105,7 @@ void CodeAssemblerState::SetInitialDebugInformation(const char* msg,
                                                     int line) {
 #if DEBUG
   AssemblerDebugInfo debug_info = {msg, file, line};
+  raw_assembler_->SetSourcePosition(file, line);
   raw_assembler_->SetInitialDebugInformation(debug_info);
 #endif  // DEBUG
 }
@@ -176,11 +177,11 @@ Handle<Code> CodeAssembler::GenerateCode(CodeAssemblerState* state,
   Handle<Code> code;
   Graph* graph = rasm->ExportForOptimization();
 
-  code =
-      Pipeline::GenerateCodeForCodeStub(
-          rasm->isolate(), rasm->call_descriptor(), graph, state->kind_,
-          state->name_, state->builtin_index_, rasm->poisoning_level(), options)
-          .ToHandleChecked();
+  code = Pipeline::GenerateCodeForCodeStub(
+             rasm->isolate(), rasm->call_descriptor(), graph,
+             rasm->source_positions(), state->kind_, state->name_,
+             state->builtin_index_, rasm->poisoning_level(), options)
+             .ToHandleChecked();
 
   state->code_generated_ = true;
   return code;
@@ -424,6 +425,10 @@ void CodeAssembler::Comment(std::string str) {
   raw_assembler()->Comment(str);
 }
 
+void CodeAssembler::SetSourcePosition(const char* file, int line) {
+  raw_assembler()->SetSourcePosition(file, line);
+}
+
 void CodeAssembler::Bind(Label* label) { return label->Bind(); }
 
 #if DEBUG
@@ -1733,6 +1738,7 @@ void CodeAssemblerLabel::Bind(AssemblerDebugInfo debug_info) {
         << "\n#    previous: " << *label_->block();
     FATAL("%s", str.str().c_str());
   }
+  state_->raw_assembler_->SetSourcePosition(debug_info.file, debug_info.line);
   state_->raw_assembler_->Bind(label_, debug_info);
   UpdateVariablesAfterBind();
 }

src/compiler/code-assembler.h

@@ -26,6 +26,7 @@
 #include "src/objects/maybe-object.h"
 #include "src/objects/oddball.h"
 #include "src/runtime/runtime.h"
+#include "src/source-position.h"
 #include "src/zone/zone-containers.h"
 
 namespace v8 {
@@ -348,6 +349,7 @@ class CodeAssemblerState;
 class Node;
 class RawMachineAssembler;
 class RawMachineLabel;
+class SourcePositionTable;
 
 typedef ZoneVector<CodeAssemblerVariable*> CodeAssemblerVariableList;
 
@@ -845,6 +847,8 @@ class V8_EXPORT_PRIVATE CodeAssembler {
     Comment(s.str());
   }
 
+  void SetSourcePosition(const char* file, int line);
+
   void Bind(Label* label);
 #if DEBUG
   void Bind(Label* label, AssemblerDebugInfo debug_info);
@@ -1663,6 +1667,7 @@ class V8_EXPORT_PRIVATE CodeAssemblerState {
   std::vector<CodeAssemblerExceptionHandlerLabel*> exception_handler_labels_;
   typedef uint32_t VariableId;
   VariableId next_variable_id_ = 0;
+
   VariableId NextVariableId() { return next_variable_id_++; }
 
   DISALLOW_COPY_AND_ASSIGN(CodeAssemblerState);

src/compiler/pipeline.cc

@@ -2068,7 +2068,8 @@ bool PipelineImpl::OptimizeGraph(Linkage* linkage) {
 
 MaybeHandle<Code> Pipeline::GenerateCodeForCodeStub(
     Isolate* isolate, CallDescriptor* call_descriptor, Graph* graph,
-    Code::Kind kind, const char* debug_name, int32_t builtin_index,
+    SourcePositionTable* source_positions, Code::Kind kind,
+    const char* debug_name, int32_t builtin_index,
     PoisoningMitigationLevel poisoning_level, const AssemblerOptions& options) {
   OptimizedCompilationInfo info(CStrVector(debug_name), graph->zone(), kind);
   info.set_builtin_index(builtin_index);
@@ -2080,12 +2081,11 @@ MaybeHandle<Code> Pipeline::GenerateCodeForCodeStub(
   // Construct a pipeline for scheduling and code generation.
   ZoneStats zone_stats(isolate->allocator());
   NodeOriginTable node_origins(graph);
-  SourcePositionTable source_positions(graph);
   JumpOptimizationInfo jump_opt;
   bool should_optimize_jumps =
       isolate->serializer_enabled() && FLAG_turbo_rewrite_far_jumps;
   PipelineData data(&zone_stats, &info, isolate, graph, nullptr,
-                    &source_positions, &node_origins,
+                    source_positions, &node_origins,
                     should_optimize_jumps ? &jump_opt : nullptr, options);
   data.set_verify_graph(FLAG_verify_csa);
   std::unique_ptr<PipelineStatistics> pipeline_statistics;

src/compiler/pipeline.h

@@ -70,7 +70,8 @@ class Pipeline : public AllStatic {
   // Run the pipeline on a machine graph and generate code.
   static MaybeHandle<Code> GenerateCodeForCodeStub(
       Isolate* isolate, CallDescriptor* call_descriptor, Graph* graph,
-      Code::Kind kind, const char* debug_name, int32_t builtin_index,
+      SourcePositionTable* source_positions, Code::Kind kind,
+      const char* debug_name, int32_t builtin_index,
       PoisoningMitigationLevel poisoning_level,
       const AssemblerOptions& options);
 

src/compiler/raw-machine-assembler.cc

@@ -4,6 +4,7 @@
 
 #include "src/compiler/raw-machine-assembler.h"
 
+#include "src/compiler/compiler-source-position-table.h"
 #include "src/compiler/node-properties.h"
 #include "src/compiler/pipeline.h"
 #include "src/compiler/scheduler.h"
@@ -21,6 +22,7 @@ RawMachineAssembler::RawMachineAssembler(
     : isolate_(isolate),
       graph_(graph),
       schedule_(new (zone()) Schedule(zone())),
+      source_positions_(new (zone()) SourcePositionTable(graph)),
       machine_(zone(), word, flags, alignment_requirements),
       common_(zone()),
       simplified_(zone()),
@@ -41,6 +43,14 @@ RawMachineAssembler::RawMachineAssembler(
         AddNode(common()->Parameter(static_cast<int>(i)), graph->start());
   }
   graph->SetEnd(graph->NewNode(common_.End(0)));
+  source_positions_->AddDecorator();
+}
+
+void RawMachineAssembler::SetSourcePosition(const char* file, int line) {
+  int file_id = isolate()->LookupOrAddExternallyCompiledFilename(file);
+  SourcePosition p = SourcePosition::External(line, file_id);
+  DCHECK(p.ExternalLine() == line);
+  source_positions()->SetCurrentPosition(p);
 }
 
 Node* RawMachineAssembler::NullConstant() {
@@ -78,6 +88,7 @@ Schedule* RawMachineAssembler::Export() {
     StdoutStream{} << *schedule_;
   }
   // Invalidate RawMachineAssembler.
+  source_positions_->RemoveDecorator();
   Schedule* schedule = schedule_;
   schedule_ = nullptr;
   return schedule;

src/compiler/raw-machine-assembler.h

@@ -25,7 +25,7 @@ namespace compiler {
 class BasicBlock;
 class RawMachineLabel;
 class Schedule;
-
+class SourcePositionTable;
 
 // The RawMachineAssembler produces a low-level IR graph. All nodes are wired
 // into a graph and also placed into a schedule immediately, hence subsequent
@@ -985,6 +985,9 @@ class V8_EXPORT_PRIVATE RawMachineAssembler {
     return AddNode(op, sizeof...(args) + 1, buffer);
   }
 
+  void SetSourcePosition(const char* file, int line);
+  SourcePositionTable* source_positions() { return source_positions_; }
+
  private:
   Node* MakeNode(const Operator* op, int input_count, Node* const* inputs);
   BasicBlock* Use(RawMachineLabel* label);
@@ -1009,6 +1012,7 @@ class V8_EXPORT_PRIVATE RawMachineAssembler {
   Isolate* isolate_;
   Graph* graph_;
   Schedule* schedule_;
+  SourcePositionTable* source_positions_;
   MachineOperatorBuilder machine_;
   CommonOperatorBuilder common_;
   SimplifiedOperatorBuilder simplified_;

src/isolate.cc

@@ -60,6 +60,7 @@
 #include "src/setup-isolate.h"
 #include "src/simulator.h"
 #include "src/snapshot/embedded-data.h"
+#include "src/snapshot/embedded-file-writer.h"
 #include "src/snapshot/startup-deserializer.h"
 #include "src/tracing/tracing-category-observer.h"
 #include "src/trap-handler/trap-handler.h"
@@ -3145,6 +3146,8 @@ void Isolate::InitializeDefaultEmbeddedBlob() {
 void Isolate::CreateAndSetEmbeddedBlob() {
   base::MutexGuard guard(current_embedded_blob_refcount_mutex_.Pointer());
 
+  PrepareBuiltinSourcePositionMap();
+
   // If a sticky blob has been set, we reuse it.
   if (StickyEmbeddedBlob() != nullptr) {
     CHECK_EQ(embedded_blob(), StickyEmbeddedBlob());
@@ -4154,6 +4157,35 @@ MaybeHandle<Object> Isolate::RunPrepareStackTraceCallback(
   return Utils::OpenHandle(*stack);
 }
 
+int Isolate::LookupOrAddExternallyCompiledFilename(const char* filename) {
+  if (embedded_file_writer_ != nullptr) {
+    return embedded_file_writer_->LookupOrAddExternallyCompiledFilename(
+        filename);
+  }
+  return 0;
+}
+
+const char* Isolate::GetExternallyCompiledFilename(int index) const {
+  if (embedded_file_writer_ != nullptr) {
+    return embedded_file_writer_->GetExternallyCompiledFilename(index);
+  }
+  return "";
+}
+
+int Isolate::GetExternallyCompiledFilenameCount() const {
+  if (embedded_file_writer_ != nullptr) {
+    return embedded_file_writer_->GetExternallyCompiledFilenameCount();
+  }
+  return 0;
+}
+
+void Isolate::PrepareBuiltinSourcePositionMap() {
+  if (embedded_file_writer_ != nullptr) {
+    return embedded_file_writer_->PrepareBuiltinSourcePositionMap(
+        this->builtins());
+  }
+}
+
 void Isolate::SetPrepareStackTraceCallback(PrepareStackTraceCallback callback) {
   prepare_stack_trace_callback_ = callback;
 }

src/isolate.h

@@ -74,6 +74,7 @@ class Counters;
 class Debug;
 class DeoptimizerData;
 class DescriptorLookupCache;
+class EmbeddedFileWriterInterface;
 class EternalHandles;
 class ExternalCallbackScope;
 class HandleScopeImplementer;
@@ -1524,6 +1525,19 @@ class Isolate final : private HiddenFactory {
   Handle<JSObject> RunHostInitializeImportMetaObjectCallback(
       Handle<Module> module);
 
+  void RegisterEmbeddedFileWriter(EmbeddedFileWriterInterface* writer) {
+    embedded_file_writer_ = writer;
+  }
+
+  int LookupOrAddExternallyCompiledFilename(const char* filename);
+  const char* GetExternallyCompiledFilename(int index) const;
+  int GetExternallyCompiledFilenameCount() const;
+  // PrepareBuiltinSourcePositionMap is necessary in order to preserve the
+  // builtin source positions before the corresponding code objects are
+  // replaced with trampolines. Those source positions are used to
+  // annotate the builtin blob with debugging information.
+  void PrepareBuiltinSourcePositionMap();
+
   void SetPrepareStackTraceCallback(PrepareStackTraceCallback callback);
   MaybeHandle<Object> RunPrepareStackTraceCallback(Handle<Context>,
                                                    Handle<JSObject> Error,
@@ -1884,6 +1898,8 @@ class Isolate final : private HiddenFactory {
 
   std::unique_ptr<TracingCpuProfilerImpl> tracing_cpu_profiler_;
 
+  EmbeddedFileWriterInterface* embedded_file_writer_ = nullptr;
+
   // The top entry of the v8::Context::BackupIncumbentScope stack.
   const v8::Context::BackupIncumbentScope* top_backup_incumbent_scope_ =
       nullptr;

src/objects.cc

@@ -15272,15 +15272,33 @@ void Code::Disassemble(const char* name, std::ostream& os, Address current_pc) {
   }
   os << "\n";
 
-  SourcePositionTableIterator it(SourcePositionTable());
-  if (!it.done()) {
-    os << "Source positions:\n pc offset  position\n";
-    for (; !it.done(); it.Advance()) {
-      os << std::setw(10) << std::hex << it.code_offset() << std::dec
-         << std::setw(10) << it.source_position().ScriptOffset()
-         << (it.is_statement() ? "  statement" : "") << "\n";
+  {
+    SourcePositionTableIterator it(
+        SourcePositionTable(), SourcePositionTableIterator::kJavaScriptOnly);
+    if (!it.done()) {
+      os << "Source positions:\n pc offset  position\n";
+      for (; !it.done(); it.Advance()) {
+        os << std::setw(10) << std::hex << it.code_offset() << std::dec
+           << std::setw(10) << it.source_position().ScriptOffset()
+           << (it.is_statement() ? "  statement" : "") << "\n";
+      }
+      os << "\n";
+    }
+  }
+
+  {
+    SourcePositionTableIterator it(SourcePositionTable(),
+                                   SourcePositionTableIterator::kExternalOnly);
+    if (!it.done()) {
+      os << "External Source positions:\n pc offset  fileid  line\n";
+      for (; !it.done(); it.Advance()) {
+        DCHECK(it.source_position().IsExternal());
+        os << std::setw(10) << std::hex << it.code_offset() << std::dec
+           << std::setw(10) << it.source_position().ExternalFileId()
+           << std::setw(10) << it.source_position().ExternalLine() << "\n";
+      }
+      os << "\n";
     }
-    os << "\n";
   }
 
   if (kind() == OPTIMIZED_FUNCTION) {

src/snapshot/embedded-file-writer.cc

@@ -6,6 +6,8 @@
 
 #include <cinttypes>
 
+#include "src/objects/code-inl.h"
+
 namespace v8 {
 namespace internal {
 
@@ -110,6 +112,20 @@ const char* DirectiveAsString(DataDirective directive) {
 #endif
 }
 
+void EmbeddedFileWriter::PrepareBuiltinSourcePositionMap(Builtins* builtins) {
+  for (int i = 0; i < Builtins::builtin_count; i++) {
+    // Retrieve the SourcePositionTable and copy it.
+    Code code = builtins->builtin(i);
+    // Verify that the code object is still the "real code" and not a
+    // trampoline (which wouldn't have source positions).
+    DCHECK(!code->is_off_heap_trampoline());
+    std::vector<unsigned char> data(
+        code->SourcePositionTable()->GetDataStartAddress(),
+        code->SourcePositionTable()->GetDataEndAddress());
+    source_positions_[i] = data;
+  }
+}
+
 // V8_OS_MACOSX
 // Fuchsia target is explicitly excluded here for Mac hosts. This is to avoid
 // generating uncompilable assembly files for the Fuchsia target.
@@ -168,6 +184,10 @@ void PlatformDependentEmbeddedFileWriter::DeclareLabel(const char* name) {
   fprintf(fp_, "_%s:\n", name);
 }
 
+void PlatformDependentEmbeddedFileWriter::SourceInfo(int fileid, int line) {
+  fprintf(fp_, ".loc %d %d\n", fileid, line);
+}
+
 void PlatformDependentEmbeddedFileWriter::DeclareFunctionBegin(
     const char* name) {
   DeclareLabel(name);
@@ -185,6 +205,11 @@ int PlatformDependentEmbeddedFileWriter::HexLiteral(uint64_t value) {
 
 void PlatformDependentEmbeddedFileWriter::FilePrologue() {}
 
+void PlatformDependentEmbeddedFileWriter::DeclareExternalFilename(
+    int fileid, const char* filename) {
+  fprintf(fp_, ".file %d \"%s\"\n", fileid, filename);
+}
+
 void PlatformDependentEmbeddedFileWriter::FileEpilogue() {}
 
 int PlatformDependentEmbeddedFileWriter::IndentedDataDirective(
@@ -247,6 +272,10 @@ void PlatformDependentEmbeddedFileWriter::DeclareLabel(const char* name) {
   fprintf(fp_, "%s:\n", name);
 }
 
+void PlatformDependentEmbeddedFileWriter::SourceInfo(int fileid, int line) {
+  fprintf(fp_, ".loc %d %d\n", fileid, line);
+}
+
 void PlatformDependentEmbeddedFileWriter::DeclareFunctionBegin(
     const char* name) {
   Newline();
@@ -267,6 +296,11 @@ int PlatformDependentEmbeddedFileWriter::HexLiteral(uint64_t value) {
 
 void PlatformDependentEmbeddedFileWriter::FilePrologue() {}
 
+void PlatformDependentEmbeddedFileWriter::DeclareExternalFilename(
+    int fileid, const char* filename) {
+  fprintf(fp_, ".file %d \"%s\"\n", fileid, filename);
+}
+
 void PlatformDependentEmbeddedFileWriter::FileEpilogue() {}
 
 int PlatformDependentEmbeddedFileWriter::IndentedDataDirective(
@@ -330,6 +364,11 @@ void PlatformDependentEmbeddedFileWriter::DeclareLabel(const char* name) {
           DirectiveAsString(kByte));
 }
 
+void PlatformDependentEmbeddedFileWriter::SourceInfo(int fileid, int line) {
+  // TODO(mvstanton): output source information for MSVC.
+  // It's syntax is #line <line> "<filename>"
+}
+
 void PlatformDependentEmbeddedFileWriter::DeclareFunctionBegin(
     const char* name) {
   fprintf(fp_, "%s%s PROC\n", SYMBOL_PREFIX, name);
@@ -349,6 +388,9 @@ void PlatformDependentEmbeddedFileWriter::FilePrologue() {
 #endif
 }
 
+void PlatformDependentEmbeddedFileWriter::DeclareExternalFilename(
+    int fileid, const char* filename) {}
+
 void PlatformDependentEmbeddedFileWriter::FileEpilogue() {
   fprintf(fp_, "END\n");
 }
@@ -527,6 +569,10 @@ void PlatformDependentEmbeddedFileWriter::DeclareLabel(const char* name) {
   fprintf(fp_, "%s%s:\n", SYMBOL_PREFIX, name);
 }
 
+void PlatformDependentEmbeddedFileWriter::SourceInfo(int fileid, int line) {
+  fprintf(fp_, ".loc %d %d\n", fileid, line);
+}
+
 void PlatformDependentEmbeddedFileWriter::DeclareFunctionBegin(
     const char* name) {
   DeclareLabel(name);
@@ -565,6 +611,11 @@ int PlatformDependentEmbeddedFileWriter::HexLiteral(uint64_t value) {
 
 void PlatformDependentEmbeddedFileWriter::FilePrologue() {}
 
+void PlatformDependentEmbeddedFileWriter::DeclareExternalFilename(
+    int fileid, const char* filename) {
+  fprintf(fp_, ".file %d \"%s\"\n", fileid, filename);
+}
+
 void PlatformDependentEmbeddedFileWriter::FileEpilogue() {}
 
 int PlatformDependentEmbeddedFileWriter::IndentedDataDirective(

src/snapshot/embedded-file-writer.h

@@ -10,6 +10,7 @@
 
 #include "src/globals.h"
 #include "src/snapshot/snapshot.h"
+#include "src/source-position-table.h"
 
 namespace v8 {
 namespace internal {
@@ -25,6 +26,7 @@ static constexpr char kDefaultEmbeddedVariant[] = "Default";
 
 // The platform-dependent logic for emitting assembly code for the generated
 // embedded.S file.
+class EmbeddedFileWriter;
 class PlatformDependentEmbeddedFileWriter final {
  public:
   void SetFile(FILE* fp) { fp_ = fp; }
@@ -41,6 +43,7 @@ class PlatformDependentEmbeddedFileWriter final {
 
   void DeclareLabel(const char* name);
 
+  void SourceInfo(int fileid, int line);
   void DeclareFunctionBegin(const char* name);
   void DeclareFunctionEnd(const char* name);
 
@@ -51,6 +54,7 @@ class PlatformDependentEmbeddedFileWriter final {
   void Newline() { fprintf(fp_, "\n"); }
 
   void FilePrologue();
+  void DeclareExternalFilename(int fileid, const char* filename);
   void FileEpilogue();
 
   int IndentedDataDirective(DataDirective directive);
@@ -64,6 +68,21 @@ class PlatformDependentEmbeddedFileWriter final {
   FILE* fp_ = nullptr;
 };
 
+// When writing out compiled builtins to a file, we
+// Detailed source-code information about builtins can only be obtained by
+// registration on the isolate during compilation.
+class EmbeddedFileWriterInterface {
+ public:
+  // We maintain a database of filenames to synthetic IDs.
+  virtual int LookupOrAddExternallyCompiledFilename(const char* filename) = 0;
+  virtual const char* GetExternallyCompiledFilename(int index) const = 0;
+  virtual int GetExternallyCompiledFilenameCount() const = 0;
+
+  // The isolate will call the method below just prior to replacing the
+  // compiled builtin Code objects with trampolines.
+  virtual void PrepareBuiltinSourcePositionMap(Builtins* builtins) = 0;
+};
+
 // Generates the embedded.S file which is later compiled into the final v8
 // binary. Its contents are exported through two symbols:
 //
@@ -73,8 +92,35 @@ class PlatformDependentEmbeddedFileWriter final {
 //     size of the embedded blob in bytes.
 //
 // The variant is usually "Default" but can be modified in multisnapshot builds.
-class EmbeddedFileWriter {
+class EmbeddedFileWriter : public EmbeddedFileWriterInterface {
  public:
+  int LookupOrAddExternallyCompiledFilename(const char* filename) override {
+    auto result = external_filenames_.find(filename);
+    if (result != external_filenames_.end()) {
+      return result->second;
+    }
+    int new_id =
+        ExternalFilenameIndexToId(static_cast<int>(external_filenames_.size()));
+    external_filenames_.insert(std::make_pair(filename, new_id));
+    external_filenames_by_index_.push_back(filename);
+    DCHECK_EQ(external_filenames_by_index_.size(), external_filenames_.size());
+    return new_id;
+  }
+
+  const char* GetExternallyCompiledFilename(int fileid) const override {
+    size_t index = static_cast<size_t>(ExternalFilenameIdToIndex(fileid));
+    DCHECK_GE(index, 0);
+    DCHECK_LT(index, external_filenames_by_index_.size());
+
+    return external_filenames_by_index_[index];
+  }
+
+  int GetExternallyCompiledFilenameCount() const override {
+    return static_cast<int>(external_filenames_.size());
+  }
+
+  void PrepareBuiltinSourcePositionMap(Builtins* builtins) override;
+
   void SetEmbeddedFile(const char* embedded_src_path) {
     embedded_src_path_ = embedded_src_path;
   }
@@ -97,6 +143,7 @@ class EmbeddedFileWriter {
     writer.SetFile(fp);
 
     WriteFilePrologue(&writer);
+    WriteExternalFilenames(&writer);
     WriteMetadataSection(&writer, blob);
     WriteInstructionStreams(&writer, blob);
     WriteFileEpilogue(&writer, blob);
@@ -113,12 +160,26 @@ class EmbeddedFileWriter {
     return fp;
   }
 
-  static void WriteFilePrologue(PlatformDependentEmbeddedFileWriter* w) {
+  void WriteFilePrologue(PlatformDependentEmbeddedFileWriter* w) const {
     w->Comment("Autogenerated file. Do not edit.");
     w->Newline();
     w->FilePrologue();
   }
 
+  void WriteExternalFilenames(PlatformDependentEmbeddedFileWriter* w) const {
+    w->Comment(
+        "Source positions in the embedded blob refer to filenames by id.");
+    w->Comment("Assembly directives here map the id to a filename.");
+    w->Newline();
+
+    // Write external filenames.
+    int size = static_cast<int>(external_filenames_by_index_.size());
+    for (int i = 0; i < size; i++) {
+      w->DeclareExternalFilename(ExternalFilenameIndexToId(i),
+                                 external_filenames_by_index_[i]);
+    }
+  }
+
   // Fairly arbitrary but should fit all symbol names.
   static constexpr int kTemporaryStringLength = 256;
 
@@ -138,34 +199,63 @@ class EmbeddedFileWriter {
                                         i::EmbeddedData::RawDataOffset());
   }
 
-  void WriteInstructionStreams(PlatformDependentEmbeddedFileWriter* w,
-                               const i::EmbeddedData* blob) const {
+  void WriteBuiltin(PlatformDependentEmbeddedFileWriter* w,
+                    const i::EmbeddedData* blob, const int builtin_id) const {
     const bool is_default_variant =
         std::strcmp(embedded_variant_, kDefaultEmbeddedVariant) == 0;
 
-    for (int i = 0; i < i::Builtins::builtin_count; i++) {
-      if (!blob->ContainsBuiltin(i)) continue;
+    char builtin_symbol[kTemporaryStringLength];
+    if (is_default_variant) {
+      // Create nicer symbol names for the default mode.
+      i::SNPrintF(i::Vector<char>(builtin_symbol), "Builtins_%s",
+                  i::Builtins::name(builtin_id));
+    } else {
+      i::SNPrintF(i::Vector<char>(builtin_symbol), "%s_Builtins_%s",
+                  embedded_variant_, i::Builtins::name(builtin_id));
+    }
 
-      char builtin_symbol[kTemporaryStringLength];
-      if (is_default_variant) {
-        // Create nicer symbol names for the default mode.
-        i::SNPrintF(i::Vector<char>(builtin_symbol), "Builtins_%s",
-                    i::Builtins::name(i));
-      } else {
-        i::SNPrintF(i::Vector<char>(builtin_symbol), "%s_Builtins_%s",
-                    embedded_variant_, i::Builtins::name(i));
+    // Labels created here will show up in backtraces. We check in
+    // Isolate::SetEmbeddedBlob that the blob layout remains unchanged, i.e.
+    // that labels do not insert bytes into the middle of the blob byte
+    // stream.
+    w->DeclareFunctionBegin(builtin_symbol);
+    const std::vector<byte>& current_positions = source_positions_[builtin_id];
+
+    // The code below interleaves bytes of assembly code for the builtin
+    // function with source positions at the appropriate offsets.
+    Vector<const byte> vpos(current_positions.data(), current_positions.size());
+    v8::internal::SourcePositionTableIterator positions(
+        vpos, SourcePositionTableIterator::kExternalOnly);
+
+    const uint8_t* data = reinterpret_cast<const uint8_t*>(
+        blob->InstructionStartOfBuiltin(builtin_id));
+    uint32_t size = blob->PaddedInstructionSizeOfBuiltin(builtin_id);
+    uint32_t i = 0;
+    uint32_t next_offset = static_cast<uint32_t>(
+        positions.done() ? size : positions.code_offset());
+    while (i < size) {
+      if (i == next_offset) {
+        // Write source directive.
+        w->SourceInfo(positions.source_position().ExternalFileId(),
+                      positions.source_position().ExternalLine());
+        positions.Advance();
+        next_offset = static_cast<uint32_t>(
+            positions.done() ? size : positions.code_offset());
       }
+      CHECK_GE(next_offset, i);
+      WriteBinaryContentsAsInlineAssembly(w, data + i, next_offset - i);
+      i = next_offset;
+    }
+
+    w->DeclareFunctionEnd(builtin_symbol);
+  }
+
+  void WriteInstructionStreams(PlatformDependentEmbeddedFileWriter* w,
+                               const i::EmbeddedData* blob) const {
+    for (int i = 0; i < i::Builtins::builtin_count; i++) {
+      if (!blob->ContainsBuiltin(i)) continue;
 
-      // Labels created here will show up in backtraces. We check in
-      // Isolate::SetEmbeddedBlob that the blob layout remains unchanged, i.e.
-      // that labels do not insert bytes into the middle of the blob byte
-      // stream.
-      w->DeclareFunctionBegin(builtin_symbol);
-      WriteBinaryContentsAsInlineAssembly(
-          w,
-          reinterpret_cast<const uint8_t*>(blob->InstructionStartOfBuiltin(i)),
-          blob->PaddedInstructionSizeOfBuiltin(i));
-      w->DeclareFunctionEnd(builtin_symbol);
+      WriteBuiltin(w, blob, i);
     }
     w->Newline();
   }
@@ -285,11 +375,10 @@ class EmbeddedFileWriter {
       PlatformDependentEmbeddedFileWriter* w, const uint8_t* data,
       uint32_t size) {
     int current_line_length = 0;
-
     uint32_t i = 0;
 
     // Begin by writing out byte chunks.
-    for (; i <= size - kByteChunkSize; i += kByteChunkSize) {
+    for (; i + kByteChunkSize < size; i += kByteChunkSize) {
       current_line_length = WriteDirectiveOrSeparator(w, current_line_length,
                                                       kByteChunkDirective);
       current_line_length = WriteByteChunk(w, current_line_length, data + i);
@@ -306,9 +395,25 @@ class EmbeddedFileWriter {
       current_line_length += w->HexLiteral(data[i]);
       current_line_length = WriteLineEndIfNeeded(w, current_line_length, 1);
     }
+
     if (current_line_length != 0) w->Newline();
   }
 
+  static int ExternalFilenameIndexToId(int index) {
+    return kFirstExternalFilenameId + index;
+  }
+
+  static int ExternalFilenameIdToIndex(int id) {
+    return id - kFirstExternalFilenameId;
+  }
+
+  std::vector<byte> source_positions_[Builtins::builtin_count];
+
+  // In assembly directives, filename ids need to begin with 1.
+  static const int kFirstExternalFilenameId = 1;
+  std::map<const char*, int> external_filenames_;
+  std::vector<const char*> external_filenames_by_index_;
+
   const char* embedded_src_path_ = nullptr;
   const char* embedded_variant_ = kDefaultEmbeddedVariant;
 };

src/snapshot/mksnapshot.cc

@@ -16,6 +16,7 @@
 #include "src/snapshot/partial-serializer.h"
 #include "src/snapshot/snapshot.h"
 #include "src/snapshot/startup-serializer.h"
+#include "src/source-position-table.h"
 
 namespace {
 
@@ -295,9 +296,14 @@ int main(int argc, char** argv) {
                 : std::min(i::kMaximalCodeRangeSize / i::MB,
                            i::kMaxPCRelativeCodeRangeInMB);
         i_isolate->heap()->ConfigureHeap(0, 0, code_range_size);
+        // The isolate contains data from builtin compilation that needs
+        // to be written out if builtins are embedded.
+        i_isolate->RegisterEmbeddedFileWriter(&embedded_writer);
       }
       v8::SnapshotCreator snapshot_creator(isolate);
-      if (i::FLAG_embedded_builtins) WriteEmbeddedFile(&embedded_writer);
+      if (i::FLAG_embedded_builtins) {
+        WriteEmbeddedFile(&embedded_writer);
+      }
       blob = CreateSnapshotDataBlob(&snapshot_creator, embed_script.get());
     }
 

src/source-position-table.cc

@@ -164,7 +164,7 @@ Handle<ByteArray> SourcePositionTableBuilder::ToSourcePositionTable(
 #ifdef ENABLE_SLOW_DCHECKS
   // Brute force testing: Record all positions and decode
   // the entire table to verify they are identical.
-  SourcePositionTableIterator it(*table);
+  SourcePositionTableIterator it(*table, SourcePositionTableIterator::kAll);
   CheckTableEquals(raw_entries_, it);
   // No additional source positions after creating the table.
   mode_ = OMIT_SOURCE_POSITIONS;
@@ -181,7 +181,8 @@ OwnedVector<byte> SourcePositionTableBuilder::ToSourcePositionTableVector() {
 #ifdef ENABLE_SLOW_DCHECKS
   // Brute force testing: Record all positions and decode
   // the entire table to verify they are identical.
-  SourcePositionTableIterator it(table.as_vector());
+  SourcePositionTableIterator it(table.as_vector(),
+                                 SourcePositionTableIterator::kAll);
   CheckTableEquals(raw_entries_, it);
   // No additional source positions after creating the table.
   mode_ = OMIT_SOURCE_POSITIONS;
@@ -189,14 +190,15 @@ OwnedVector<byte> SourcePositionTableBuilder::ToSourcePositionTableVector() {
   return table;
 }
 
-SourcePositionTableIterator::SourcePositionTableIterator(ByteArray byte_array)
-    : raw_table_(VectorFromByteArray(byte_array)) {
+SourcePositionTableIterator::SourcePositionTableIterator(ByteArray byte_array,
+                                                         IterationFilter filter)
+    : raw_table_(VectorFromByteArray(byte_array)), filter_(filter) {
   Advance();
 }
 
 SourcePositionTableIterator::SourcePositionTableIterator(
-    Handle<ByteArray> byte_array)
-    : table_(byte_array) {
+    Handle<ByteArray> byte_array, IterationFilter filter)
+    : table_(byte_array), filter_(filter) {
   Advance();
 #ifdef DEBUG
   // We can enable allocation because we keep the table in a handle.
@@ -205,8 +207,8 @@ SourcePositionTableIterator::SourcePositionTableIterator(
 }
 
 SourcePositionTableIterator::SourcePositionTableIterator(
-    Vector<const byte> bytes)
-    : raw_table_(bytes) {
+    Vector<const byte> bytes, IterationFilter filter)
+    : raw_table_(bytes), filter_(filter) {
   Advance();
 #ifdef DEBUG
   // We can enable allocation because the underlying vector does not move.
@@ -219,12 +221,19 @@ void SourcePositionTableIterator::Advance() {
       table_.is_null() ? raw_table_ : VectorFromByteArray(*table_);
   DCHECK(!done());
   DCHECK(index_ >= 0 && index_ <= bytes.length());
-  if (index_ >= bytes.length()) {
-    index_ = kDone;
-  } else {
-    PositionTableEntry tmp;
-    DecodeEntry(bytes, &index_, &tmp);
-    AddAndSetEntry(current_, tmp);
+  bool filter_satisfied = false;
+  while (!done() && !filter_satisfied) {
+    if (index_ >= bytes.length()) {
+      index_ = kDone;
+    } else {
+      PositionTableEntry tmp;
+      DecodeEntry(bytes, &index_, &tmp);
+      AddAndSetEntry(current_, tmp);
+      SourcePosition p = source_position();
+      filter_satisfied = (filter_ == kAll) ||
+                         (filter_ == kJavaScriptOnly && p.IsJavaScript()) ||
+                         (filter_ == kExternalOnly && p.IsExternal());
+    }
   }
 }
 

src/source-position-table.h

@@ -59,10 +59,13 @@ class V8_EXPORT_PRIVATE SourcePositionTableBuilder {
 
 class V8_EXPORT_PRIVATE SourcePositionTableIterator {
  public:
+  enum IterationFilter { kJavaScriptOnly = 0, kExternalOnly = 1, kAll = 2 };
+
   // Used for saving/restoring the iterator.
-  struct IndexAndPosition {
+  struct IndexAndPositionState {
     int index_;
     PositionTableEntry position_;
+    IterationFilter filter_;
   };
 
   // We expose three flavours of the iterator, depending on the argument passed
@@ -70,16 +73,19 @@ class V8_EXPORT_PRIVATE SourcePositionTableIterator {
 
   // Handlified iterator allows allocation, but it needs a handle (and thus
   // a handle scope). This is the preferred version.
-  explicit SourcePositionTableIterator(Handle<ByteArray> byte_array);
+  explicit SourcePositionTableIterator(
+      Handle<ByteArray> byte_array, IterationFilter filter = kJavaScriptOnly);
 
   // Non-handlified iterator does not need a handle scope, but it disallows
   // allocation during its lifetime. This is useful if there is no handle
   // scope around.
-  explicit SourcePositionTableIterator(ByteArray byte_array);
+  explicit SourcePositionTableIterator(
+      ByteArray byte_array, IterationFilter filter = kJavaScriptOnly);
 
   // Handle-safe iterator based on an a vector located outside the garbage
   // collected heap, allows allocation during its lifetime.
-  explicit SourcePositionTableIterator(Vector<const byte> bytes);
+  explicit SourcePositionTableIterator(
+      Vector<const byte> bytes, IterationFilter filter = kJavaScriptOnly);
 
   void Advance();
 
@@ -97,11 +103,12 @@ class V8_EXPORT_PRIVATE SourcePositionTableIterator {
   }
   bool done() const { return index_ == kDone; }
 
-  IndexAndPosition GetState() const { return {index_, current_}; }
+  IndexAndPositionState GetState() const { return {index_, current_, filter_}; }
 
-  void RestoreState(const IndexAndPosition& saved_state) {
+  void RestoreState(const IndexAndPositionState& saved_state) {
     index_ = saved_state.index_;
     current_ = saved_state.position_;
+    filter_ = saved_state.filter_;
   }
 
  private:
@@ -111,6 +118,7 @@ class V8_EXPORT_PRIVATE SourcePositionTableIterator {
   Handle<ByteArray> table_;
   int index_ = 0;
   PositionTableEntry current_;
+  IterationFilter filter_;
   DISALLOW_HEAP_ALLOCATION(no_gc);
 };
 

src/source-position.cc

@@ -37,7 +37,12 @@ std::ostream& operator<<(std::ostream& out, const SourcePosition& pos) {
   } else {
     out << "<not inlined:";
   }
-  out << pos.ScriptOffset() << ">";
+
+  if (pos.IsExternal()) {
+    out << pos.ExternalLine() << ", " << pos.ExternalFileId() << ">";
+  } else {
+    out << pos.ScriptOffset() << ">";
+  }
   return out;
 }
 
@@ -96,8 +101,14 @@ void SourcePosition::Print(std::ostream& out,
 }
 
 void SourcePosition::PrintJson(std::ostream& out) const {
-  out << "{ \"scriptOffset\" : " << ScriptOffset() << ", "
-      << "  \"inliningId\" : " << InliningId() << "}";
+  if (IsExternal()) {
+    out << "{ \"line\" : " << ExternalLine() << ", "
+        << "  \"fileId\" : " << ExternalFileId() << ", "
+        << "  \"inliningId\" : " << InliningId() << "}";
+  } else {
+    out << "{ \"scriptOffset\" : " << ScriptOffset() << ", "
+        << "  \"inliningId\" : " << InliningId() << "}";
+  }
 }
 
 void SourcePosition::Print(std::ostream& out, Code code) const {

src/source-position.h

@@ -22,9 +22,23 @@ class SharedFunctionInfo;
 struct SourcePositionInfo;
 
 // SourcePosition stores
-// - script_offset (31 bit non-negative int or kNoSourcePosition)
+// - is_external (1 bit true/false)
+//
+// - if is_external is true:
+// - external_line (20 bits, non-negative int)
+// - external_file_id (10 bits, non-negative int)
+//
+// - if is_external is false:
+// - script_offset (30 bit non-negative int or kNoSourcePosition)
+//
+// - In both cases, there is an inlining_id.
 // - inlining_id (16 bit non-negative int or kNotInlined).
 //
+// An "external" SourcePosition is one given by a file_id and a line,
+// suitable for embedding references to .cc or .tq files.
+// Otherwise, a SourcePosition contains an offset into a JavaScript
+// file.
+//
 // A defined inlining_id refers to positions in
 // OptimizedCompilationInfo::inlined_functions or
 // DeoptimizationData::InliningPositions, depending on the compilation stage.
@@ -32,15 +46,39 @@ class SourcePosition final {
  public:
   explicit SourcePosition(int script_offset, int inlining_id = kNotInlined)
       : value_(0) {
+    SetIsExternal(false);
     SetScriptOffset(script_offset);
     SetInliningId(inlining_id);
   }
 
+  // External SourcePositions should use the following method to construct
+  // SourcePositions to avoid confusion.
+  static SourcePosition External(int line, int file_id) {
+    return SourcePosition(line, file_id, kNotInlined);
+  }
+
   static SourcePosition Unknown() { return SourcePosition(kNoSourcePosition); }
   bool IsKnown() const {
+    if (IsExternal()) return true;
     return ScriptOffset() != kNoSourcePosition || InliningId() != kNotInlined;
   }
-  bool isInlined() const { return InliningId() != kNotInlined; }
+  bool isInlined() const {
+    if (IsExternal()) return false;
+    return InliningId() != kNotInlined;
+  }
+
+  bool IsExternal() const { return IsExternalField::decode(value_); }
+  bool IsJavaScript() const { return !IsExternal(); }
+
+  int ExternalLine() const {
+    DCHECK(IsExternal());
+    return ExternalLineField::decode(value_);
+  }
+
+  int ExternalFileId() const {
+    DCHECK(IsExternal());
+    return ExternalFileIdField::decode(value_);
+  }
 
   // Assumes that the code object is optimized
   std::vector<SourcePositionInfo> InliningStack(Handle<Code> code) const;
@@ -50,10 +88,28 @@ class SourcePosition final {
   void Print(std::ostream& out, Code code) const;
   void PrintJson(std::ostream& out) const;
 
-  int ScriptOffset() const { return ScriptOffsetField::decode(value_) - 1; }
+  int ScriptOffset() const {
+    DCHECK(IsJavaScript());
+    return ScriptOffsetField::decode(value_) - 1;
+  }
   int InliningId() const { return InliningIdField::decode(value_) - 1; }
 
+  void SetIsExternal(bool external) {
+    value_ = IsExternalField::update(value_, external);
+  }
+  void SetExternalLine(int line) {
+    DCHECK(IsExternal());
+    DCHECK(line <= ExternalLineField::kMax - 1);
+    value_ = ExternalLineField::update(value_, line);
+  }
+  void SetExternalFileId(int file_id) {
+    DCHECK(IsExternal());
+    DCHECK(file_id <= ExternalFileIdField::kMax - 1);
+    value_ = ExternalFileIdField::update(value_, file_id);
+  }
+
   void SetScriptOffset(int script_offset) {
+    DCHECK(IsJavaScript());
     DCHECK(script_offset <= ScriptOffsetField::kMax - 2);
     DCHECK_GE(script_offset, kNoSourcePosition);
     value_ = ScriptOffsetField::update(value_, script_offset + 1);
@@ -76,12 +132,29 @@ class SourcePosition final {
   }
 
  private:
+  // Used by SourcePosition::External(line, file_id).
+  SourcePosition(int line, int file_id, int inlining_id) : value_(0) {
+    SetIsExternal(true);
+    SetExternalLine(line);
+    SetExternalFileId(file_id);
+    SetInliningId(inlining_id);
+  }
+
   void Print(std::ostream& out, SharedFunctionInfo function) const;
 
+  typedef BitField64<bool, 0, 1> IsExternalField;
+
+  // The two below are only used if IsExternal() is true.
+  typedef BitField64<int, 1, 20> ExternalLineField;
+  typedef BitField64<int, 21, 10> ExternalFileIdField;
+
+  // ScriptOffsetField is only used if IsExternal() is false.
+  typedef BitField64<int, 1, 30> ScriptOffsetField;
+
   // InliningId is in the high bits for better compression in
   // SourcePositionTable.
-  typedef BitField64<int, 0, 31> ScriptOffsetField;
   typedef BitField64<int, 31, 16> InliningIdField;
+
   // Leaving the highest bit untouched to allow for signed conversion.
   uint64_t value_;
 };

tools/snapshot/asm_to_inline_asm.py

@@ -17,6 +17,8 @@ def asm_to_inl_asm(in_filename, out_filename):
   with open(in_filename, 'r') as infile, open(out_filename, 'wb') as outfile:
     outfile.write('__asm__(\n')
     for line in infile:
+      # Escape " in .S file before outputing it to inline asm file.
+      line = line.replace('"', '\\"')
       outfile.write('  "%s\\n"\n' % line.rstrip())
     outfile.write(');\n')
   return 0