[torque] Change SourcePosition to include the 'end' of a token

This change will enable basic "goto definition" support in the upcoming
Torque language server.

R=tebbi@chromium.org

Bug: v8:7793
Change-Id: I8e50cc58288991a2f6024d06bf38f4fd66f21eea
Reviewed-on: https://chromium-review.googlesource.com/c/1477055
Commit-Queue: Simon Zünd <szuend@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59694}

src/torque/csa-generator.cc

@@ -56,11 +56,11 @@ Stack<std::string> CSAGenerator::EmitBlock(const Block* block) {
 
 void CSAGenerator::EmitSourcePosition(SourcePosition pos, bool always_emit) {
   const std::string& file = SourceFileMap::GetSource(pos.source);
-  if (always_emit || !previous_position_.CompareIgnoreColumn(pos)) {
+  if (always_emit || !previous_position_.CompareStartIgnoreColumn(pos)) {
     // Lines in Torque SourcePositions are zero-based, while the
     // CodeStubAssembler and downwind systems are one-based.
-    out_ << "    ca_.SetSourcePosition(\"" << file << "\", " << (pos.line + 1)
-         << ");\n";
+    out_ << "    ca_.SetSourcePosition(\"" << file << "\", "
+         << (pos.start.line + 1) << ");\n";
     previous_position_ = pos;
   }
 }
@@ -679,7 +679,7 @@ void CSAGenerator::EmitInstruction(const AbortInstruction& instruction,
           StringLiteralQuote(SourceFileMap::GetSource(instruction.pos.source));
       out_ << "    CodeStubAssembler(state_).FailAssert("
            << StringLiteralQuote(instruction.message) << ", " << file << ", "
-           << instruction.pos.line + 1 << ");\n";
+           << instruction.pos.start.line + 1 << ");\n";
       break;
     }
   }

src/torque/earley-parser.cc

@@ -17,18 +17,27 @@ namespace torque {
 
 namespace {
 
-void UpdateSourcePosition(InputPosition from, InputPosition to,
-                          SourcePosition* pos) {
-  while (from != to) {
-    if (*from == '\n') {
-      pos->line += 1;
-      pos->column = 0;
-    } else {
-      pos->column += 1;
+struct LineAndColumnTracker {
+  LineAndColumn previous{0, 0};
+  LineAndColumn current{0, 0};
+
+  void Advance(InputPosition from, InputPosition to) {
+    previous = current;
+    while (from != to) {
+      if (*from == '\n') {
+        current.line += 1;
+        current.column = 0;
+      } else {
+        current.column += 1;
+      }
+      ++from;
     }
-    ++from;
   }
-}
+
+  SourcePosition ToSourcePosition() {
+    return {CurrentSourceFile::Get(), previous, current};
+  }
+};
 
 }  // namespace
 
@@ -107,11 +116,11 @@ LexerResult Lexer::RunLexer(const std::string& input) {
   InputPosition const end = begin + input.size();
   InputPosition pos = begin;
   InputPosition token_start = pos;
-  CurrentSourcePosition::Scope scope(
-      SourcePosition{CurrentSourceFile::Get(), 0, 0});
+  LineAndColumnTracker line_column_tracker;
+
   match_whitespace_(&pos);
   while (pos != end) {
-    UpdateSourcePosition(token_start, pos, &CurrentSourcePosition::Get());
+    line_column_tracker.Advance(token_start, pos);
     token_start = pos;
     Symbol* symbol = MatchToken(&pos, end);
     if (!symbol) {
@@ -120,14 +129,17 @@ LexerResult Lexer::RunLexer(const std::string& input) {
                       token_start, token_start + std::min<ptrdiff_t>(
                                                      end - token_start, 10))));
     }
+    line_column_tracker.Advance(token_start, pos);
     result.token_symbols.push_back(symbol);
     result.token_contents.push_back(
-        {token_start, pos, CurrentSourcePosition::Get()});
+        {token_start, pos, line_column_tracker.ToSourcePosition()});
     match_whitespace_(&pos);
   }
-  UpdateSourcePosition(token_start, pos, &CurrentSourcePosition::Get());
+
   // Add an additional token position to simplify corner cases.
-  result.token_contents.push_back({pos, pos, CurrentSourcePosition::Get()});
+  line_column_tracker.Advance(token_start, pos);
+  result.token_contents.push_back(
+      {pos, pos, line_column_tracker.ToSourcePosition()});
   return result;
 }
 
@@ -176,7 +188,7 @@ const Item* RunEarleyAlgorithm(
   // Worklist for items at the next position.
   std::vector<Item> future_items;
   CurrentSourcePosition::Scope source_position(
-      SourcePosition{CurrentSourceFile::Get(), 0, 0});
+      SourcePosition{CurrentSourceFile::Get(), {0, 0}, {0, 0}});
   std::vector<const Item*> completed_items;
   std::unordered_map<std::pair<size_t, Symbol*>, std::set<const Item*>,
                      base::hash<std::pair<size_t, Symbol*>>>

src/torque/global-context.h

@@ -20,7 +20,7 @@ class GlobalContext : public ContextualClass<GlobalContext> {
   explicit GlobalContext(Ast ast) : verbose_(false), ast_(std::move(ast)) {
     CurrentScope::Scope current_scope(nullptr);
     CurrentSourcePosition::Scope current_source_position(
-        SourcePosition{CurrentSourceFile::Get(), -1, -1});
+        SourcePosition{CurrentSourceFile::Get(), {-1, -1}, {-1, -1}});
     default_namespace_ =
         RegisterDeclarable(base::make_unique<Namespace>(kBaseNamespaceName));
   }

src/torque/implementation-visitor.cc

@@ -2558,7 +2558,7 @@ void ImplementationVisitor::GenerateBuiltinDefinitions(std::string& file_name) {
         Declarations::FindSomeInternalBuiltinWithType(type);
     if (!example_builtin) {
       CurrentSourcePosition::Scope current_source_position(
-          SourcePosition{CurrentSourceFile::Get(), -1, -1});
+          SourcePosition{CurrentSourceFile::Get(), {-1, -1}, {-1, -1}});
       ReportError("unable to find any builtin with type \"", *type, "\"");
     }
     new_contents_stream << "  V(" << type->function_pointer_type_id() << ","

src/torque/source-positions.h

@@ -24,20 +24,26 @@ class SourceId {
   friend class SourceFileMap;
 };
 
-struct SourcePosition {
-  SourceId source;
+struct LineAndColumn {
   int line;
   int column;
+
+  static LineAndColumn Invalid() { return {-1, -1}; }
+};
+
+struct SourcePosition {
+  SourceId source;
+  LineAndColumn start;
+  LineAndColumn end;
+
   static SourcePosition Invalid() {
-    SourcePosition pos{SourceId::Invalid(), -1, -1};
+    SourcePosition pos{SourceId::Invalid(), LineAndColumn::Invalid(),
+                       LineAndColumn::Invalid()};
     return pos;
   }
-  int operator==(const SourcePosition& pos) const {
-    return line == pos.line && column == pos.column && source == pos.source;
-  }
-  int operator!=(const SourcePosition& pos) const { return !(*this == pos); }
-  bool CompareIgnoreColumn(const SourcePosition& pos) const {
-    return line == pos.line && source == pos.source;
+
+  bool CompareStartIgnoreColumn(const SourcePosition& pos) const {
+    return start.line == pos.start.line && source == pos.source;
   }
 };
 
@@ -62,7 +68,8 @@ class SourceFileMap : public ContextualClass<SourceFileMap> {
 
 inline std::string PositionAsString(SourcePosition pos) {
   return SourceFileMap::GetSource(pos.source) + ":" +
-         std::to_string(pos.line + 1) + ":" + std::to_string(pos.column + 1);
+         std::to_string(pos.start.line + 1) + ":" +
+         std::to_string(pos.start.column + 1);
 }
 
 inline std::ostream& operator<<(std::ostream& out, SourcePosition pos) {