[torque] add unittest for Earley parser

Change-Id: I02c117ef66480eb73eb9cc1d4f80bbc64e9d3624
Reviewed-on: https://chromium-review.googlesource.com/1146655
Commit-Queue: Tobias Tebbi <tebbi@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Jaroslav Sevcik <jarin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#54649}

BUILD.gn

@@ -2860,6 +2860,46 @@ v8_source_set("v8_base") {
   }
 }
 
+v8_source_set("torque_base") {
+  visibility = [ ":*" ]  # Only targets in this file can depend on this.
+
+  sources = [
+    "src/torque/ast.h",
+    "src/torque/contextual.h",
+    "src/torque/declarable.cc",
+    "src/torque/declarable.h",
+    "src/torque/declaration-visitor.cc",
+    "src/torque/declaration-visitor.h",
+    "src/torque/declarations.cc",
+    "src/torque/declarations.h",
+    "src/torque/earley-parser.cc",
+    "src/torque/earley-parser.h",
+    "src/torque/file-visitor.cc",
+    "src/torque/file-visitor.h",
+    "src/torque/global-context.h",
+    "src/torque/implementation-visitor.cc",
+    "src/torque/implementation-visitor.h",
+    "src/torque/scope.cc",
+    "src/torque/scope.h",
+    "src/torque/source-positions.cc",
+    "src/torque/source-positions.h",
+    "src/torque/torque-parser.cc",
+    "src/torque/torque-parser.h",
+    "src/torque/type-oracle.cc",
+    "src/torque/type-oracle.h",
+    "src/torque/types.cc",
+    "src/torque/types.h",
+    "src/torque/utils.cc",
+    "src/torque/utils.h",
+  ]
+
+  deps = [
+    ":v8_libbase",
+  ]
+
+  configs = [ ":internal_config" ]
+}
+
 v8_component("v8_libbase") {
   sources = [
     "src/base/adapters.h",
@@ -3165,45 +3205,15 @@ if (current_toolchain == v8_snapshot_toolchain) {
     visibility = [ ":*" ]  # Only targets in this file can depend on this.
 
     sources = [
-      "src/torque/ast.h",
-      "src/torque/contextual.h",
-      "src/torque/declarable.cc",
-      "src/torque/declarable.h",
-      "src/torque/declaration-visitor.cc",
-      "src/torque/declaration-visitor.h",
-      "src/torque/declarations.cc",
-      "src/torque/declarations.h",
-      "src/torque/earley-parser.cc",
-      "src/torque/earley-parser.h",
-      "src/torque/file-visitor.cc",
-      "src/torque/file-visitor.h",
-      "src/torque/global-context.h",
-      "src/torque/implementation-visitor.cc",
-      "src/torque/implementation-visitor.h",
-      "src/torque/scope.cc",
-      "src/torque/scope.h",
-      "src/torque/source-positions.cc",
-      "src/torque/source-positions.h",
-      "src/torque/torque-parser.cc",
-      "src/torque/torque-parser.h",
       "src/torque/torque.cc",
-      "src/torque/type-oracle.cc",
-      "src/torque/type-oracle.h",
-      "src/torque/types.cc",
-      "src/torque/types.h",
-      "src/torque/utils.cc",
-      "src/torque/utils.h",
     ]
 
     deps = [
-      ":v8_libbase",
+      ":torque_base",
       "//build/win:default_exe_manifest",
     ]
 
-    configs = [
-      ":external_config",
-      ":internal_config_base",
-    ]
+    configs = [ ":internal_config" ]
   }
 }
 
@@ -3308,6 +3318,7 @@ if (is_component_build) {
     ]
 
     public_deps = [
+      ":torque_base",
       ":v8_base",
       ":v8_maybe_snapshot",
     ]
@@ -3334,6 +3345,7 @@ if (is_component_build) {
     testonly = true
 
     public_deps = [
+      ":torque_base",
       ":v8_base",
       ":v8_maybe_snapshot",
     ]

src/base/macros.h

@@ -416,4 +416,30 @@ bool is_inbounds(float_t v) {
          (kUpperBoundIsMax ? (v <= kUpperBound) : (v < kUpperBound));
 }
 
+#ifdef V8_OS_WIN
+
+// Setup for Windows shared library export.
+#ifdef BUILDING_V8_SHARED
+#define V8_EXPORT_PRIVATE __declspec(dllexport)
+#elif USING_V8_SHARED
+#define V8_EXPORT_PRIVATE __declspec(dllimport)
+#else
+#define V8_EXPORT_PRIVATE
+#endif  // BUILDING_V8_SHARED
+
+#else  // V8_OS_WIN
+
+// Setup for Linux shared library export.
+#if V8_HAS_ATTRIBUTE_VISIBILITY
+#ifdef BUILDING_V8_SHARED
+#define V8_EXPORT_PRIVATE __attribute__((visibility("default")))
+#else
+#define V8_EXPORT_PRIVATE
+#endif
+#else
+#define V8_EXPORT_PRIVATE
+#endif
+
+#endif  // V8_OS_WIN
+
 #endif  // V8_BASE_MACROS_H_

src/globals.h

@@ -17,32 +17,6 @@
 #include "src/base/logging.h"
 #include "src/base/macros.h"
 
-#ifdef V8_OS_WIN
-
-// Setup for Windows shared library export.
-#ifdef BUILDING_V8_SHARED
-#define V8_EXPORT_PRIVATE __declspec(dllexport)
-#elif USING_V8_SHARED
-#define V8_EXPORT_PRIVATE __declspec(dllimport)
-#else
-#define V8_EXPORT_PRIVATE
-#endif  // BUILDING_V8_SHARED
-
-#else  // V8_OS_WIN
-
-// Setup for Linux shared library export.
-#if V8_HAS_ATTRIBUTE_VISIBILITY
-#ifdef BUILDING_V8_SHARED
-#define V8_EXPORT_PRIVATE __attribute__((visibility("default")))
-#else
-#define V8_EXPORT_PRIVATE
-#endif
-#else
-#define V8_EXPORT_PRIVATE
-#endif
-
-#endif  // V8_OS_WIN
-
 #define V8_INFINITY std::numeric_limits<double>::infinity()
 
 namespace v8 {

src/torque/contextual.h

@@ -38,13 +38,13 @@ class ContextualVariable {
    public:
     template <class... Args>
     explicit Scope(Args&&... args)
-        : current_(std::forward<Args>(args)...), previous_(top_) {
-      top_ = &current_;
+        : current_(std::forward<Args>(args)...), previous_(Top()) {
+      Top() = &current_;
     }
     ~Scope() {
       // Ensure stack discipline.
-      DCHECK_EQ(&current_, top_);
-      top_ = previous_;
+      DCHECK_EQ(&current_, Top());
+      Top() = previous_;
     }
 
    private:
@@ -61,12 +61,12 @@ class ContextualVariable {
   // Access the most recent active {Scope}. There has to be an active {Scope}
   // for this contextual variable.
   static VarType& Get() {
-    DCHECK_NOT_NULL(top_);
-    return *top_;
+    DCHECK_NOT_NULL(Top());
+    return *Top();
   }
 
  private:
-  static thread_local VarType* top_;
+  V8_EXPORT_PRIVATE static VarType*& Top();
 };
 
 // Usage: DECLARE_CONTEXTUAL_VARIABLE(VarName, VarType)
@@ -74,10 +74,13 @@ class ContextualVariable {
   struct VarName                                  \
       : v8::internal::torque::ContextualVariable<VarName, __VA_ARGS__> {};
 
-#define DEFINE_CONTEXTUAL_VARIABLE(VarName) \
-  template <>                               \
-  thread_local VarName::VariableType*       \
-      ContextualVariable<VarName, VarName::VariableType>::top_ = nullptr;
+#define DEFINE_CONTEXTUAL_VARIABLE(VarName)                   \
+  template <>                                                 \
+  V8_EXPORT_PRIVATE VarName::VariableType*&                   \
+  ContextualVariable<VarName, VarName::VariableType>::Top() { \
+    static thread_local VarName::VariableType* top = nullptr; \
+    return top;                                               \
+  }
 
 // By inheriting from {ContextualClass} a class can become a contextual variable
 // of itself, which is very similar to a singleton.

src/torque/declarable.cc

@@ -81,6 +81,8 @@ std::ostream& operator<<(std::ostream& os, const Generic& g) {
   return os;
 }
 
+size_t Label::next_id_ = 0;
+
 }  // namespace torque
 }  // namespace internal
 }  // namespace v8

src/torque/earley-parser.h

@@ -48,7 +48,7 @@ class ParseResultHolder : public ParseResultHolderBase {
       : ParseResultHolderBase(id), value_(std::move(value)) {}
 
  private:
-  static const TypeId id;
+  V8_EXPORT_PRIVATE static const TypeId id;
   friend class ParseResultHolderBase;
   T value_;
 };
@@ -157,8 +157,8 @@ class Rule final {
     left_hand_side_ = left_hand_side;
   }
 
-  base::Optional<ParseResult> RunAction(const Item* completed_item,
-                                        const LexerResult& tokens) const;
+  V8_EXPORT_PRIVATE base::Optional<ParseResult> RunAction(
+      const Item* completed_item, const LexerResult& tokens) const;
 
  private:
   Symbol* left_hand_side_ = nullptr;
@@ -178,7 +178,7 @@ class Symbol {
   Symbol() : Symbol({}) {}
   Symbol(std::initializer_list<Rule> rules) { *this = rules; }
 
-  Symbol& operator=(std::initializer_list<Rule> rules);
+  V8_EXPORT_PRIVATE Symbol& operator=(std::initializer_list<Rule> rules);
 
   bool IsTerminal() const { return rules_.empty(); }
   Rule* rule(size_t index) const { return rules_[index].get(); }
@@ -189,8 +189,8 @@ class Symbol {
     rules_.back()->SetLeftHandSide(this);
   }
 
-  base::Optional<ParseResult> RunAction(const Item* item,
-                                        const LexerResult& tokens);
+  V8_EXPORT_PRIVATE base::Optional<ParseResult> RunAction(
+      const Item* item, const LexerResult& tokens);
 
  private:
   std::vector<std::unique_ptr<Rule>> rules_;
@@ -292,7 +292,7 @@ inline base::Optional<ParseResult> Symbol::RunAction(
   return item->rule()->RunAction(item, tokens);
 }
 
-const Item* RunEarleyAlgorithm(
+V8_EXPORT_PRIVATE const Item* RunEarleyAlgorithm(
     Symbol* start, const LexerResult& tokens,
     std::unordered_set<Item, base::hash<Item>>* processed);
 
@@ -327,7 +327,7 @@ class Lexer {
 
   Symbol* Pattern(PatternFunction pattern) { return &patterns_[pattern]; }
   Symbol* Token(const std::string& keyword) { return &keywords_[keyword]; }
-  LexerResult RunLexer(const std::string& input);
+  V8_EXPORT_PRIVATE LexerResult RunLexer(const std::string& input);
 
  private:
   PatternFunction match_whitespace_ = [](InputPosition*) { return false; };
@@ -365,10 +365,12 @@ class Grammar {
 
   // Helper functions to define lexer patterns. If they match, they return true
   // and advance {pos}. Otherwise, {pos} is unchanged.
-  static bool MatchChar(int (*char_class)(int), InputPosition* pos);
-  static bool MatchChar(bool (*char_class)(char), InputPosition* pos);
-  static bool MatchAnyChar(InputPosition* pos);
-  static bool MatchString(const char* s, InputPosition* pos);
+  V8_EXPORT_PRIVATE static bool MatchChar(int (*char_class)(int),
+                                          InputPosition* pos);
+  V8_EXPORT_PRIVATE static bool MatchChar(bool (*char_class)(char),
+                                          InputPosition* pos);
+  V8_EXPORT_PRIVATE static bool MatchAnyChar(InputPosition* pos);
+  V8_EXPORT_PRIVATE static bool MatchString(const char* s, InputPosition* pos);
 
   // The action MatchInput() produces the input matched by the rule as
   // result.

src/torque/implementation-visitor.cc

@@ -7,8 +7,6 @@
 #include "src/torque/implementation-visitor.h"
 #include "src/torque/parameter-difference.h"
 
-#include "include/v8.h"
-
 namespace v8 {
 namespace internal {
 namespace torque {

src/torque/torque-parser.cc

@@ -53,86 +53,104 @@ enum class ParseResultHolderBase::TypeId {
 };
 
 template <>
-const ParseResultTypeId ParseResultHolder<std::string>::id =
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<std::string>::id =
     ParseResultTypeId::kStdString;
 template <>
-const ParseResultTypeId ParseResultHolder<bool>::id = ParseResultTypeId::kBool;
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<bool>::id =
+    ParseResultTypeId::kBool;
 template <>
-const ParseResultTypeId ParseResultHolder<std::vector<std::string>>::id =
-    ParseResultTypeId::kStdVectorOfString;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<std::vector<std::string>>::id =
+        ParseResultTypeId::kStdVectorOfString;
 template <>
-const ParseResultTypeId ParseResultHolder<Declaration*>::id =
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<Declaration*>::id =
     ParseResultTypeId::kDeclarationPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<TypeExpression*>::id =
-    ParseResultTypeId::kTypeExpressionPtr;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<TypeExpression*>::id =
+        ParseResultTypeId::kTypeExpressionPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<LabelBlock*>::id =
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<LabelBlock*>::id =
     ParseResultTypeId::kLabelBlockPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<Expression*>::id =
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<Expression*>::id =
     ParseResultTypeId::kExpressionPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<LocationExpression*>::id =
-    ParseResultTypeId::kLocationExpressionPtr;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<LocationExpression*>::id =
+        ParseResultTypeId::kLocationExpressionPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<Statement*>::id =
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<Statement*>::id =
     ParseResultTypeId::kStatementPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<NameAndTypeExpression>::id =
-    ParseResultTypeId::kNameAndTypeExpression;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<NameAndTypeExpression>::id =
+        ParseResultTypeId::kNameAndTypeExpression;
 template <>
-const ParseResultTypeId
+V8_EXPORT_PRIVATE const ParseResultTypeId
     ParseResultHolder<std::vector<NameAndTypeExpression>>::id =
         ParseResultTypeId::kStdVectorOfNameAndTypeExpression;
 template <>
-const ParseResultTypeId ParseResultHolder<IncrementDecrementOperator>::id =
-    ParseResultTypeId::kIncrementDecrementOperator;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<IncrementDecrementOperator>::id =
+        ParseResultTypeId::kIncrementDecrementOperator;
 template <>
-const ParseResultTypeId ParseResultHolder<base::Optional<std::string>>::id =
-    ParseResultTypeId::kOptionalStdString;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<base::Optional<std::string>>::id =
+        ParseResultTypeId::kOptionalStdString;
 template <>
-const ParseResultTypeId ParseResultHolder<std::vector<Statement*>>::id =
-    ParseResultTypeId::kStdVectorOfStatementPtr;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<std::vector<Statement*>>::id =
+        ParseResultTypeId::kStdVectorOfStatementPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<std::vector<Declaration*>>::id =
-    ParseResultTypeId::kStdVectorOfDeclarationPtr;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<std::vector<Declaration*>>::id =
+        ParseResultTypeId::kStdVectorOfDeclarationPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<std::vector<Expression*>>::id =
-    ParseResultTypeId::kStdVectorOfExpressionPtr;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<std::vector<Expression*>>::id =
+        ParseResultTypeId::kStdVectorOfExpressionPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<ExpressionWithSource>::id =
-    ParseResultTypeId::kExpressionWithSource;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<ExpressionWithSource>::id =
+        ParseResultTypeId::kExpressionWithSource;
 template <>
-const ParseResultTypeId ParseResultHolder<ParameterList>::id =
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<ParameterList>::id =
     ParseResultTypeId::kParameterList;
 template <>
-const ParseResultTypeId ParseResultHolder<RangeExpression>::id =
-    ParseResultTypeId::kRangeExpression;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<RangeExpression>::id =
+        ParseResultTypeId::kRangeExpression;
 template <>
-const ParseResultTypeId ParseResultHolder<base::Optional<RangeExpression>>::id =
-    ParseResultTypeId::kOptionalRangeExpression;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<base::Optional<RangeExpression>>::id =
+        ParseResultTypeId::kOptionalRangeExpression;
 template <>
-const ParseResultTypeId ParseResultHolder<TypeList>::id =
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<TypeList>::id =
     ParseResultTypeId::kTypeList;
 template <>
-const ParseResultTypeId ParseResultHolder<base::Optional<TypeList>>::id =
-    ParseResultTypeId::kOptionalTypeList;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<base::Optional<TypeList>>::id =
+        ParseResultTypeId::kOptionalTypeList;
 template <>
-const ParseResultTypeId ParseResultHolder<LabelAndTypes>::id =
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<LabelAndTypes>::id =
     ParseResultTypeId::kLabelAndTypes;
 template <>
-const ParseResultTypeId ParseResultHolder<std::vector<LabelAndTypes>>::id =
-    ParseResultTypeId::kStdVectorOfLabelAndTypes;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<std::vector<LabelAndTypes>>::id =
+        ParseResultTypeId::kStdVectorOfLabelAndTypes;
 template <>
-const ParseResultTypeId ParseResultHolder<std::vector<LabelBlock*>>::id =
-    ParseResultTypeId::kStdVectorOfLabelBlockPtr;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<std::vector<LabelBlock*>>::id =
+        ParseResultTypeId::kStdVectorOfLabelBlockPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<base::Optional<Statement*>>::id =
-    ParseResultTypeId::kOptionalStatementPtr;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<base::Optional<Statement*>>::id =
+        ParseResultTypeId::kOptionalStatementPtr;
 template <>
-const ParseResultTypeId ParseResultHolder<base::Optional<Expression*>>::id =
-    ParseResultTypeId::kOptionalExpressionPtr;
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<base::Optional<Expression*>>::id =
+        ParseResultTypeId::kOptionalExpressionPtr;
 
 namespace {
 

src/torque/torque.cc

@@ -19,8 +19,6 @@ namespace v8 {
 namespace internal {
 namespace torque {
 
-size_t Label::next_id_ = 0;
-
 int WrappedMain(int argc, const char** argv) {
   std::string output_directory;
   bool verbose = false;

test/unittests/BUILD.gn

@@ -191,6 +191,7 @@ v8_source_set("unittests_sources") {
     "test-helpers.h",
     "test-utils.cc",
     "test-utils.h",
+    "torque/earley-parser-unittest.cc",
     "unicode-unittest.cc",
     "utils-unittest.cc",
     "value-serializer-unittest.cc",

test/unittests/torque/earley-parser-unittest.cc

+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/torque/earley-parser.h"
+#include "test/unittests/test-utils.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+
+namespace {
+
+template <int op(int, int)>
+base::Optional<ParseResult> MakeBinop(ParseResultIterator* child_results) {
+  // Ideally, we would want to use int as a result type here instead of
+  // std::string. This is possible, but requires adding int to the list of
+  // supported ParseResult types in torque-parser.cc. To avoid changing that
+  // code, we use std::string here, which is already used in the Torque parser.
+  auto a = child_results->NextAs<std::string>();
+  auto b = child_results->NextAs<std::string>();
+  return ParseResult{std::to_string(op(std::stoi(a), std::stoi(b)))};
+}
+
+int plus(int a, int b) { return a + b; }
+int minus(int a, int b) { return a - b; }
+int mul(int a, int b) { return a * b; }
+
+}  // namespace
+
+struct SimpleArithmeticGrammar : Grammar {
+  static bool MatchWhitespace(InputPosition* pos) {
+    while (MatchChar(std::isspace, pos)) {
+    }
+    return true;
+  }
+
+  static bool MatchInteger(InputPosition* pos) {
+    InputPosition current = *pos;
+    MatchString("-", &current);
+    if (MatchChar(std::isdigit, &current)) {
+      while (MatchChar(std::isdigit, &current)) {
+      }
+      *pos = current;
+      return true;
+    }
+    return false;
+  }
+
+  SimpleArithmeticGrammar() : Grammar(&sum_expression) {
+    SetWhitespace(MatchWhitespace);
+  }
+
+  Symbol integer = {Rule({Pattern(MatchInteger)}, YieldMatchedInput)};
+
+  Symbol atomic_expression = {Rule({&integer}),
+                              Rule({Token("("), &sum_expression, Token(")")})};
+
+  Symbol mul_expression = {
+      Rule({&atomic_expression}),
+      Rule({&mul_expression, Token("*"), &atomic_expression}, MakeBinop<mul>)};
+
+  Symbol sum_expression = {
+      Rule({&mul_expression}),
+      Rule({&sum_expression, Token("+"), &mul_expression}, MakeBinop<plus>),
+      Rule({&sum_expression, Token("-"), &mul_expression}, MakeBinop<minus>)};
+};
+
+TEST(EarleyParser, SimpleArithmetic) {
+  SimpleArithmeticGrammar grammar;
+  SourceFileMap::Scope source_file_map;
+  CurrentSourceFile::Scope current_source_file{
+      SourceFileMap::AddSource("dummy_filename")};
+  std::string result1 =
+      grammar.Parse("-5 - 5 + (3 + 5) * 2")->Cast<std::string>();
+  ASSERT_EQ("6", result1);
+  std::string result2 = grammar.Parse("((-1 + (1) * 2 + 3 - 4 * 5 + -6 * 7))")
+                            ->Cast<std::string>();
+  ASSERT_EQ("-58", result2);
+}
+
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8