[torque-ls] Implement basic Json parser for the language server

This CL contains a basic Json parser used to read and write the
Json-RPC messages of the language server protocol.

This CL is part of the initial language server implementation but
submitted separately for easier review.

R=tebbi@chromium.org

Bug: v8:8880
Change-Id: Icea040975e1ed1d587954c3342d8d876e01c26b8
Reviewed-on: https://chromium-review.googlesource.com/c/1479956
Commit-Queue: Simon Zünd <szuend@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59848}

BUILD.gn

@@ -3145,6 +3145,24 @@ v8_source_set("torque_base") {
   }
 }
 
+v8_source_set("torque_ls_base") {
+  sources = [
+    "src/torque/ls/json-parser.cc",
+    "src/torque/ls/json-parser.h",
+    "src/torque/ls/json.cc",
+    "src/torque/ls/json.h",
+  ]
+
+  deps = [
+    ":torque_base",
+  ]
+
+  configs = [ ":internal_config" ]
+  if (is_win && is_asan) {
+    remove_configs = [ "//build/config/sanitizers:default_sanitizer_flags" ]
+  }
+}
+
 v8_component("v8_libbase") {
   sources = [
     "src/base/adapters.h",

src/torque/earley-parser.h

@@ -39,6 +39,49 @@ class ParseResultHolderBase {
   const TypeId type_id_;
 };
 
+enum class ParseResultHolderBase::TypeId {
+  kStdString,
+  kBool,
+  kStdVectorOfString,
+  kExpressionPtr,
+  kLocationExpressionPtr,
+  kStatementPtr,
+  kDeclarationPtr,
+  kTypeExpressionPtr,
+  kOptionalTypeExpressionPtr,
+  kLabelBlockPtr,
+  kOptionalLabelBlockPtr,
+  kNameAndTypeExpression,
+  kClassFieldExpression,
+  kStructFieldExpression,
+  kStdVectorOfNameAndTypeExpression,
+  kStdVectorOfClassFieldExpression,
+  kStdVectorOfStructFieldExpression,
+  kIncrementDecrementOperator,
+  kOptionalStdString,
+  kStdVectorOfStatementPtr,
+  kStdVectorOfDeclarationPtr,
+  kStdVectorOfExpressionPtr,
+  kExpressionWithSource,
+  kParameterList,
+  kRangeExpression,
+  kOptionalRangeExpression,
+  kTypeList,
+  kOptionalTypeList,
+  kLabelAndTypes,
+  kStdVectorOfLabelAndTypes,
+  kStdVectorOfLabelBlockPtr,
+  kOptionalStatementPtr,
+  kOptionalExpressionPtr,
+  kTypeswitchCase,
+  kStdVectorOfTypeswitchCase,
+
+  kJsonValue,
+  kJsonMember,
+  kStdVectorOfJsonValue,
+  kStdVectorOfJsonMember,
+};
+
 using ParseResultTypeId = ParseResultHolderBase::TypeId;
 
 template <class T>
@@ -71,13 +114,17 @@ class ParseResult {
   explicit ParseResult(T x) : value_(new ParseResultHolder<T>(std::move(x))) {}
 
   template <class T>
-  const T& Cast() const {
+  const T& Cast() const& {
     return value_->Cast<T>();
   }
   template <class T>
-  T& Cast() {
+  T& Cast() & {
     return value_->Cast<T>();
   }
+  template <class T>
+  T&& Cast() && {
+    return std::move(value_->Cast<T>());
+  }
 
  private:
   std::unique_ptr<ParseResultHolderBase> value_;

src/torque/ls/json-parser.cc

+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/torque/ls/json-parser.h"
+
+#include <cctype>
+#include "src/torque/earley-parser.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+
+template <>
+V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<ls::JsonValue>::id =
+    ParseResultTypeId::kJsonValue;
+
+template <>
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<std::pair<std::string, ls::JsonValue>>::id =
+        ParseResultTypeId::kJsonMember;
+
+template <>
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<std::vector<ls::JsonValue>>::id =
+        ParseResultTypeId::kStdVectorOfJsonValue;
+
+template <>
+V8_EXPORT_PRIVATE const ParseResultTypeId
+    ParseResultHolder<std::vector<std::pair<std::string, ls::JsonValue>>>::id =
+        ParseResultTypeId::kStdVectorOfJsonMember;
+
+namespace ls {
+
+using JsonMember = std::pair<std::string, JsonValue>;
+
+template <bool value>
+base::Optional<ParseResult> MakeBoolLiteral(
+    ParseResultIterator* child_results) {
+  return ParseResult{From(value)};
+}
+
+base::Optional<ParseResult> MakeNullLiteral(
+    ParseResultIterator* child_results) {
+  JsonValue result;
+  result.tag = JsonValue::IS_NULL;
+  return ParseResult{std::move(result)};
+}
+
+base::Optional<ParseResult> MakeNumberLiteral(
+    ParseResultIterator* child_results) {
+  auto number = child_results->NextAs<std::string>();
+  double d = std::stod(number.c_str());
+  return ParseResult{From(d)};
+}
+
+base::Optional<ParseResult> MakeStringLiteral(
+    ParseResultIterator* child_results) {
+  std::string literal = child_results->NextAs<std::string>();
+  return ParseResult{From(StringLiteralUnquote(literal))};
+}
+
+base::Optional<ParseResult> MakeArray(ParseResultIterator* child_results) {
+  JsonArray array = child_results->NextAs<JsonArray>();
+  return ParseResult{From(std::move(array))};
+}
+
+base::Optional<ParseResult> MakeMember(ParseResultIterator* child_results) {
+  JsonMember result;
+  std::string key = child_results->NextAs<std::string>();
+  result.first = StringLiteralUnquote(key);
+  result.second = child_results->NextAs<JsonValue>();
+  return ParseResult{std::move(result)};
+}
+
+base::Optional<ParseResult> MakeObject(ParseResultIterator* child_results) {
+  using MemberList = std::vector<JsonMember>;
+  MemberList members = child_results->NextAs<MemberList>();
+
+  JsonObject object;
+  for (auto& member : members) object.insert(std::move(member));
+
+  return ParseResult{From(std::move(object))};
+}
+
+class JsonGrammar : public Grammar {
+  static bool MatchWhitespace(InputPosition* pos) {
+    while (MatchChar(std::isspace, pos)) {
+    }
+    return true;
+  }
+
+  static bool MatchStringLiteral(InputPosition* pos) {
+    InputPosition current = *pos;
+    if (MatchString("\"", &current)) {
+      while (
+          (MatchString("\\", &current) && MatchAnyChar(&current)) ||
+          MatchChar([](char c) { return c != '"' && c != '\n'; }, &current)) {
+      }
+      if (MatchString("\"", &current)) {
+        *pos = current;
+        return true;
+      }
+    }
+    current = *pos;
+    if (MatchString("'", &current)) {
+      while (
+          (MatchString("\\", &current) && MatchAnyChar(&current)) ||
+          MatchChar([](char c) { return c != '\'' && c != '\n'; }, &current)) {
+      }
+      if (MatchString("'", &current)) {
+        *pos = current;
+        return true;
+      }
+    }
+    return false;
+  }
+
+  static bool MatchHexLiteral(InputPosition* pos) {
+    InputPosition current = *pos;
+    MatchString("-", &current);
+    if (MatchString("0x", &current) && MatchChar(std::isxdigit, &current)) {
+      while (MatchChar(std::isxdigit, &current)) {
+      }
+      *pos = current;
+      return true;
+    }
+    return false;
+  }
+
+  static bool MatchDecimalLiteral(InputPosition* pos) {
+    InputPosition current = *pos;
+    bool found_digit = false;
+    MatchString("-", &current);
+    while (MatchChar(std::isdigit, &current)) found_digit = true;
+    MatchString(".", &current);
+    while (MatchChar(std::isdigit, &current)) found_digit = true;
+    if (!found_digit) return false;
+    *pos = current;
+    if ((MatchString("e", &current) || MatchString("E", &current)) &&
+        (MatchString("+", &current) || MatchString("-", &current) || true) &&
+        MatchChar(std::isdigit, &current)) {
+      while (MatchChar(std::isdigit, &current)) {
+      }
+      *pos = current;
+      return true;
+    }
+    return true;
+  }
+
+ public:
+  JsonGrammar() : Grammar(&file) { SetWhitespace(MatchWhitespace); }
+
+  Symbol trueLiteral = {Rule({Token("true")})};
+  Symbol falseLiteral = {Rule({Token("false")})};
+  Symbol nullLiteral = {Rule({Token("null")})};
+
+  Symbol decimalLiteral = {
+      Rule({Pattern(MatchDecimalLiteral)}, YieldMatchedInput),
+      Rule({Pattern(MatchHexLiteral)}, YieldMatchedInput)};
+
+  Symbol stringLiteral = {
+      Rule({Pattern(MatchStringLiteral)}, YieldMatchedInput)};
+
+  Symbol* elementList = List<JsonValue>(&value, Token(","));
+  Symbol array = {Rule({Token("["), elementList, Token("]")})};
+
+  Symbol member = {Rule({&stringLiteral, Token(":"), &value}, MakeMember)};
+  Symbol* memberList = List<JsonMember>(&member, Token(","));
+  Symbol object = {Rule({Token("{"), memberList, Token("}")})};
+
+  Symbol value = {Rule({&trueLiteral}, MakeBoolLiteral<true>),
+                  Rule({&falseLiteral}, MakeBoolLiteral<false>),
+                  Rule({&nullLiteral}, MakeNullLiteral),
+                  Rule({&decimalLiteral}, MakeNumberLiteral),
+                  Rule({&stringLiteral}, MakeStringLiteral),
+                  Rule({&object}, MakeObject),
+                  Rule({&array}, MakeArray)};
+
+  Symbol file = {Rule({&value})};
+};
+
+JsonValue ParseJson(const std::string& input) {
+  // Torque needs a CurrentSourceFile scope during parsing.
+  // As JSON lives in memory only, a unknown file scope is created.
+  CurrentSourceFile::Scope unkown_file(SourceId::Invalid());
+
+  return (*JsonGrammar().Parse(input)).Cast<JsonValue>();
+}
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8

src/torque/ls/json-parser.h

+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_TORQUE_LS_JSON_PARSER_H_
+#define V8_TORQUE_LS_JSON_PARSER_H_
+
+#include "src/torque/ls/json.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+JsonValue ParseJson(const std::string& input);
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_TORQUE_LS_JSON_PARSER_H_

src/torque/ls/json.cc

+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/torque/ls/json.h"
+
+#include <iostream>
+#include <sstream>
+#include "src/torque/utils.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+namespace {
+
+void SerializeToString(std::stringstream& str, const JsonValue& value) {
+  switch (value.tag) {
+    case JsonValue::NUMBER:
+      str << value.number;
+      break;
+    case JsonValue::STRING:
+      str << StringLiteralQuote(value.string);
+      break;
+    case JsonValue::IS_NULL:
+      str << "\"null\"";
+      break;
+    case JsonValue::BOOL:
+      str << (value.flag ? "\"true\"" : "\"false\"");
+      break;
+    case JsonValue::OBJECT: {
+      str << "{";
+      size_t i = 0;
+      for (const auto& pair : *value.object) {
+        str << "\"" << pair.first << "\":";
+        SerializeToString(str, pair.second);
+        if (++i < value.object->size()) str << ",";
+      }
+      str << "}";
+      break;
+    }
+    case JsonValue::ARRAY: {
+      str << "[";
+      size_t i = 0;
+      for (const auto& element : *value.array) {
+        SerializeToString(str, element);
+        if (++i < value.array->size()) str << ",";
+      }
+      str << "]";
+      break;
+    }
+    default:
+      break;
+  }
+}
+
+}  // namespace
+
+std::string SerializeToString(const JsonValue& value) {
+  std::stringstream result;
+  SerializeToString(result, value);
+  return result.str();
+}
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8

src/torque/ls/json.h

+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_TORQUE_LS_JSON_H_
+#define V8_TORQUE_LS_JSON_H_
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "src/base/template-utils.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+struct JsonValue;
+
+using JsonObject = std::map<std::string, JsonValue>;
+using JsonArray = std::vector<JsonValue>;
+
+struct JsonValue {
+  enum { OBJECT, ARRAY, STRING, NUMBER, BOOL, IS_NULL } tag;
+  std::unique_ptr<JsonObject> object;
+  std::unique_ptr<JsonArray> array;
+  std::string string;
+  double number = 0;
+  bool flag = false;
+};
+
+inline JsonValue From(JsonObject object) {
+  JsonValue result;
+  result.tag = JsonValue::OBJECT;
+  result.object = base::make_unique<JsonObject>(std::move(object));
+  return result;
+}
+
+inline JsonValue From(const std::string& string) {
+  JsonValue result;
+  result.tag = JsonValue::STRING;
+  result.string = string;
+  return result;
+}
+
+inline JsonValue From(double number) {
+  JsonValue result;
+  result.tag = JsonValue::NUMBER;
+  result.number = number;
+  return result;
+}
+
+inline JsonValue From(bool b) {
+  JsonValue result;
+  result.tag = JsonValue::BOOL;
+  result.flag = b;
+  return result;
+}
+
+inline JsonValue From(JsonArray array) {
+  JsonValue result;
+  result.tag = JsonValue::ARRAY;
+  result.array = base::make_unique<JsonArray>(std::move(array));
+  return result;
+}
+
+std::string SerializeToString(const JsonValue& value);
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_TORQUE_LS_JSON_H_

src/torque/torque-parser.cc

@@ -29,44 +29,6 @@ struct TypeswitchCase {
   Statement* block;
 };
 
-enum class ParseResultHolderBase::TypeId {
-  kStdString,
-  kBool,
-  kStdVectorOfString,
-  kExpressionPtr,
-  kLocationExpressionPtr,
-  kStatementPtr,
-  kDeclarationPtr,
-  kTypeExpressionPtr,
-  kOptionalTypeExpressionPtr,
-  kLabelBlockPtr,
-  kOptionalLabelBlockPtr,
-  kNameAndTypeExpression,
-  kClassFieldExpression,
-  kStructFieldExpression,
-  kStdVectorOfNameAndTypeExpression,
-  kStdVectorOfClassFieldExpression,
-  kStdVectorOfStructFieldExpression,
-  kIncrementDecrementOperator,
-  kOptionalStdString,
-  kStdVectorOfStatementPtr,
-  kStdVectorOfDeclarationPtr,
-  kStdVectorOfExpressionPtr,
-  kExpressionWithSource,
-  kParameterList,
-  kRangeExpression,
-  kOptionalRangeExpression,
-  kTypeList,
-  kOptionalTypeList,
-  kLabelAndTypes,
-  kStdVectorOfLabelAndTypes,
-  kStdVectorOfLabelBlockPtr,
-  kOptionalStatementPtr,
-  kOptionalExpressionPtr,
-  kTypeswitchCase,
-  kStdVectorOfTypeswitchCase
-};
-
 template <>
 V8_EXPORT_PRIVATE const ParseResultTypeId ParseResultHolder<std::string>::id =
     ParseResultTypeId::kStdString;

test/cctest/BUILD.gn

@@ -250,6 +250,7 @@ v8_source_set("cctest_sources") {
     "test-version.cc",
     "test-weakmaps.cc",
     "test-weaksets.cc",
+    "torque/test-torque-ls-json.cc",
     "torque/test-torque.cc",
     "trace-extension.cc",
     "trace-extension.h",
@@ -371,6 +372,7 @@ v8_source_set("cctest_sources") {
     ":cctest_headers",
     ":resources",
     "..:common_test_headers",
+    "../..:torque_ls_base",
     "../..:v8_initializers",
     "../..:v8_libbase",
     "../..:v8_libplatform",

test/cctest/torque/test-torque-ls-json.cc

+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/torque/ls/json-parser.h"
+#include "src/torque/ls/json.h"
+#include "test/cctest/cctest.h"
+
+namespace v8 {
+namespace internal {
+namespace torque {
+namespace ls {
+
+TEST(TestJsonPrimitives) {
+  const JsonValue true_result = ParseJson("true");
+  CHECK_EQ(true_result.tag, JsonValue::BOOL);
+  CHECK_EQ(true_result.flag, true);
+
+  const JsonValue false_result = ParseJson("false");
+  CHECK_EQ(false_result.tag, JsonValue::BOOL);
+  CHECK_EQ(false_result.flag, false);
+
+  const JsonValue null_result = ParseJson("null");
+  CHECK_EQ(null_result.tag, JsonValue::IS_NULL);
+
+  const JsonValue number = ParseJson("42");
+  CHECK_EQ(number.tag, JsonValue::NUMBER);
+  CHECK_EQ(number.number, 42);
+}
+
+TEST(TestJsonStrings) {
+  const JsonValue basic = ParseJson("\"basic\"");
+  CHECK_EQ(basic.tag, JsonValue::STRING);
+  CHECK_EQ(basic.string, "basic");
+
+  const JsonValue singleQuote = ParseJson("\"'\"");
+  CHECK_EQ(singleQuote.tag, JsonValue::STRING);
+  CHECK_EQ(singleQuote.string, "'");
+}
+
+TEST(TestJsonArrays) {
+  const JsonValue empty_array = ParseJson("[]");
+  CHECK_EQ(empty_array.tag, JsonValue::ARRAY);
+  CHECK_EQ(empty_array.array->size(), 0);
+
+  const JsonValue number_array = ParseJson("[1, 2, 3, 4]");
+  CHECK_EQ(number_array.tag, JsonValue::ARRAY);
+
+  const JsonArray& array = *number_array.array;
+  CHECK_EQ(array.size(), 4);
+  CHECK_EQ(array[1].tag, JsonValue::NUMBER);
+  CHECK_EQ(array[1].number, 2);
+
+  const JsonValue string_array_object = ParseJson("[\"a\", \"b\"]");
+  CHECK_EQ(string_array_object.tag, JsonValue::ARRAY);
+
+  const JsonArray& string_array = *string_array_object.array;
+  CHECK_EQ(string_array.size(), 2);
+  CHECK_EQ(string_array[1].tag, JsonValue::STRING);
+  CHECK_EQ(string_array[1].string, "b");
+}
+
+TEST(TestJsonObjects) {
+  const JsonValue empty_object = ParseJson("{}");
+  CHECK_EQ(empty_object.tag, JsonValue::OBJECT);
+  CHECK_EQ(empty_object.object->size(), 0);
+
+  const JsonValue primitive_fields = ParseJson("{ \"flag\": true, \"id\": 5}");
+  CHECK_EQ(primitive_fields.tag, JsonValue::OBJECT);
+
+  const JsonValue& flag = primitive_fields.object->at("flag");
+  CHECK_EQ(flag.tag, JsonValue::BOOL);
+  CHECK(flag.flag);
+
+  const JsonValue& id = primitive_fields.object->at("id");
+  CHECK_EQ(id.tag, JsonValue::NUMBER);
+  CHECK_EQ(id.number, 5);
+
+  const JsonValue& complex_fields =
+      ParseJson("{ \"array\": [], \"object\": { \"name\": \"torque\" } }");
+  CHECK_EQ(complex_fields.tag, JsonValue::OBJECT);
+
+  const JsonValue& array = complex_fields.object->at("array");
+  CHECK_EQ(array.tag, JsonValue::ARRAY);
+  CHECK_EQ(array.array->size(), 0);
+
+  const JsonValue& object = complex_fields.object->at("object");
+  CHECK_EQ(object.tag, JsonValue::OBJECT);
+  CHECK_EQ(object.object->at("name").tag, JsonValue::STRING);
+  CHECK_EQ(object.object->at("name").string, "torque");
+}
+
+}  // namespace ls
+}  // namespace torque
+}  // namespace internal
+}  // namespace v8