[esnext] Make ECMAScript a syntactic superset of JSON

This patch makes ECMAScript a syntactic superset of JSON by allowing
U+2028 and U+2029 in string literals.

Proposal repo: https://github.com/tc39/proposal-json-superset

Bug: v8:7418
Change-Id: I7ef4ae6d85854ebc44a66e0eaf789814576832b7
Reviewed-on: https://chromium-review.googlesource.com/921228Reviewed-by: Sathya Gunasekaran <gsathya@chromium.org>
Reviewed-by: Marja Hölttä <marja@chromium.org>
Commit-Queue: Mathias Bynens <mathias@chromium.org>
Cr-Commit-Position: refs/heads/master@{#51313}

src/bootstrapper.cc

@@ -4281,6 +4281,7 @@ EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_dynamic_import)
 EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_import_meta)
 EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_restrict_constructor_return)
 EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_optional_catch_binding)
+EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_subsume_json)
 
 void InstallPublicSymbol(Factory* factory, Handle<Context> native_context,
                          const char* name, Handle<Symbol> value) {

src/flag-definitions.h

@@ -203,8 +203,11 @@ DEFINE_IMPLICATION(harmony_class_fields, harmony_public_fields)
 DEFINE_IMPLICATION(harmony_class_fields, harmony_static_fields)
 DEFINE_IMPLICATION(harmony_class_fields, harmony_private_fields)
 
+// Update bootstrapper.cc whenever adding a new feature flag.
+
 // Features that are still work in progress (behind individual flags).
 #define HARMONY_INPROGRESS(V)                                         \
+  V(harmony_subsume_json, "harmony subsume JSON")                     \
   V(harmony_array_prototype_values, "harmony Array.prototype.values") \
   V(harmony_function_sent, "harmony function.sent")                   \
   V(harmony_do_expressions, "harmony do-expressions")                 \

src/parsing/scanner.cc

@@ -1071,8 +1071,11 @@ Token::Value Scanner::ScanString() {
     AddLiteralChar(c);
   }
 
-  while (c0_ != quote && c0_ != kEndOfInput &&
-         !unibrow::IsLineTerminator(c0_)) {
+  bool (*line_terminator_func)(unsigned int) =
+      FLAG_harmony_subsume_json ? unibrow::IsStringLiteralLineTerminator
+                                : unibrow::IsLineTerminator;
+
+  while (c0_ != quote && c0_ != kEndOfInput && !line_terminator_func(c0_)) {
     uc32 c = c0_;
     Advance();
     if (c == '\\') {

src/unicode.h

@@ -204,6 +204,10 @@ V8_INLINE bool IsLineTerminator(uchar c) {
   return c == 0x000A || c == 0x000D || c == 0x2028 || c == 0x2029;
 }
 
+V8_INLINE bool IsStringLiteralLineTerminator(uchar c) {
+  return c == 0x000A || c == 0x000D;
+}
+
 #ifndef V8_INTL_SUPPORT
 struct ToLowercase {
   static const int kMaxWidth = 3;

test/cctest/test-parsing.cc

@@ -273,7 +273,7 @@ TEST(UsingCachedData) {
   CcTest::i_isolate()->stack_guard()->SetStackLimit(
       i::GetCurrentStackPosition() - 128 * 1024);
 
-  // Source containing functions that might be lazily compiled  and all types
+  // Source containing functions that might be lazily compiled and all types
   // of symbols (string, propertyName, regexp).
   const char* source =
       "var x = 42;"
@@ -1478,7 +1478,10 @@ void TestParserSync(const char* source, const ParserFlag* varying_flags,
                     bool is_module = false, bool test_preparser = true,
                     bool ignore_error_msg = false) {
   i::Handle<i::String> str =
-      CcTest::i_isolate()->factory()->NewStringFromAsciiChecked(source);
+      CcTest::i_isolate()
+          ->factory()
+          ->NewStringFromUtf8(Vector<const char>(source, strlen(source)))
+          .ToHandleChecked();
   for (int bits = 0; bits < (1 << varying_flags_length); bits++) {
     i::EnumSet<ParserFlag> flags;
     for (size_t flag_index = 0; flag_index < varying_flags_length;
@@ -3899,15 +3902,42 @@ TEST(BothModesUseCount) {
 
 TEST(LineOrParagraphSeparatorAsLineTerminator) {
   // Tests that both preparsing and parsing accept U+2028 LINE SEPARATOR and
-  // U+2029 PARAGRAPH SEPARATOR as LineTerminator symbols.
+  // U+2029 PARAGRAPH SEPARATOR as LineTerminator symbols outside of string
+  // literals.
   const char* context_data[][2] = {{"", ""}, {nullptr, nullptr}};
-  const char* statement_data[] = {"\x31\xE2\x80\xA8\x32",  // "1<U+2028>2"
-                                  "\x31\xE2\x80\xA9\x32",  // "1<U+2029>2"
+  const char* statement_data[] = {"\x31\xE2\x80\xA8\x32",  // 1<U+2028>2
+                                  "\x31\xE2\x80\xA9\x32",  // 1<U+2029>2
+                                  nullptr};
+
+  RunParserSyncTest(context_data, statement_data, kSuccess);
+}
+
+TEST(LineOrParagraphSeparatorInStringLiteral) {
+  // Tests that both preparsing and parsing treat U+2028 LINE SEPARATOR and
+  // U+2029 PARAGRAPH SEPARATOR as line terminators within string literals.
+  const char* context_data[][2] = {
+      {"\"", "\""}, {"'", "'"}, {nullptr, nullptr}};
+  const char* statement_data[] = {"\x31\xE2\x80\xA8\x32",  // 1<U+2028>2
+                                  "\x31\xE2\x80\xA9\x32",  // 1<U+2029>2
                                   nullptr};
 
   RunParserSyncTest(context_data, statement_data, kError);
 }
 
+TEST(LineOrParagraphSeparatorInStringLiteralHarmony) {
+  // Tests that both preparsing and parsing don't treat U+2028 LINE SEPARATOR
+  // and U+2029 PARAGRAPH SEPARATOR as line terminators within string literals
+  // with the "subsume JSON" flag enabled.
+  v8::internal::FLAG_harmony_subsume_json = true;
+  const char* context_data[][2] = {
+      {"\"", "\""}, {"'", "'"}, {nullptr, nullptr}};
+  const char* statement_data[] = {"\x31\xE2\x80\xA8\x32",  // 1<U+2028>2
+                                  "\x31\xE2\x80\xA9\x32",  // 1<U+2029>2
+                                  nullptr};
+
+  RunParserSyncTest(context_data, statement_data, kSuccess);
+}
+
 TEST(ErrorsArrowFormalParameters) {
   const char* context_data[][2] = {
     { "()", "=>{}" },