Reland proposal-numeric-separator.

Revert "Revert "[parser] Implements proposal-numeric-separator.""

This reverts commit 782f6401.

Original CL is https://chromium-review.googlesource.com/c/v8/v8/+/923441

Bug: v8:7317
Change-Id: I6f541c038bad0cff625094ba84aebe582bdeb12f
Reviewed-on: https://chromium-review.googlesource.com/945034Reviewed-by: Sathya Gunasekaran <gsathya@chromium.org>
Commit-Queue: Sathya Gunasekaran <gsathya@chromium.org>
Cr-Commit-Position: refs/heads/master@{#51749}

src/bootstrapper.cc

@@ -4157,6 +4157,7 @@ EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_import_meta)
 EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_restrict_constructor_return)
 EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_optional_catch_binding)
 EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_subsume_json)
+EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_numeric_separator)
 
 #undef EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE
 

src/char-predicates-inl.h

@@ -48,25 +48,21 @@ inline bool IsDecimalDigit(uc32 c) {
   return IsInRange(c, '0', '9');
 }
 
-
 inline bool IsHexDigit(uc32 c) {
   // ECMA-262, 3rd, 7.6 (p 15)
   return IsDecimalDigit(c) || IsInRange(AsciiAlphaToLower(c), 'a', 'f');
 }
 
-
 inline bool IsOctalDigit(uc32 c) {
   // ECMA-262, 6th, 7.8.3
   return IsInRange(c, '0', '7');
 }
 
-
 inline bool IsBinaryDigit(uc32 c) {
   // ECMA-262, 6th, 7.8.3
   return c == '0' || c == '1';
 }
 
-
 inline bool IsRegExpWord(uc16 c) {
   return IsInRange(AsciiAlphaToLower(c), 'a', 'z')
       || IsDecimalDigit(c)

src/flag-definitions.h

@@ -212,7 +212,8 @@ DEFINE_IMPLICATION(harmony_class_fields, harmony_private_fields)
   V(harmony_array_prototype_values, "harmony Array.prototype.values") \
   V(harmony_do_expressions, "harmony do-expressions")                 \
   V(harmony_class_fields, "harmony fields in class literals")         \
-  V(harmony_static_fields, "harmony static fields in class literals")
+  V(harmony_static_fields, "harmony static fields in class literals") \
+  V(harmony_numeric_separator, "harmony numeric separator between digits")
 
 // Features that are complete (but still behind --harmony/es-staging flag).
 #define HARMONY_STAGED(V)                                               \

src/messages.h

@@ -557,6 +557,10 @@ class ErrorUtils : public AllStatic {
   T(LocaleMatcher, "Illegal value for localeMatcher:%")                        \
   T(NormalizationForm, "The normalization form should be one of %.")           \
   T(NumberFormatRange, "% argument must be between 0 and 100")                 \
+  T(TrailingNumericSeparator,                                                  \
+    "Numeric separators are not allowed at the end of numeric literals")       \
+  T(ContinuousNumericSeparator,                                                \
+    "Only one underscore is allowed as numeric separator")                     \
   T(PropertyValueOutOfRange, "% value is out of range.")                       \
   T(StackOverflow, "Maximum call stack size exceeded")                         \
   T(ToPrecisionFormatRange,                                                    \

src/parsing/parser-base.h

@@ -307,6 +307,12 @@ class ParserBase {
   void set_allow_harmony_bigint(bool allow) {
     scanner()->set_allow_harmony_bigint(allow);
   }
+  bool allow_harmony_numeric_separator() const {
+    return scanner()->allow_harmony_numeric_separator();
+  }
+  void set_allow_harmony_numeric_separator(bool allow) {
+    scanner()->set_allow_harmony_numeric_separator(allow);
+  }
 
   bool allow_harmony_private_fields() const {
     return scanner()->allow_harmony_private_fields();

src/parsing/parser.cc

@@ -448,6 +448,7 @@ Parser::Parser(ParseInfo* info)
   set_allow_harmony_dynamic_import(FLAG_harmony_dynamic_import);
   set_allow_harmony_import_meta(FLAG_harmony_import_meta);
   set_allow_harmony_bigint(FLAG_harmony_bigint);
+  set_allow_harmony_numeric_separator(FLAG_harmony_numeric_separator);
   set_allow_harmony_optional_catch_binding(FLAG_harmony_optional_catch_binding);
   set_allow_harmony_private_fields(FLAG_harmony_private_fields);
   for (int feature = 0; feature < v8::Isolate::kUseCounterFeatureCount;

src/parsing/scanner.h

@@ -366,6 +366,12 @@ class Scanner {
   void set_allow_harmony_private_fields(bool allow) {
     allow_harmony_private_fields_ = allow;
   }
+  bool allow_harmony_numeric_separator() const {
+    return allow_harmony_numeric_separator_;
+  }
+  void set_allow_harmony_numeric_separator(bool allow) {
+    allow_harmony_numeric_separator_ = allow;
+  }
 
  private:
   // Scoped helper for saving & restoring scanner error state.
@@ -490,6 +496,15 @@ class Scanner {
     Token::Value contextual_token;
   };
 
+  enum NumberKind {
+    BINARY,
+    OCTAL,
+    IMPLICIT_OCTAL,
+    HEX,
+    DECIMAL,
+    DECIMAL_WITH_LEADING_ZERO
+  };
+
   static const int kCharacterLookaheadBufferSize = 1;
   const int kMaxAscii = 127;
 
@@ -720,12 +735,20 @@ class Scanner {
   // Scans a possible HTML comment -- begins with '<!'.
   Token::Value ScanHtmlComment();
 
-  void ScanDecimalDigits();
-  bool ScanHexDigits();
-  bool ScanBinaryDigits();
-  bool ScanSignedInteger();
-  bool ScanOctalDigits();
-  bool ScanImplicitOctalDigits(int start_pos);
+  bool ScanDigitsWithNumericSeparators(bool (*predicate)(uc32 ch),
+                                       int start_pos,
+                                       bool is_check_first_digit);
+  bool ScanDecimalDigits(int start_pos);
+  // Optimized function to scan decimal number as Smi.
+  bool ScanDecimalAsSmi(int start_pos, uint64_t* value);
+  bool ScanDecimalAsSmiWithNumericSeparators(int start_pos, uint64_t* value);
+  bool ScanHexDigits(int start_pos);
+  bool ScanBinaryDigits(int start_pos);
+  bool ScanSignedInteger(int start_pos);
+  bool ScanOctalDigits(int start_pos);
+  bool ScanImplicitOctalDigits(int start_pos, NumberKind* kind);
+  bool ScanImplicitOctalDigitsWithNumericSeparators(int start_pos,
+                                                    NumberKind* kind);
 
   Token::Value ScanNumber(bool seen_period);
   Token::Value ScanIdentifierOrKeyword();
@@ -817,6 +840,7 @@ class Scanner {
   // Harmony flags to allow ESNext features.
   bool allow_harmony_bigint_;
   bool allow_harmony_private_fields_;
+  bool allow_harmony_numeric_separator_;
 
   MessageTemplate::Template scanner_error_;
   Location scanner_error_location_;

test/cctest/test-parsing.cc

@@ -1122,6 +1122,7 @@ enum ParserFlag {
   kAllowHarmonyImportMeta,
   kAllowHarmonyDoExpressions,
   kAllowHarmonyOptionalCatchBinding,
+  kAllowHarmonyNumericSeparator
 };
 
 enum ParserSyncTestResult {
@@ -1140,6 +1141,8 @@ void SetGlobalFlags(i::EnumSet<ParserFlag> flags) {
   i::FLAG_harmony_do_expressions = flags.Contains(kAllowHarmonyDoExpressions);
   i::FLAG_harmony_optional_catch_binding =
       flags.Contains(kAllowHarmonyOptionalCatchBinding);
+  i::FLAG_harmony_numeric_separator =
+      flags.Contains(kAllowHarmonyNumericSeparator);
 }
 
 void SetParserFlags(i::PreParser* parser, i::EnumSet<ParserFlag> flags) {
@@ -1158,6 +1161,8 @@ void SetParserFlags(i::PreParser* parser, i::EnumSet<ParserFlag> flags) {
       flags.Contains(kAllowHarmonyDoExpressions));
   parser->set_allow_harmony_optional_catch_binding(
       flags.Contains(kAllowHarmonyOptionalCatchBinding));
+  parser->set_allow_harmony_numeric_separator(
+      flags.Contains(kAllowHarmonyNumericSeparator));
 }
 
 void TestParserSyncWithFlags(i::Handle<i::String> source,
@@ -1480,6 +1485,91 @@ void RunModuleParserSyncTest(
                     always_false_len, true, test_preparser, ignore_error_msg);
 }
 
+TEST(NumericSeparator) {
+  v8::HandleScope handles(CcTest::isolate());
+  v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
+  v8::Context::Scope context_scope(context);
+
+  const char* context_data[][2] = {
+      {"", ""}, {"\"use strict\";", ""}, {nullptr, nullptr}};
+  const char* statement_data[] = {
+      "1_0_0_0", "1_0e+1",  "1_0e+1_0", "0xF_F_FF", "0o7_7_7", "0b0_1_0_1_0",
+      ".3_2_1",  "0.0_2_1", "1_0.0_1",  ".0_1_2",   nullptr};
+
+  static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
+  RunParserSyncTest(context_data, statement_data, kSuccess, nullptr, 0, flags,
+                    1);
+
+  RunParserSyncTest(context_data, statement_data, kError);
+}
+
+TEST(NumericSeparatorErrors) {
+  v8::HandleScope handles(CcTest::isolate());
+  v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
+  v8::Context::Scope context_scope(context);
+
+  const char* context_data[][2] = {
+      {"", ""}, {"\"use strict\";", ""}, {nullptr, nullptr}};
+  const char* statement_data[] = {
+      "1_0_0_0_", "1e_1",    "1e+_1", "1_e+1",  "1__0",    "0x_1",
+      "0x1__1",   "0x1_",    "0_x1",  "0_x_1",  "0b_0101", "0b11_",
+      "0b1__1",   "0_b1",    "0_b_1", "0o777_", "0o_777",  "0o7__77",
+      "0.0_2_1_", "0.0__21", "0_.01", "0._01",  nullptr};
+
+  static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
+  RunParserSyncTest(context_data, statement_data, kError, nullptr, 0, flags, 1,
+                    nullptr, 0, false, true, true);
+
+  RunParserSyncTest(context_data, statement_data, kError);
+}
+
+TEST(NumericSeparatorImplicitOctals) {
+  v8::HandleScope handles(CcTest::isolate());
+  v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
+  v8::Context::Scope context_scope(context);
+
+  const char* context_data[][2] = {
+      {"", ""}, {nullptr, nullptr}, {nullptr, nullptr}};
+  const char* statement_data[] = {"07_7_7", "0_7_7_7", "0_777", nullptr};
+
+  static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
+  RunParserSyncTest(context_data, statement_data, kSuccess, nullptr, 0, flags,
+                    1);
+
+  RunParserSyncTest(context_data, statement_data, kError);
+}
+
+TEST(NumericSeparatorImplicitOctalsErrors) {
+  v8::HandleScope handles(CcTest::isolate());
+  v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
+  v8::Context::Scope context_scope(context);
+
+  const char* context_data[][2] = {
+      {"", ""}, {nullptr, nullptr}, {nullptr, nullptr}};
+  const char* statement_data[] = {"07_7_7_", "07__77", "0__777", nullptr};
+
+  static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
+  RunParserSyncTest(context_data, statement_data, kError, nullptr, 0, flags, 1,
+                    nullptr, 0, false, true, true);
+
+  RunParserSyncTest(context_data, statement_data, kError);
+}
+
+TEST(NumericSeparatorUnicodeEscapeSequencesErrors) {
+  v8::HandleScope handles(CcTest::isolate());
+  v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
+  v8::Context::Scope context_scope(context);
+
+  const char* context_data[][2] = {
+      {"", ""}, {"'use strict'", ""}, {nullptr, nullptr}};
+  // https://github.com/tc39/proposal-numeric-separator/issues/25
+  const char* statement_data[] = {"\\u{10_FFFF}", nullptr};
+
+  static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
+  RunParserSyncTest(context_data, statement_data, kError, nullptr, 0, flags, 1);
+
+  RunParserSyncTest(context_data, statement_data, kError);
+}
 
 TEST(ErrorsEvalAndArguments) {
   // Tests that both preparsing and parsing produce the right kind of errors for

test/mjsunit/harmony/numeric-separator.js

+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --harmony-numeric-separator
+
+{
+  const basic = 1_0_0_0;
+  assertEquals(basic, 1000);
+}
+{
+  const exponent = 1_0e+1;
+  assertEquals(exponent, 10e+1);
+}
+{
+  const exponent2 = 1_0e+1_0;
+  assertEquals(exponent2, 10e+10);
+}
+{
+  const hex = 0xF_F_FF;
+  assertEquals(hex, 0xFFFF);
+}
+{
+  const octal = 0o7_7_7;
+  assertEquals(octal, 0o777);
+}
+{
+  const implicitOctal = 07_7_7;
+  assertEquals(implicitOctal, 0o777);
+}
+
+{
+  let exception = false;
+  try {
+    const code = `"use strict" const implicitOctal = 07_7_7`;
+    eval(code);
+  } catch(e) {
+    exception = true;
+    assertInstanceof(e, SyntaxError);
+  }
+  assertTrue(exception);
+}
+
+{
+  const binary = 0b0_1_0_1_0;
+  assertEquals(binary, 0b01010);
+}
+{
+  const leadingZeros = 09_1_3;
+  assertEquals(leadingZeros, 0913);
+}
+
+assertThrows('1_0_0_0_', SyntaxError);
+assertThrows('1e_1', SyntaxError);
+assertThrows('1e+_1', SyntaxError);
+assertThrows('1_e+1', SyntaxError);
+assertThrows('1__0', SyntaxError);
+assertThrows('0x_1', SyntaxError);
+assertThrows('0x1__1', SyntaxError);
+assertThrows('0x1_', SyntaxError);
+assertThrows('0b_0101', SyntaxError);
+assertThrows('0b11_', SyntaxError);
+assertThrows('0b1__1', SyntaxError);
+assertThrows('0o777_', SyntaxError);
+assertThrows('0o_777', SyntaxError);
+assertThrows('0o7__77', SyntaxError);
+assertThrows('0777_', SyntaxError);
+assertThrows('07__77', SyntaxError);