Commit 517df524 authored by Taketoshi Aono's avatar Taketoshi Aono Committed by Commit Bot

[parser] Implements proposal-numeric-separator.

https://github.com/tc39/proposal-numeric-separator

This proposal-numeric-separator extends NumericLiteral and
allows developers to insert underscore(_) inside numeric literal.

Bug: v8:7317
Change-Id: I2a1a45cd6fe09cc5df63433bc915988fde687a33
Reviewed-on: https://chromium-review.googlesource.com/923441
Commit-Queue: Sathya Gunasekaran <gsathya@chromium.org>
Reviewed-by: 's avatarSathya Gunasekaran <gsathya@chromium.org>
Cr-Commit-Position: refs/heads/master@{#51671}
parent 4ab2af5b
......@@ -4214,6 +4214,7 @@ EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_import_meta)
EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_restrict_constructor_return)
EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_optional_catch_binding)
EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_subsume_json)
EMPTY_INITIALIZE_GLOBAL_FOR_FEATURE(harmony_numeric_separator)
void InstallPublicSymbol(Factory* factory, Handle<Context> native_context,
const char* name, Handle<Symbol> value) {
......
......@@ -48,25 +48,21 @@ inline bool IsDecimalDigit(uc32 c) {
return IsInRange(c, '0', '9');
}
inline bool IsHexDigit(uc32 c) {
// ECMA-262, 3rd, 7.6 (p 15)
return IsDecimalDigit(c) || IsInRange(AsciiAlphaToLower(c), 'a', 'f');
}
inline bool IsOctalDigit(uc32 c) {
// ECMA-262, 6th, 7.8.3
return IsInRange(c, '0', '7');
}
inline bool IsBinaryDigit(uc32 c) {
// ECMA-262, 6th, 7.8.3
return c == '0' || c == '1';
}
inline bool IsRegExpWord(uc16 c) {
return IsInRange(AsciiAlphaToLower(c), 'a', 'z')
|| IsDecimalDigit(c)
......
......@@ -213,7 +213,8 @@ DEFINE_IMPLICATION(harmony_class_fields, harmony_private_fields)
V(harmony_do_expressions, "harmony do-expressions") \
V(harmony_class_fields, "harmony fields in class literals") \
V(harmony_static_fields, "harmony static fields in class literals") \
V(harmony_bigint, "harmony arbitrary precision integers")
V(harmony_bigint, "harmony arbitrary precision integers") \
V(harmony_numeric_separator, "harmony numeric separator between digits")
// Features that are complete (but still behind --harmony/es-staging flag).
#define HARMONY_STAGED(V) \
......
......@@ -557,6 +557,10 @@ class ErrorUtils : public AllStatic {
T(LocaleMatcher, "Illegal value for localeMatcher:%") \
T(NormalizationForm, "The normalization form should be one of %.") \
T(NumberFormatRange, "% argument must be between 0 and 100") \
T(TrailingNumericSeparator, \
"Numeric separators are not allowed at the end of numeric literals") \
T(ContinuousNumericSeparator, \
"Only one underscore is allowed as numeric separator") \
T(PropertyValueOutOfRange, "% value is out of range.") \
T(StackOverflow, "Maximum call stack size exceeded") \
T(ToPrecisionFormatRange, \
......
......@@ -307,6 +307,12 @@ class ParserBase {
void set_allow_harmony_bigint(bool allow) {
scanner()->set_allow_harmony_bigint(allow);
}
bool allow_harmony_numeric_separator() const {
return scanner()->allow_harmony_numeric_separator();
}
void set_allow_harmony_numeric_separator(bool allow) {
scanner()->set_allow_harmony_numeric_separator(allow);
}
bool allow_harmony_private_fields() const {
return scanner()->allow_harmony_private_fields();
......
......@@ -448,6 +448,7 @@ Parser::Parser(ParseInfo* info)
set_allow_harmony_dynamic_import(FLAG_harmony_dynamic_import);
set_allow_harmony_import_meta(FLAG_harmony_import_meta);
set_allow_harmony_bigint(FLAG_harmony_bigint);
set_allow_harmony_numeric_separator(FLAG_harmony_numeric_separator);
set_allow_harmony_optional_catch_binding(FLAG_harmony_optional_catch_binding);
set_allow_harmony_private_fields(FLAG_harmony_private_fields);
for (int feature = 0; feature < v8::Isolate::kUseCounterFeatureCount;
......
......@@ -1223,73 +1223,209 @@ Handle<String> Scanner::SourceMappingUrl(Isolate* isolate) const {
return tmp;
}
void Scanner::ScanDecimalDigits() {
while (IsDecimalDigit(c0_))
bool Scanner::ScanDigitsWithNumericSeparators(bool (*predicate)(uc32 ch),
int start_pos,
bool is_check_first_digit) {
// we must have at least one digit after 'x'/'b'/'o'
if (is_check_first_digit && !predicate(c0_)) return false;
bool separator_seen = false;
while (predicate(c0_) || c0_ == '_') {
if (c0_ == '_') {
Advance<false, false>();
if (c0_ == '_') {
ReportScannerError(Location(start_pos, source_pos()),
MessageTemplate::kContinuousNumericSeparator);
return false;
}
separator_seen = true;
continue;
}
separator_seen = false;
AddLiteralCharAdvance();
}
if (separator_seen) {
ReportScannerError(Location(start_pos, source_pos()),
MessageTemplate::kTrailingNumericSeparator);
return false;
}
return true;
}
bool Scanner::ScanDecimalDigits(int start_pos) {
if (allow_harmony_numeric_separator()) {
return ScanDigitsWithNumericSeparators(&IsDecimalDigit, start_pos, false);
}
while (IsDecimalDigit(c0_)) {
AddLiteralCharAdvance();
}
return true;
}
bool Scanner::ScanBinaryDigits() {
bool Scanner::ScanDecimalAsSmiWithNumericSeparators(int start_pos,
uint64_t* value) {
bool separator_seen = false;
while (IsDecimalDigit(c0_) || c0_ == '_') {
if (c0_ == '_') {
Advance<false, false>();
if (c0_ == '_') {
ReportScannerError(Location(start_pos, source_pos()),
MessageTemplate::kContinuousNumericSeparator);
return false;
}
separator_seen = true;
continue;
}
separator_seen = false;
*value = 10 * *value + (c0_ - '0');
uc32 first_char = c0_;
Advance<false, false>();
AddLiteralChar(first_char);
}
if (separator_seen) {
ReportScannerError(Location(start_pos, source_pos()),
MessageTemplate::kTrailingNumericSeparator);
return false;
}
return true;
}
bool Scanner::ScanDecimalAsSmi(int start_pos, uint64_t* value) {
if (allow_harmony_numeric_separator()) {
return ScanDecimalAsSmiWithNumericSeparators(start_pos, value);
}
while (IsDecimalDigit(c0_)) {
*value = 10 * *value + (c0_ - '0');
uc32 first_char = c0_;
Advance<false, false>();
AddLiteralChar(first_char);
}
return true;
}
bool Scanner::ScanBinaryDigits(int start_pos) {
if (allow_harmony_numeric_separator()) {
return ScanDigitsWithNumericSeparators(&IsBinaryDigit, start_pos, true);
}
// we must have at least one binary digit after 'b'/'B'
if (!IsBinaryDigit(c0_)) return false;
if (!IsBinaryDigit(c0_)) {
return false;
}
while (IsBinaryDigit(c0_)) {
AddLiteralCharAdvance();
}
return true;
}
bool Scanner::ScanOctalDigits() {
bool Scanner::ScanOctalDigits(int start_pos) {
if (allow_harmony_numeric_separator()) {
return ScanDigitsWithNumericSeparators(&IsOctalDigit, start_pos, true);
}
// we must have at least one octal digit after 'o'/'O'
if (!IsOctalDigit(c0_)) return false;
if (!IsOctalDigit(c0_)) {
return false;
}
while (IsOctalDigit(c0_)) {
AddLiteralCharAdvance();
}
return true;
}
bool Scanner::ScanImplicitOctalDigits(int start_pos) {
// (possible) octal number
bool Scanner::ScanImplicitOctalDigitsWithNumericSeparators(
int start_pos, Scanner::NumberKind* kind) {
bool separator_seen = false;
while (true) {
if (c0_ == '8' || c0_ == '9') return false;
if (c0_ == '_') {
Advance<false, false>();
if (c0_ == '_') {
ReportScannerError(Location(start_pos, source_pos()),
MessageTemplate::kContinuousNumericSeparator);
return false;
}
separator_seen = true;
continue;
}
if (c0_ == '8' || c0_ == '9') {
*kind = DECIMAL_WITH_LEADING_ZERO;
return true;
}
if (c0_ < '0' || '7' < c0_) {
// Octal literal finished.
octal_pos_ = Location(start_pos, source_pos());
octal_message_ = MessageTemplate::kStrictOctalLiteral;
break;
if (separator_seen) {
ReportScannerError(Location(start_pos, source_pos()),
MessageTemplate::kTrailingNumericSeparator);
return false;
}
return true;
}
separator_seen = false;
AddLiteralCharAdvance();
}
}
bool Scanner::ScanImplicitOctalDigits(int start_pos,
Scanner::NumberKind* kind) {
*kind = IMPLICIT_OCTAL;
if (allow_harmony_numeric_separator()) {
return ScanImplicitOctalDigitsWithNumericSeparators(start_pos, kind);
}
while (true) {
// (possible) octal number
if (c0_ == '8' || c0_ == '9') {
*kind = DECIMAL_WITH_LEADING_ZERO;
return true;
}
if (c0_ < '0' || '7' < c0_) {
// Octal literal finished.
octal_pos_ = Location(start_pos, source_pos());
octal_message_ = MessageTemplate::kStrictOctalLiteral;
return true;
}
AddLiteralCharAdvance();
}
return true;
}
bool Scanner::ScanHexDigits() {
bool Scanner::ScanHexDigits(int start_pos) {
if (allow_harmony_numeric_separator()) {
return ScanDigitsWithNumericSeparators(&IsHexDigit, start_pos, true);
}
// we must have at least one hex digit after 'x'/'X'
if (!IsHexDigit(c0_)) return false;
if (!IsHexDigit(c0_)) {
return false;
}
while (IsHexDigit(c0_)) {
AddLiteralCharAdvance();
}
return true;
}
bool Scanner::ScanSignedInteger() {
bool Scanner::ScanSignedInteger(int start_pos) {
if (c0_ == '+' || c0_ == '-') AddLiteralCharAdvance();
// we must have at least one decimal digit after 'e'/'E'
if (!IsDecimalDigit(c0_)) return false;
ScanDecimalDigits();
return true;
return ScanDecimalDigits(start_pos);
}
Token::Value Scanner::ScanNumber(bool seen_period) {
DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction
enum {
DECIMAL,
DECIMAL_WITH_LEADING_ZERO,
HEX,
OCTAL,
IMPLICIT_OCTAL,
BINARY
} kind = DECIMAL;
NumberKind kind = DECIMAL;
LiteralScope literal(this);
bool at_start = !seen_period;
......@@ -1297,8 +1433,11 @@ Token::Value Scanner::ScanNumber(bool seen_period) {
if (seen_period) {
// we have already seen a decimal point of the float
AddLiteralChar('.');
ScanDecimalDigits(); // we know we have at least one digit
if (allow_harmony_numeric_separator() && c0_ == '_') {
return Token::ILLEGAL;
}
// we know we have at least one digit
if (!ScanDecimalDigits(start_pos)) return Token::ILLEGAL;
} else {
// if the first character is '0' we must check for octals and hex
if (c0_ == '0') {
......@@ -1309,19 +1448,21 @@ Token::Value Scanner::ScanNumber(bool seen_period) {
if (c0_ == 'x' || c0_ == 'X') {
AddLiteralCharAdvance();
kind = HEX;
if (!ScanHexDigits()) return Token::ILLEGAL;
if (!ScanHexDigits(start_pos)) return Token::ILLEGAL;
} else if (c0_ == 'o' || c0_ == 'O') {
AddLiteralCharAdvance();
kind = OCTAL;
if (!ScanOctalDigits()) return Token::ILLEGAL;
if (!ScanOctalDigits(start_pos)) return Token::ILLEGAL;
} else if (c0_ == 'b' || c0_ == 'B') {
AddLiteralCharAdvance();
kind = BINARY;
if (!ScanBinaryDigits()) return Token::ILLEGAL;
if (!ScanBinaryDigits(start_pos)) return Token::ILLEGAL;
} else if ('0' <= c0_ && c0_ <= '7') {
kind = IMPLICIT_OCTAL;
if (!ScanImplicitOctalDigits(start_pos)) {
kind = DECIMAL_WITH_LEADING_ZERO;
if (!ScanImplicitOctalDigits(start_pos, &kind)) {
return Token::ILLEGAL;
}
if (kind == DECIMAL_WITH_LEADING_ZERO) {
at_start = false;
}
} else if (c0_ == '8' || c0_ == '9') {
......@@ -1334,12 +1475,9 @@ Token::Value Scanner::ScanNumber(bool seen_period) {
// This is an optimization for parsing Decimal numbers as Smi's.
if (at_start) {
uint64_t value = 0;
while (IsDecimalDigit(c0_)) {
value = 10 * value + (c0_ - '0');
uc32 first_char = c0_;
Advance<false, false>();
AddLiteralChar(first_char);
// scan subsequent decimal digits
if (!ScanDecimalAsSmi(start_pos, &value)) {
return Token::ILLEGAL;
}
if (next_.literal_chars->one_byte_literal().length() <= 10 &&
......@@ -1358,11 +1496,14 @@ Token::Value Scanner::ScanNumber(bool seen_period) {
HandleLeadSurrogate();
}
ScanDecimalDigits(); // optional
if (!ScanDecimalDigits(start_pos)) return Token::ILLEGAL;
if (c0_ == '.') {
seen_period = true;
AddLiteralCharAdvance();
ScanDecimalDigits(); // optional
if (allow_harmony_numeric_separator() && c0_ == '_') {
return Token::ILLEGAL;
}
if (!ScanDecimalDigits(start_pos)) return Token::ILLEGAL;
}
}
}
......@@ -1393,7 +1534,7 @@ Token::Value Scanner::ScanNumber(bool seen_period) {
// scan exponent
AddLiteralCharAdvance();
if (!ScanSignedInteger()) return Token::ILLEGAL;
if (!ScanSignedInteger(start_pos)) return Token::ILLEGAL;
}
// The source character immediately following a numeric literal must
......
......@@ -366,6 +366,12 @@ class Scanner {
void set_allow_harmony_private_fields(bool allow) {
allow_harmony_private_fields_ = allow;
}
bool allow_harmony_numeric_separator() const {
return allow_harmony_numeric_separator_;
}
void set_allow_harmony_numeric_separator(bool allow) {
allow_harmony_numeric_separator_ = allow;
}
private:
// Scoped helper for saving & restoring scanner error state.
......@@ -490,6 +496,15 @@ class Scanner {
Token::Value contextual_token;
};
enum NumberKind {
BINARY,
OCTAL,
IMPLICIT_OCTAL,
HEX,
DECIMAL,
DECIMAL_WITH_LEADING_ZERO
};
static const int kCharacterLookaheadBufferSize = 1;
const int kMaxAscii = 127;
......@@ -720,12 +735,20 @@ class Scanner {
// Scans a possible HTML comment -- begins with '<!'.
Token::Value ScanHtmlComment();
void ScanDecimalDigits();
bool ScanHexDigits();
bool ScanBinaryDigits();
bool ScanSignedInteger();
bool ScanOctalDigits();
bool ScanImplicitOctalDigits(int start_pos);
bool ScanDigitsWithNumericSeparators(bool (*predicate)(uc32 ch),
int start_pos,
bool is_check_first_digit);
bool ScanDecimalDigits(int start_pos);
// Optimized function to scan decimal number as Smi.
bool ScanDecimalAsSmi(int start_pos, uint64_t* value);
bool ScanDecimalAsSmiWithNumericSeparators(int start_pos, uint64_t* value);
bool ScanHexDigits(int start_pos);
bool ScanBinaryDigits(int start_pos);
bool ScanSignedInteger(int start_pos);
bool ScanOctalDigits(int start_pos);
bool ScanImplicitOctalDigits(int start_pos, NumberKind* kind);
bool ScanImplicitOctalDigitsWithNumericSeparators(int start_pos,
NumberKind* kind);
Token::Value ScanNumber(bool seen_period);
Token::Value ScanIdentifierOrKeyword();
......@@ -817,6 +840,7 @@ class Scanner {
// Harmony flags to allow ESNext features.
bool allow_harmony_bigint_;
bool allow_harmony_private_fields_;
bool allow_harmony_numeric_separator_;
MessageTemplate::Template scanner_error_;
Location scanner_error_location_;
......
......@@ -1122,6 +1122,7 @@ enum ParserFlag {
kAllowHarmonyImportMeta,
kAllowHarmonyDoExpressions,
kAllowHarmonyOptionalCatchBinding,
kAllowHarmonyNumericSeparator
};
enum ParserSyncTestResult {
......@@ -1140,6 +1141,8 @@ void SetGlobalFlags(i::EnumSet<ParserFlag> flags) {
i::FLAG_harmony_do_expressions = flags.Contains(kAllowHarmonyDoExpressions);
i::FLAG_harmony_optional_catch_binding =
flags.Contains(kAllowHarmonyOptionalCatchBinding);
i::FLAG_harmony_numeric_separator =
flags.Contains(kAllowHarmonyNumericSeparator);
}
void SetParserFlags(i::PreParser* parser, i::EnumSet<ParserFlag> flags) {
......@@ -1158,6 +1161,8 @@ void SetParserFlags(i::PreParser* parser, i::EnumSet<ParserFlag> flags) {
flags.Contains(kAllowHarmonyDoExpressions));
parser->set_allow_harmony_optional_catch_binding(
flags.Contains(kAllowHarmonyOptionalCatchBinding));
parser->set_allow_harmony_numeric_separator(
flags.Contains(kAllowHarmonyNumericSeparator));
}
void TestParserSyncWithFlags(i::Handle<i::String> source,
......@@ -1480,6 +1485,91 @@ void RunModuleParserSyncTest(
always_false_len, true, test_preparser, ignore_error_msg);
}
TEST(NumericSeparator) {
v8::HandleScope handles(CcTest::isolate());
v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
v8::Context::Scope context_scope(context);
const char* context_data[][2] = {
{"", ""}, {"\"use strict\";", ""}, {nullptr, nullptr}};
const char* statement_data[] = {
"1_0_0_0", "1_0e+1", "1_0e+1_0", "0xF_F_FF", "0o7_7_7", "0b0_1_0_1_0",
".3_2_1", "0.0_2_1", "1_0.0_1", ".0_1_2", nullptr};
static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
RunParserSyncTest(context_data, statement_data, kSuccess, nullptr, 0, flags,
1);
RunParserSyncTest(context_data, statement_data, kError);
}
TEST(NumericSeparatorErrors) {
v8::HandleScope handles(CcTest::isolate());
v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
v8::Context::Scope context_scope(context);
const char* context_data[][2] = {
{"", ""}, {"\"use strict\";", ""}, {nullptr, nullptr}};
const char* statement_data[] = {
"1_0_0_0_", "1e_1", "1e+_1", "1_e+1", "1__0", "0x_1",
"0x1__1", "0x1_", "0_x1", "0_x_1", "0b_0101", "0b11_",
"0b1__1", "0_b1", "0_b_1", "0o777_", "0o_777", "0o7__77",
"0.0_2_1_", "0.0__21", "0_.01", "0._01", nullptr};
static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
RunParserSyncTest(context_data, statement_data, kError, nullptr, 0, flags, 1,
nullptr, 0, false, true, true);
RunParserSyncTest(context_data, statement_data, kError);
}
TEST(NumericSeparatorImplicitOctals) {
v8::HandleScope handles(CcTest::isolate());
v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
v8::Context::Scope context_scope(context);
const char* context_data[][2] = {
{"", ""}, {nullptr, nullptr}, {nullptr, nullptr}};
const char* statement_data[] = {"07_7_7", "0_7_7_7", "0_777", nullptr};
static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
RunParserSyncTest(context_data, statement_data, kSuccess, nullptr, 0, flags,
1);
RunParserSyncTest(context_data, statement_data, kError);
}
TEST(NumericSeparatorImplicitOctalsErrors) {
v8::HandleScope handles(CcTest::isolate());
v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
v8::Context::Scope context_scope(context);
const char* context_data[][2] = {
{"", ""}, {nullptr, nullptr}, {nullptr, nullptr}};
const char* statement_data[] = {"07_7_7_", "07__77", "0__777", nullptr};
static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
RunParserSyncTest(context_data, statement_data, kError, nullptr, 0, flags, 1,
nullptr, 0, false, true, true);
RunParserSyncTest(context_data, statement_data, kError);
}
TEST(NumericSeparatorUnicodeEscapeSequencesErrors) {
v8::HandleScope handles(CcTest::isolate());
v8::Local<v8::Context> context = v8::Context::New(CcTest::isolate());
v8::Context::Scope context_scope(context);
const char* context_data[][2] = {
{"", ""}, {"'use strict'", ""}, {nullptr, nullptr}};
// https://github.com/tc39/proposal-numeric-separator/issues/25
const char* statement_data[] = {"\\u{10_FFFF}", nullptr};
static const ParserFlag flags[] = {kAllowHarmonyNumericSeparator};
RunParserSyncTest(context_data, statement_data, kError, nullptr, 0, flags, 1);
RunParserSyncTest(context_data, statement_data, kError);
}
TEST(ErrorsEvalAndArguments) {
// Tests that both preparsing and parsing produce the right kind of errors for
......
// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Flags: --harmony-numeric-separator
{
const basic = 1_0_0_0;
assertEquals(basic, 1000);
}
{
const exponent = 1_0e+1;
assertEquals(exponent, 10e+1);
}
{
const exponent2 = 1_0e+1_0;
assertEquals(exponent2, 10e+10);
}
{
const hex = 0xF_F_FF;
assertEquals(hex, 0xFFFF);
}
{
const octal = 0o7_7_7;
assertEquals(octal, 0o777);
}
{
const implicitOctal = 07_7_7;
assertEquals(implicitOctal, 0o777);
}
{
let exception = false;
try {
const code = `"use strict" const implicitOctal = 07_7_7`;
eval(code);
} catch(e) {
exception = true;
assertInstanceof(e, SyntaxError);
}
assertTrue(exception);
}
{
const binary = 0b0_1_0_1_0;
assertEquals(binary, 0b01010);
}
{
const leadingZeros = 09_1_3;
assertEquals(leadingZeros, 0913);
}
assertThrows('1_0_0_0_', SyntaxError);
assertThrows('1e_1', SyntaxError);
assertThrows('1e+_1', SyntaxError);
assertThrows('1_e+1', SyntaxError);
assertThrows('1__0', SyntaxError);
assertThrows('0x_1', SyntaxError);
assertThrows('0x1__1', SyntaxError);
assertThrows('0x1_', SyntaxError);
assertThrows('0b_0101', SyntaxError);
assertThrows('0b11_', SyntaxError);
assertThrows('0b1__1', SyntaxError);
assertThrows('0o777_', SyntaxError);
assertThrows('0o_777', SyntaxError);
assertThrows('0o7__77', SyntaxError);
assertThrows('0777_', SyntaxError);
assertThrows('07__77', SyntaxError);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment