[foozzie] Ignore error messages in correctness fuzzing

Error messages are unspecified in JavaScript and occasional small
differences in the compared configurations lead to an unjustified
maintenance burden of correctness-fuzzing issues.

This CL replaces most error messages with a fixed suppression
message during correctness fuzzing (behind a flag).

The flag covering all extra behavior for correctness fuzzing is now
renamed to --correctness-fuzzer-suppressions.

Bug: chromium:958668,chromium:946476
Change-Id: Iba1197f765138a962d5bbb176730322e5a411707
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1594730
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Michael Starzinger <mstarzinger@chromium.org>
Reviewed-by: Clemens Hammacher <clemensh@chromium.org>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#61249}

src/flag-definitions.h

@@ -1130,12 +1130,11 @@ DEFINE_BOOL(stack_trace_on_illegal, false,
             "print stack trace when an illegal exception is thrown")
 DEFINE_BOOL(abort_on_uncaught_exception, false,
             "abort program (dump core) when an uncaught exception is thrown")
-// TODO(jgruber,machenbach): Rename to --correctness-fuzzer-suppressions.
-DEFINE_BOOL(abort_on_stack_or_string_length_overflow, false,
-            "Abort program when the stack overflows or a string exceeds "
-            "maximum length (as opposed to throwing RangeError). This is "
-            "useful for fuzzing where the spec behaviour would introduce "
-            "nondeterminism.")
+DEFINE_BOOL(correctness_fuzzer_suppressions, false,
+            "Suppress certain unspecified behaviors to ease correctness "
+            "fuzzing: Abort program when the stack overflows or a string "
+            "exceeds maximum length (as opposed to throwing RangeError). "
+            "Use a fixed suppression string for error messages.")
 DEFINE_BOOL(randomize_hashes, true,
             "randomize hashes to avoid predictable hash collisions "
             "(with snapshots this option cannot override the baked-in seed)")

src/heap/factory.cc

@@ -2404,7 +2404,7 @@ Handle<Object> Factory::NewError(Handle<JSFunction> constructor,
 }
 
 Handle<Object> Factory::NewInvalidStringLengthError() {
-  if (FLAG_abort_on_stack_or_string_length_overflow) {
+  if (FLAG_correctness_fuzzer_suppressions) {
     FATAL("Aborting on invalid string length");
   }
   // Invalidate the "string length" protector.

src/isolate.cc

@@ -1370,7 +1370,7 @@ bool Isolate::MayAccess(Handle<Context> accessing_context,
 }
 
 Object Isolate::StackOverflow() {
-  if (FLAG_abort_on_stack_or_string_length_overflow) {
+  if (FLAG_correctness_fuzzer_suppressions) {
     FATAL("Aborting on stack overflow");
   }
 

src/messages.cc

@@ -1304,11 +1304,19 @@ MaybeHandle<Object> ErrorUtils::MakeGenericError(
     // pending exceptions would be cleared. Preserve this behavior.
     isolate->clear_pending_exception();
   }
+  Handle<String> msg;
+  if (FLAG_correctness_fuzzer_suppressions) {
+    // Ignore error messages in correctness fuzzing, because the spec leaves
+    // room for undefined behavior.
+    msg = isolate->factory()->InternalizeUtf8String(
+        "Message suppressed for fuzzers (--correctness-fuzzer-suppressions)");
+  } else {
+    msg = DoFormatMessage(isolate, index, arg0, arg1, arg2);
+  }
 
   DCHECK(mode != SKIP_UNTIL_SEEN);
 
   Handle<Object> no_caller;
-  Handle<String> msg = DoFormatMessage(isolate, index, arg0, arg1, arg2);
   return ErrorUtils::Construct(isolate, constructor, constructor, msg, mode,
                                no_caller, false);
 }

src/regexp/jsregexp.cc

@@ -340,7 +340,7 @@ bool RegExpImpl::CompileIrregexp(Isolate* isolate, Handle<JSRegExp> re,
                             sample_subject, is_one_byte);
   if (result.error_message != nullptr) {
     // Unable to compile regexp.
-    if (FLAG_abort_on_stack_or_string_length_overflow &&
+    if (FLAG_correctness_fuzzer_suppressions &&
         strncmp(result.error_message, "Stack overflow", 15) == 0) {
       FATAL("Aborting on stack overflow");
     }

src/regexp/regexp-parser.cc

@@ -77,7 +77,7 @@ void RegExpParser::Advance() {
   if (has_next()) {
     StackLimitCheck check(isolate());
     if (check.HasOverflowed()) {
-      if (FLAG_abort_on_stack_or_string_length_overflow) {
+      if (FLAG_correctness_fuzzer_suppressions) {
         FATAL("Aborting on stack overflow");
       }
       ReportError(CStrVector(

src/v8.cc

@@ -88,7 +88,7 @@ void V8::InitializeOncePerProcessImpl() {
   // continue exposing wasm on correctness fuzzers even in jitless mode.
   // TODO(jgruber): Remove this once / if wasm can run without executable
   // memory.
-  if (FLAG_jitless && !FLAG_abort_on_stack_or_string_length_overflow) {
+  if (FLAG_jitless && !FLAG_correctness_fuzzer_suppressions) {
     FLAG_expose_wasm = false;
   }
 

src/wasm/wasm-memory.cc

@@ -84,10 +84,10 @@ void* TryAllocateBackingStore(WasmMemoryTracker* memory_tracker, Heap* heap,
     // We are over the address space limit. Fail.
     //
     // When running under the correctness fuzzer (i.e.
-    // --abort-on-stack-or-string-length-overflow is preset), we crash
+    // --correctness-fuzzer-suppressions is preset), we crash
     // instead so it is not incorrectly reported as a correctness
     // violation. See https://crbug.com/828293#c4
-    if (FLAG_abort_on_stack_or_string_length_overflow) {
+    if (FLAG_correctness_fuzzer_suppressions) {
       FATAL("could not allocate wasm memory");
     }
     AddAllocationStatusSample(

tools/clusterfuzz/testdata/failure_output.txt

@@ -9,9 +9,9 @@
 # Compared x64,ignition with x64,ignition_turbo
 #
 # Flags of x64,ignition:
---abort-on-stack-or-string-length-overflow --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --no-wasm-async-compilation --suppress-asm-messages --random-seed 12345 --turbo-filter=~ --noopt --liftoff --no-wasm-tier-up
+--correctness-fuzzer-suppressions --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --no-wasm-async-compilation --suppress-asm-messages --random-seed 12345 --turbo-filter=~ --noopt --liftoff --no-wasm-tier-up
 # Flags of x64,ignition_turbo:
---abort-on-stack-or-string-length-overflow --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --no-wasm-async-compilation --suppress-asm-messages --random-seed 12345 --stress-scavenge=100
+--correctness-fuzzer-suppressions --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --no-wasm-async-compilation --suppress-asm-messages --random-seed 12345 --stress-scavenge=100
 #
 # Difference:
 - unknown

tools/clusterfuzz/testdata/sanity_check_output.txt

@@ -9,9 +9,9 @@
 # Compared x64,ignition with x64,ignition_turbo
 #
 # Flags of x64,ignition:
---abort-on-stack-or-string-length-overflow --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --no-wasm-async-compilation --suppress-asm-messages --random-seed 12345 --turbo-filter=~ --noopt --liftoff --no-wasm-tier-up
+--correctness-fuzzer-suppressions --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --no-wasm-async-compilation --suppress-asm-messages --random-seed 12345 --turbo-filter=~ --noopt --liftoff --no-wasm-tier-up
 # Flags of x64,ignition_turbo:
---abort-on-stack-or-string-length-overflow --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --no-wasm-async-compilation --suppress-asm-messages --random-seed 12345 --stress-scavenge=100
+--correctness-fuzzer-suppressions --expose-gc --allow-natives-syntax --invoke-weak-callbacks --omit-quit --es-staging --no-wasm-async-compilation --suppress-asm-messages --random-seed 12345 --stress-scavenge=100
 #
 # Difference:
 - unknown

tools/clusterfuzz/v8_foozzie.py

@@ -103,7 +103,7 @@ PREAMBLE = [
 ARCH_MOCKS = os.path.join(BASE_PATH, 'v8_mock_archs.js')
 SANITY_CHECKS = os.path.join(BASE_PATH, 'v8_sanity_checks.js')
 
-FLAGS = ['--abort-on-stack-or-string-length-overflow', '--expose-gc',
+FLAGS = ['--correctness-fuzzer-suppressions', '--expose-gc',
          '--allow-natives-syntax', '--invoke-weak-callbacks', '--omit-quit',
          '--es-staging', '--no-wasm-async-compilation',
          '--suppress-asm-messages']