Check that function was prepared before optimizing using manually

With lazy feedback allocation and bytecode flushing we need to call
%PrepareFunctionForOptimize before we call %OptimizeFunctionOnNextCall/
%OptimizeOsr. This cl:
1. Adds an additional state in pending optimized table to check if the
optimization was triggered manually.
2. Changes the compilation pipeline to delete the entry from pending
optimized table only if the optimization was triggered through
%OptimizeFunctionOnNextCall / %OptimizeOsr.
3. Adds a check to enforce %PrepareFunctionForOptimize was called.
4. Adds a new run-time flag to only check in the d8 test runner. We
don't want this check enabled in other cases like clusterfuzz that doesn't
ensure %PrepareFunctionForOptimize is called.

Bug: v8:8394, v8:8801, v8:9183
Change-Id: I9ae2b2da812e313c746b6df0b2da864c2ed5de51
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1664810
Commit-Queue: Mythri Alle <mythria@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Ross McIlroy <rmcilroy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#62653}

BUILD.gn

@@ -2087,6 +2087,8 @@ v8_source_set("v8_base_without_compiler") {
     "src/codegen/macro-assembler.h",
     "src/codegen/optimized-compilation-info.cc",
     "src/codegen/optimized-compilation-info.h",
+    "src/codegen/pending-optimization-table.cc",
+    "src/codegen/pending-optimization-table.h",
     "src/codegen/register-arch.h",
     "src/codegen/register-configuration.cc",
     "src/codegen/register-configuration.h",

src/codegen/compiler.cc

@@ -15,6 +15,7 @@
 #include "src/codegen/assembler-inl.h"
 #include "src/codegen/compilation-cache.h"
 #include "src/codegen/optimized-compilation-info.h"
+#include "src/codegen/pending-optimization-table.h"
 #include "src/codegen/unoptimized-compilation-info.h"
 #include "src/common/globals.h"
 #include "src/common/message-template.h"
@@ -797,18 +798,10 @@ MaybeHandle<Code> GetOptimizedCode(Handle<JSFunction> function,
     return MaybeHandle<Code>();
   }
 
-  // If code was pending optimization for testing, delete remove the strong root
-  // that was preventing the bytecode from being flushed between marking and
-  // optimization.
-  if (!isolate->heap()->pending_optimize_for_test_bytecode().IsUndefined()) {
-    Handle<ObjectHashTable> table =
-        handle(ObjectHashTable::cast(
-                   isolate->heap()->pending_optimize_for_test_bytecode()),
-               isolate);
-    bool was_present;
-    table = table->Remove(isolate, table, handle(function->shared(), isolate),
-                          &was_present);
-    isolate->heap()->SetPendingOptimizeForTestBytecode(*table);
+  // If code was pending optimization for testing, delete remove the entry
+  // from the table that was preventing the bytecode from being flushed
+  if (V8_UNLIKELY(FLAG_testing_d8_test_runner)) {
+    PendingOptimizationTable::FunctionWasOptimized(isolate, function);
   }
 
   Handle<Code> cached_code;

src/codegen/pending-optimization-table.cc

+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/codegen/pending-optimization-table.h"
+
+#include "src/execution/isolate-inl.h"
+#include "src/heap/heap-inl.h"
+#include "src/objects/hash-table.h"
+#include "src/objects/js-objects.h"
+
+namespace v8 {
+namespace internal {
+
+enum class FunctionStatus { kPrepareForOptimize, kMarkForOptimize };
+
+void PendingOptimizationTable::PreparedForOptimization(
+    Isolate* isolate, Handle<JSFunction> function) {
+  DCHECK(FLAG_testing_d8_test_runner);
+
+  Handle<ObjectHashTable> table =
+      isolate->heap()->pending_optimize_for_test_bytecode().IsUndefined()
+          ? ObjectHashTable::New(isolate, 1)
+          : handle(ObjectHashTable::cast(
+                       isolate->heap()->pending_optimize_for_test_bytecode()),
+                   isolate);
+  Handle<Tuple2> tuple = isolate->factory()->NewTuple2(
+      handle(function->shared().GetBytecodeArray(), isolate),
+      handle(
+          Smi::FromInt(static_cast<int>(FunctionStatus::kPrepareForOptimize)),
+          isolate),
+      AllocationType::kYoung);
+  table =
+      ObjectHashTable::Put(table, handle(function->shared(), isolate), tuple);
+  isolate->heap()->SetPendingOptimizeForTestBytecode(*table);
+}
+
+void PendingOptimizationTable::MarkedForOptimization(
+    Isolate* isolate, Handle<JSFunction> function) {
+  DCHECK(FLAG_testing_d8_test_runner);
+
+  Handle<Object> table =
+      handle(isolate->heap()->pending_optimize_for_test_bytecode(), isolate);
+  Handle<Object> entry =
+      table->IsUndefined()
+          ? handle(ReadOnlyRoots(isolate).the_hole_value(), isolate)
+          : handle(Handle<ObjectHashTable>::cast(table)->Lookup(
+                       handle(function->shared(), isolate)),
+                   isolate);
+  if (entry->IsTheHole()) {
+    PrintF("Error: Function ");
+    function->ShortPrint();
+    PrintF(
+        " should be prepared for optimization with "
+        "%%PrepareFunctionForOptimize before  "
+        "%%OptimizeFunctionOnNextCall / %%OptimizeOSR ");
+    UNREACHABLE();
+  }
+
+  DCHECK(entry->IsTuple2());
+  Handle<Tuple2>::cast(entry)->set_value2(
+      Smi::FromInt(static_cast<int>(FunctionStatus::kMarkForOptimize)));
+  table = ObjectHashTable::Put(Handle<ObjectHashTable>::cast(table),
+                               handle(function->shared(), isolate), entry);
+  isolate->heap()->SetPendingOptimizeForTestBytecode(*table);
+}
+
+void PendingOptimizationTable::FunctionWasOptimized(
+    Isolate* isolate, Handle<JSFunction> function) {
+  DCHECK(FLAG_testing_d8_test_runner);
+
+  if (isolate->heap()->pending_optimize_for_test_bytecode().IsUndefined()) {
+    return;
+  }
+
+  Handle<ObjectHashTable> table =
+      handle(ObjectHashTable::cast(
+                 isolate->heap()->pending_optimize_for_test_bytecode()),
+             isolate);
+  Handle<Object> value(table->Lookup(handle(function->shared(), isolate)),
+                       isolate);
+  // Remove only if we have already seen %OptimizeFunctionOnNextCall. If it is
+  // optimized for other reasons, still keep holding the bytecode since we may
+  // optimize it later.
+  if (!value->IsTheHole() &&
+      Smi::cast(Handle<Tuple2>::cast(value)->value2()).value() ==
+          static_cast<int>(FunctionStatus::kMarkForOptimize)) {
+    bool was_present;
+    table = table->Remove(isolate, table, handle(function->shared(), isolate),
+                          &was_present);
+    DCHECK(was_present);
+    isolate->heap()->SetPendingOptimizeForTestBytecode(*table);
+  }
+}
+
+}  // namespace internal
+}  // namespace v8

src/codegen/pending-optimization-table.h

+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_CODEGEN_PENDING_OPTIMIZATION_TABLE_H_
+#define V8_CODEGEN_PENDING_OPTIMIZATION_TABLE_H_
+
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+// This class adds the functionality to properly test the optimized code. This
+// is only for use in tests. All these functions should only be called when
+// testing_d8_flag_for_tests is set.
+class PendingOptimizationTable {
+ public:
+  // This function should be called before we mark the function for
+  // optimization. Calling this function ensures that |function| is compiled and
+  // has a feedback vector allocated. This also holds on to the bytecode
+  // strongly in pending optimization table preventing the bytecode to be
+  // flushed.
+  static void PreparedForOptimization(Isolate* isolate,
+                                      Handle<JSFunction> function);
+
+  // This function should be called when the function is marked for optimization
+  // via the intrinsics. This will update the state of the bytecode array in the
+  // pending optimization table, so that the entry can be removed once the
+  // function is optimized. If the function is already optimized it removes the
+  // entry from the table.
+  static void MarkedForOptimization(Isolate* isolate,
+                                    Handle<JSFunction> function);
+
+  // This function should be called once the function is optimized. If there is
+  // an entry in the pending optimization table and it is marked for removal
+  // then this function removes the entry from pending optimization table.
+  static void FunctionWasOptimized(Isolate* isolate,
+                                   Handle<JSFunction> function);
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_CODEGEN_PENDING_OPTIMIZATION_TABLE_H_

src/flags/flag-definitions.h

@@ -1208,6 +1208,12 @@ DEFINE_FLOAT(testing_float_flag, 2.5, "float-flag")
 DEFINE_STRING(testing_string_flag, "Hello, world!", "string-flag")
 DEFINE_INT(testing_prng_seed, 42, "Seed used for threading test randomness")
 
+// Test flag for a check in %OptimizeFunctionOnNextCall
+DEFINE_BOOL(
+    testing_d8_test_runner, false,
+    "test runner turns on this flag to enable a check that the funciton was "
+    "prepared for optimization before marking it for optimization")
+
 // mksnapshot.cc
 DEFINE_STRING(embedded_src, nullptr,
               "Path for the generated embedded data file. (mksnapshot only)")

src/runtime/runtime-test.cc

@@ -11,6 +11,7 @@
 #include "src/base/platform/mutex.h"
 #include "src/codegen/assembler-inl.h"
 #include "src/codegen/compiler.h"
+#include "src/codegen/pending-optimization-table.h"
 #include "src/compiler-dispatcher/optimizing-compile-dispatcher.h"
 #include "src/deoptimizer/deoptimizer.h"
 #include "src/execution/arguments-inl.h"
@@ -218,28 +219,6 @@ RUNTIME_FUNCTION(Runtime_IsConcurrentRecompilationSupported) {
       isolate->concurrent_recompilation_enabled());
 }
 
-namespace {
-
-void RemoveBytecodeFromPendingOptimizeTable(v8::internal::Isolate* isolate,
-                                            Handle<JSFunction> function) {
-  // TODO(mythria): Remove the check for undefined, once we fix all tests to
-  // add PrepareForOptimization when using OptimizeFunctionOnNextCall.
-  if (isolate->heap()->pending_optimize_for_test_bytecode().IsUndefined()) {
-    return;
-  }
-
-  Handle<ObjectHashTable> table =
-      handle(ObjectHashTable::cast(
-                 isolate->heap()->pending_optimize_for_test_bytecode()),
-             isolate);
-  bool was_present;
-  table = table->Remove(isolate, table, handle(function->shared(), isolate),
-                        &was_present);
-  isolate->heap()->SetPendingOptimizeForTestBytecode(*table);
-}
-
-}  // namespace
-
 RUNTIME_FUNCTION(Runtime_OptimizeFunctionOnNextCall) {
   HandleScope scope(isolate);
 
@@ -271,9 +250,9 @@ RUNTIME_FUNCTION(Runtime_OptimizeFunctionOnNextCall) {
     return ReadOnlyRoots(isolate).undefined_value();
   }
 
-  if (function->shared().optimization_disabled() &&
-      function->shared().disable_optimization_reason() ==
-          BailoutReason::kNeverOptimize) {
+  if (!FLAG_opt || (function->shared().optimization_disabled() &&
+                    function->shared().disable_optimization_reason() ==
+                        BailoutReason::kNeverOptimize)) {
     return ReadOnlyRoots(isolate).undefined_value();
   }
 
@@ -281,25 +260,18 @@ RUNTIME_FUNCTION(Runtime_OptimizeFunctionOnNextCall) {
     return ReadOnlyRoots(isolate).undefined_value();
   }
 
+  if (FLAG_testing_d8_test_runner) {
+    PendingOptimizationTable::MarkedForOptimization(isolate, function);
+  }
+
   if (function->HasOptimizedCode()) {
     DCHECK(function->IsOptimized() || function->ChecksOptimizationMarker());
-    // If function is already optimized, remove the bytecode array from the
-    // pending optimize for test table and return. It is OK if there is no
-    // entry in the table since if the function got optimized before executing
-    // %OptimizeFunctionOnNextCall the entry would have been removed.
-    RemoveBytecodeFromPendingOptimizeTable(isolate, function);
+    if (FLAG_testing_d8_test_runner) {
+      PendingOptimizationTable::FunctionWasOptimized(isolate, function);
+    }
     return ReadOnlyRoots(isolate).undefined_value();
   }
 
-  // Check we called PrepareFunctionForOptimization and hold the bytecode
-  // array to prevent it from getting flushed.
-  // TODO(mythria): Enable this check once we add PrepareForOptimization in all
-  // tests before calling OptimizeFunctionOnNextCall.
-  // CHECK(!ObjectHashTable::cast(
-  //          isolate->heap()->pending_optimize_for_test_bytecode())
-  //          ->Lookup(handle(function->shared(), isolate))
-  //          ->IsTheHole());
-
   ConcurrencyMode concurrency_mode = ConcurrencyMode::kNotConcurrent;
   if (args.length() == 2) {
     // Ignore invalid inputs produced by fuzzers.
@@ -396,16 +368,9 @@ RUNTIME_FUNCTION(Runtime_PrepareFunctionForOptimization) {
 
   // Hold onto the bytecode array between marking and optimization to ensure
   // it's not flushed.
-  Handle<ObjectHashTable> table =
-      isolate->heap()->pending_optimize_for_test_bytecode().IsUndefined()
-          ? ObjectHashTable::New(isolate, 1)
-          : handle(ObjectHashTable::cast(
-                       isolate->heap()->pending_optimize_for_test_bytecode()),
-                   isolate);
-  table = ObjectHashTable::Put(
-      table, handle(function->shared(), isolate),
-      handle(function->shared().GetBytecodeArray(), isolate));
-  isolate->heap()->SetPendingOptimizeForTestBytecode(*table);
+  if (FLAG_testing_d8_test_runner) {
+    PendingOptimizationTable::PreparedForOptimization(isolate, function);
+  }
 
   return ReadOnlyRoots(isolate).undefined_value();
 }
@@ -425,26 +390,23 @@ RUNTIME_FUNCTION(Runtime_OptimizeOsr) {
   if (!it.done()) function = handle(it.frame()->function(), isolate);
   if (function.is_null()) return ReadOnlyRoots(isolate).undefined_value();
 
-  if (function->shared().optimization_disabled() &&
-      function->shared().disable_optimization_reason() ==
-          BailoutReason::kNeverOptimize) {
+  if (!FLAG_opt || (function->shared().optimization_disabled() &&
+                    function->shared().disable_optimization_reason() ==
+                        BailoutReason::kNeverOptimize)) {
     return ReadOnlyRoots(isolate).undefined_value();
   }
 
-  // Check we called PrepareFunctionForOptimization and hold the bytecode
-  // array to prevent it from getting flushed.
-  // TODO(mythria): Enable this check once we add PrepareForOptimization in all
-  // tests before calling OptimizeOsr.
-  // CHECK(!ObjectHashTable::cast(
-  //          isolate->heap()->pending_optimize_for_test_bytecode())
-  //          ->Lookup(handle(function->shared(), isolate))
-  //          ->IsTheHole());
+  if (FLAG_testing_d8_test_runner) {
+    PendingOptimizationTable::MarkedForOptimization(isolate, function);
+  }
 
   if (function->HasOptimizedCode()) {
     DCHECK(function->IsOptimized() || function->ChecksOptimizationMarker());
     // If function is already optimized, remove the bytecode array from the
     // pending optimize for test table and return.
-    RemoveBytecodeFromPendingOptimizeTable(isolate, function);
+    if (FLAG_testing_d8_test_runner) {
+      PendingOptimizationTable::FunctionWasOptimized(isolate, function);
+    }
     return ReadOnlyRoots(isolate).undefined_value();
   }
 

test/mjsunit/compiler/osr-block-scope-id.js

@@ -28,6 +28,7 @@ function foo() {
 
 
 function check() {
+  %PrepareFunctionForOptimization(foo);
   var r = foo();
   assertEquals(45, r.pop());
   for (var i = 9; i >= 0; i--) {

test/mjsunit/compiler/osr-block-scope.js

@@ -24,8 +24,11 @@ function nest(body, name, depth) {
 }
 
 function test(expected, func, depth) {
+  %PrepareFunctionForOptimization(func);
   assertEquals(expected, func());
+  %PrepareFunctionForOptimization(func);
   assertEquals(expected, func());
+  %PrepareFunctionForOptimization(func);
   assertEquals(expected, func());
 
   var orig = func.toString();

test/mjsunit/compiler/osr-infinite.js

@@ -19,6 +19,7 @@ function thrower() {
 function test(func) {
   for (var i = 0; i < 3; i++) {
     global_counter = 0;
+    %PrepareFunctionForOptimization(func);
     assertThrows(func);
   }
 }

test/mjsunit/compiler/osr-simple.js

@@ -15,9 +15,9 @@ function f() {
   }
   return sum;
 }
-%PrepareFunctionForOptimization(f);
 
 
 for (var i = 0; i < 2; i++) {
+  %PrepareFunctionForOptimization(f);
   assertEquals(509500, f());
 }

test/mjsunit/compiler/regress-977670.js

@@ -13,6 +13,7 @@ function foo() {
   i['' + 'length'] = 42;
 }
 
+%PrepareFunctionForOptimization(foo);
 foo();
 foo();
 foo();

test/mjsunit/es6/symbols.js

@@ -43,8 +43,8 @@ function isValidSymbolString(s) {
 function TestNew() {
   function indirectSymbol() { return Symbol() }
   function indirect() { return indirectSymbol() }
-  %PrepareFunctionForOptimization(indirect);
   for (var i = 0; i < 2; ++i) {
+    %PrepareFunctionForOptimization(indirect);
     for (var j = 0; j < 5; ++j) {
       symbols.push(Symbol())
       symbols.push(Symbol(undefined))

test/mjsunit/regress/cross-script-vars.js

@@ -71,14 +71,13 @@ function DefineLoadVar() {
       'var x;' +
       'function ' + name + '() {' +
       '  return x;' +
-      '};' +
-      (cfg.optimize ? '%PrepareFunctionForOptimization(' + name + ');' : '');
+      '};'
   return Realm.eval(test_realm, AddStrict(code, cfg));
 }
 
 function LoadVar() {
   var name = 'LoadVar_' + test_realm;
-  var code =
+  var code = (cfg.optimize ? '%PrepareFunctionForOptimization(' + name + ');' : '') +
       (cfg.optimize ? '%OptimizeFunctionOnNextCall(' + name + ');' : '') +
       name + '();';
   return Realm.eval(test_realm, AddStrict(code, cfg));
@@ -89,16 +88,14 @@ function DefineStoreVar() {
   var code = 'var g = (Function("return this"))();' +
       'var x;' +
       'function ' + name + '(v) {' +
-//      '  %DebugPrint(g);' +
       '  return x = v;' +
-      '};' +
-      (cfg.optimize ? '%PrepareFunctionForOptimization(' + name + ');' : '');
+      '};';
   return Realm.eval(test_realm, AddStrict(code, cfg));
 }
 
 function StoreVar(v) {
   var name = 'StoreVar_' + test_realm;
-  var code =
+  var code = (cfg.optimize ? '%PrepareFunctionForOptimization(' + name + ');' : '') +
       (cfg.optimize ? '%OptimizeFunctionOnNextCall(' + name + ');' : '') +
       name + '(' + v + ');';
   return Realm.eval(test_realm, AddStrict(code, cfg));

test/mjsunit/regress/regress-379770.js

@@ -6,7 +6,10 @@
 
 function foo(obj) {
   var counter = 1;
-  for (var i = 0; i < obj.length; i++) %OptimizeOsr();
+  for (var i = 0; i < obj.length; i++) {
+    %OptimizeOsr();
+    %PrepareFunctionForOptimization(foo);
+  }
   counter += obj;
   return counter;
 }

test/mjsunit/regress/regress-385565.js

@@ -61,6 +61,7 @@ assertTrue(r2 === r3);
 
 %OptimizeFunctionOnNextCall(callsFReceiver);
 r1 = callsFReceiver(o1);
+%PrepareFunctionForOptimization(callsFReceiver);
 callsFReceiver(o1);
 %OptimizeFunctionOnNextCall(callsFReceiver);
 r2 = callsFReceiver(o1);

test/mjsunit/regress/regress-5252.js

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Flags: --allow-natives-syntax --ignition-osr
+// Flags: --allow-natives-syntax
 
 (function TestNonLoopyLoop() {
   function f() {
@@ -13,6 +13,7 @@
   }
   %PrepareFunctionForOptimization(f);
   assertEquals(23, f());
+  %PrepareFunctionForOptimization(f);
   assertEquals(23, f());
 })();
 

test/mjsunit/regress/regress-634-debug.js

@@ -35,6 +35,7 @@ function f() {
   %SetAllocationTimeout(-1, -1, true);
 }
 
+%PrepareFunctionForOptimization(f);
 f();
 f();
 %OptimizeFunctionOnNextCall(f);

test/mjsunit/regress/regress-crbug-138887.js

@@ -41,10 +41,10 @@ function factory(worker) {
 var f1 = factory(worker1);
 var f2 = factory(f1);
 %PrepareFunctionForOptimization(f1);
-%PrepareFunctionForOptimization(f2);
 assertEquals(11, f2(1));  // Result: 1 + f1(0) == 1 + 10.
 assertEquals(11, f2(1));
 %OptimizeFunctionOnNextCall(f1);
+%PrepareFunctionForOptimization(f2);
 assertEquals(10, f1(0));  // Terminates immediately -> returns 10.
 %OptimizeFunctionOnNextCall(f2);
 assertEquals(102, f2(1000));  // 1 + f1(999) == 1 + 1 + worker1(998) == 102

test/mjsunit/regress/regress-crbug-147475.js

@@ -41,9 +41,9 @@ function factory(worker) {
 var f1 = factory(worker1);
 var f2 = factory(f1);
 %PrepareFunctionForOptimization(f1);
-%PrepareFunctionForOptimization(f2);
 assertEquals(11, f2(1));
 %OptimizeFunctionOnNextCall(f1);
+%PrepareFunctionForOptimization(f2);
 assertEquals(10, f1(0));
 %OptimizeFunctionOnNextCall(f2);
 assertEquals(102, f2(2));

test/mjsunit/regress/regress-crbug-638551.js

@@ -17,4 +17,5 @@ gc();  // Make sure that ...
 gc();  // ... code flushing ...
 gc();  // ... clears code ...
 gc();  // ... attached to {g}.
+%PrepareFunctionForOptimization(f);
 f();

tools/testrunner/base_runner.py

@@ -120,8 +120,9 @@ class ModeConfig(object):
     self.execution_mode = execution_mode
 
 
-DEBUG_FLAGS = ["--nohard-abort", "--enable-slow-asserts", "--verify-heap"]
-RELEASE_FLAGS = ["--nohard-abort"]
+DEBUG_FLAGS = ["--nohard-abort", "--enable-slow-asserts", "--verify-heap",
+               "--testing-d8-test-runner"]
+RELEASE_FLAGS = ["--nohard-abort", "--testing-d8-test-runner"]
 MODES = {
   "debug": ModeConfig(
     flags=DEBUG_FLAGS,

tools/unittests/testdata/expected_test_results1.json

@@ -4,7 +4,7 @@
   "mode": "release",
   "results": [
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "exit_code": 1,
       "expected": [
@@ -14,7 +14,8 @@
         "--test",
         "strawberries",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "framework_name": "standard_runner",
       "name": "sweet/strawberries",
@@ -22,13 +23,13 @@
       "result": "FAIL",
       "run": 1,
       "stderr": "",
-      "stdout": "--test strawberries --random-seed=123 --nohard-abort\n",
+      "stdout": "--test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
       "target_name": "d8_mocked.py",
       "variant": "default",
       "variant_flags": []
     },
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "exit_code": 1,
       "expected": [
@@ -38,7 +39,8 @@
         "--test",
         "strawberries",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "framework_name": "standard_runner",
       "name": "sweet/strawberries",
@@ -46,13 +48,13 @@
       "result": "FAIL",
       "run": 2,
       "stderr": "",
-      "stdout": "--test strawberries --random-seed=123 --nohard-abort\n",
+      "stdout": "--test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
       "target_name": "d8_mocked.py",
       "variant": "default",
       "variant_flags": []
     },
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "exit_code": 1,
       "expected": [
@@ -62,7 +64,8 @@
         "--test",
         "strawberries",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "framework_name": "standard_runner",
       "name": "sweet/strawberries",
@@ -70,7 +73,7 @@
       "result": "FAIL",
       "run": 3,
       "stderr": "",
-      "stdout": "--test strawberries --random-seed=123 --nohard-abort\n",
+      "stdout": "--test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
       "target_name": "d8_mocked.py",
       "variant": "default",
       "variant_flags": []
@@ -78,37 +81,40 @@
   ],
   "slowest_tests": [
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "flags": [
         "--test",
         "strawberries",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "marked_slow": true,
       "name": "sweet/strawberries"
     },
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "flags": [
         "--test",
         "strawberries",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "marked_slow": true,
       "name": "sweet/strawberries"
     },
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "flags": [
         "--test",
         "strawberries",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "marked_slow": true,
       "name": "sweet/strawberries"

tools/unittests/testdata/expected_test_results2.json

@@ -4,7 +4,7 @@
   "mode": "release",
   "results": [
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "exit_code": 1,
       "expected": [
@@ -13,7 +13,8 @@
       "flags": [
         "bananaflakes",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "framework_name": "standard_runner",
       "name": "sweet/bananaflakes",
@@ -21,13 +22,13 @@
       "result": "FAIL",
       "run": 1,
       "stderr": "",
-      "stdout": "bananaflakes --random-seed=123 --nohard-abort\n",
+      "stdout": "bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
       "target_name": "d8_mocked.py",
       "variant": "default",
       "variant_flags": []
     },
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "exit_code": 0,
       "expected": [
@@ -36,7 +37,8 @@
       "flags": [
         "bananaflakes",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "framework_name": "standard_runner",
       "name": "sweet/bananaflakes",
@@ -44,7 +46,7 @@
       "result": "PASS",
       "run": 2,
       "stderr": "",
-      "stdout": "bananaflakes --random-seed=123 --nohard-abort\n",
+      "stdout": "bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
       "target_name": "d8_mocked.py",
       "variant": "default",
       "variant_flags": []
@@ -52,23 +54,25 @@
   ],
   "slowest_tests": [
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "flags": [
         "bananaflakes",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "marked_slow": false,
       "name": "sweet/bananaflakes"
     },
     {
-      "command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
+      "command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
       "duration": 1,
       "flags": [
         "bananaflakes",
         "--random-seed=123",
-        "--nohard-abort"
+        "--nohard-abort",
+	"--testing-d8-test-runner"
       ],
       "marked_slow": false,
       "name": "sweet/bananaflakes"