[maglev] Mark SFIs when failing maglev compilation

Prevent thrashing tier up attempts when maglev fails to compile.

Bug: v8:7700
Change-Id: I31f6bd331d1c5aefa0384fcdcb055203b9055f8c
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3579143
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Auto-Submit: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Commit-Queue: Toon Verwaest <verwaest@chromium.org>
Cr-Commit-Position: refs/heads/main@{#79879}

src/execution/tiering-manager.cc

@@ -329,7 +329,8 @@ OptimizationDecision TieringManager::ShouldOptimize(JSFunction function,
                                                     JavaScriptFrame* frame) {
   DCHECK_EQ(code_kind, function.GetActiveTier().value());
 
-  if (TiersUpToMaglev(code_kind)) {
+  if (TiersUpToMaglev(code_kind) &&
+      !function.shared(isolate_).maglev_compilation_failed()) {
     return OptimizationDecision::Maglev();
   } else if (code_kind == CodeKind::TURBOFAN) {
     // Already in the top tier.

src/maglev/maglev-compiler.cc

@@ -195,11 +195,20 @@ void MaglevCompiler::Compile() {
 MaybeHandle<CodeT> MaglevCompiler::GenerateCode(
     MaglevCompilationUnit* toplevel_compilation_unit) {
   Graph* const graph = toplevel_compilation_unit->info()->graph();
-  if (graph == nullptr) return {};  // Compilation failed.
+  if (graph == nullptr) {
+    // Compilation failed.
+    toplevel_compilation_unit->shared_function_info()
+        .object()
+        ->set_maglev_compilation_failed(true);
+    return {};
+  }
 
   Handle<Code> code;
   if (!MaglevCodeGenerator::Generate(toplevel_compilation_unit, graph)
            .ToHandle(&code)) {
+    toplevel_compilation_unit->shared_function_info()
+        .object()
+        ->set_maglev_compilation_failed(true);
     return {};
   }
 

src/objects/shared-function-info-inl.h

@@ -261,6 +261,9 @@ BIT_FIELD_ACCESSORS(SharedFunctionInfo, flags2,
                     has_static_private_methods_or_accessors,
                     SharedFunctionInfo::HasStaticPrivateMethodsOrAccessorsBit)
 
+BIT_FIELD_ACCESSORS(SharedFunctionInfo, flags2, maglev_compilation_failed,
+                    SharedFunctionInfo::MaglevCompilationFailedBit)
+
 BIT_FIELD_ACCESSORS(SharedFunctionInfo, relaxed_flags, syntax_kind,
                     SharedFunctionInfo::FunctionSyntaxKindBits)
 

src/objects/shared-function-info.h

@@ -447,6 +447,8 @@ class SharedFunctionInfo
   DECL_BOOLEAN_ACCESSORS(class_scope_has_private_brand)
   DECL_BOOLEAN_ACCESSORS(has_static_private_methods_or_accessors)
 
+  DECL_BOOLEAN_ACCESSORS(maglev_compilation_failed)
+
   // Is this function a top-level function (scripts, evals).
   DECL_BOOLEAN_ACCESSORS(is_toplevel)
 

src/objects/shared-function-info.tq

@@ -44,6 +44,7 @@ bitfield struct SharedFunctionInfoFlags extends uint32 {
 bitfield struct SharedFunctionInfoFlags2 extends uint8 {
   class_scope_has_private_brand: bool: 1 bit;
   has_static_private_methods_or_accessors: bool: 1 bit;
+  maglev_compilation_failed: bool: 1 bit;
 }
 
 @generateBodyDescriptor