[nci] Implement tier-up (part 3, spawn task & install)

This is the final part of the tier-up commit series. It implements:

- A prologue in NCI code objects that checks and acts upon the
optimization marker.
- Currently, handling is deferred to the InterpreterEntryTrampoline
but this will change in the future.
- The lifecycle is otherwise like Ignition-to-Turbofan; the runtime
profiler marks a function for optimization, the next call to that
function triggers optimization by calling into runtime, and the
finished code object is installed both on the JSFunction and the
optimized code cache.
- The feedback vector's kOptimizedCodeWeakOrSmiOffset slot is
currently reused for the mid-to-top tier up.

Cq-Include-Trybots: luci.v8.try:v8_linux64_fyi_rel_ng
Bug: v8:8888
Change-Id: Iff50b05ddcc68b25d7ed0f1e0d20af076a1522a0
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2361466Reviewed-by: Ross McIlroy <rmcilroy@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Reviewed-by: Mythri Alle <mythria@chromium.org>
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#69808}

src/codegen/compiler.cc

@@ -66,6 +66,27 @@ bool IsForNativeContextIndependentCachingOnly(CodeKind kind) {
          !FLAG_turbo_nci_as_midtier;
 }
 
+// This predicate is currently needed only because the nci-as-midtier testing
+// configuration is special. A quick summary of compilation configurations:
+//
+// - Turbofan (and currently Turboprop) uses both the optimization marker and
+// the optimized code cache (underneath, the marker and the cache share the same
+// slot on the feedback vector).
+// - Native context independent (NCI) code uses neither the marker nor the
+// cache.
+// - The NCI-as-midtier testing configuration uses the marker, but not the
+// cache.
+//
+// This predicate supports that last case. In the near future, this last case is
+// expected to change s.t. code kinds use the marker iff they use the optimized
+// code cache (details still TBD). In that case, the existing
+// CodeKindIsStoredInOptimizedCodeCache is sufficient and this extra predicate
+// can be removed.
+// TODO(jgruber,rmcilroy,v8:8888): Remove this predicate once that has happened.
+bool UsesOptimizationMarker(CodeKind kind) {
+  return !IsForNativeContextIndependentCachingOnly(kind);
+}
+
 class CompilerTracer : public AllStatic {
  public:
   static void PrintTracePrefix(const CodeTracer::Scope& scope,
@@ -843,7 +864,7 @@ V8_WARN_UNUSED_RESULT MaybeHandle<Code> GetCodeFromOptimizedCodeCache(
 }
 
 void ClearOptimizedCodeCache(OptimizedCompilationInfo* compilation_info) {
-  DCHECK(CodeKindIsStoredInOptimizedCodeCache(compilation_info->code_kind()));
+  DCHECK(UsesOptimizationMarker(compilation_info->code_kind()));
   Handle<JSFunction> function = compilation_info->closure();
   if (compilation_info->osr_offset().IsNone()) {
     Handle<FeedbackVector> vector =
@@ -855,7 +876,12 @@ void ClearOptimizedCodeCache(OptimizedCompilationInfo* compilation_info) {
 void InsertCodeIntoOptimizedCodeCache(
     OptimizedCompilationInfo* compilation_info) {
   const CodeKind kind = compilation_info->code_kind();
-  if (!CodeKindIsStoredInOptimizedCodeCache(kind)) return;
+  if (!CodeKindIsStoredInOptimizedCodeCache(kind)) {
+    if (UsesOptimizationMarker(kind)) {
+      ClearOptimizedCodeCache(compilation_info);
+    }
+    return;
+  }
 
   if (compilation_info->function_context_specializing()) {
     // Function context specialization folds-in the function context, so no
@@ -987,11 +1013,26 @@ bool GetOptimizedCodeLater(std::unique_ptr<OptimizedCompilationJob> job,
   if (CodeKindIsStoredInOptimizedCodeCache(code_kind)) {
     function->SetOptimizationMarker(OptimizationMarker::kInOptimizationQueue);
   }
-  DCHECK(function->ActiveTierIsIgnition());
+  DCHECK(function->ActiveTierIsIgnition() || function->ActiveTierIsNCI());
   DCHECK(function->shared().HasBytecodeArray());
   return true;
 }
 
+// Returns the code object at which execution continues after a concurrent
+// optimization job has been started (but not finished).
+Handle<Code> ContinuationForConcurrentOptimization(
+    Isolate* isolate, Handle<JSFunction> function) {
+  Handle<Code> cached_code;
+  if (FLAG_turbo_nci && function->NextTier() == CodeKindForTopTier() &&
+      GetCodeFromCompilationCache(isolate, handle(function->shared(), isolate))
+          .ToHandle(&cached_code)) {
+    // Tiering up to Turbofan and cached optimized code exists. Continue
+    // execution there until TF optimization has finished.
+    return cached_code;
+  }
+  return BUILTIN_CODE(isolate, InterpreterEntryTrampoline);
+}
+
 MaybeHandle<Code> GetOptimizedCode(Handle<JSFunction> function,
                                    ConcurrencyMode mode, CodeKind code_kind,
                                    BailoutId osr_offset = BailoutId::None(),
@@ -1006,8 +1047,7 @@ MaybeHandle<Code> GetOptimizedCode(Handle<JSFunction> function,
   // If compiling for NCI caching only (which does not use the optimization
   // marker), don't touch the marker to avoid interfering with Turbofan
   // compilation.
-  if (CodeKindIsStoredInOptimizedCodeCache(code_kind) &&
-      function->HasOptimizationMarker()) {
+  if (UsesOptimizationMarker(code_kind) && function->HasOptimizationMarker()) {
     function->ClearOptimizationMarker();
   }
 
@@ -1077,7 +1117,7 @@ MaybeHandle<Code> GetOptimizedCode(Handle<JSFunction> function,
   if (mode == ConcurrencyMode::kConcurrent) {
     if (GetOptimizedCodeLater(std::move(job), isolate, compilation_info,
                               code_kind, function)) {
-      return BUILTIN_CODE(isolate, InterpreterEntryTrampoline);
+      return ContinuationForConcurrentOptimization(isolate, function);
     }
   } else {
     DCHECK_EQ(mode, ConcurrencyMode::kNotConcurrent);
@@ -1837,7 +1877,8 @@ bool Compiler::CompileOptimized(Handle<JSFunction> function,
                                 ConcurrencyMode mode, CodeKind code_kind) {
   DCHECK(CodeKindIsOptimizedJSFunction(code_kind));
 
-  if (function->HasAttachedOptimizedCode()) return true;
+  // If the requested code kind is already available, do nothing.
+  if (function->HasAvailableCodeKind(code_kind)) return true;
 
   Isolate* isolate = function->GetIsolate();
   DCHECK(AllowCompilation::IsAllowed(isolate));
@@ -1860,7 +1901,7 @@ bool Compiler::CompileOptimized(Handle<JSFunction> function,
   DCHECK(!isolate->has_pending_exception());
   DCHECK(function->shared().is_compiled());
   DCHECK(function->is_compiled());
-  if (CodeKindIsStoredInOptimizedCodeCache(code_kind)) {
+  if (UsesOptimizationMarker(code_kind)) {
     DCHECK_IMPLIES(function->HasOptimizationMarker(),
                    function->IsInOptimizationQueue());
     DCHECK_IMPLIES(function->HasOptimizationMarker(),
@@ -2974,7 +3015,7 @@ bool Compiler::FinalizeOptimizedCompilationJob(OptimizedCompilationJob* job,
   CompilerTracer::TraceAbortedJob(isolate, compilation_info);
   compilation_info->closure()->set_code(shared->GetCode());
   // Clear the InOptimizationQueue marker, if it exists.
-  if (CodeKindIsStoredInOptimizedCodeCache(code_kind) &&
+  if (UsesOptimizationMarker(code_kind) &&
       compilation_info->closure()->IsInOptimizationQueue()) {
     compilation_info->closure()->ClearOptimizationMarker();
   }

src/compiler-dispatcher/optimizing-compile-dispatcher.cc

@@ -214,7 +214,7 @@ void OptimizingCompileDispatcher::InstallOptimizedFunctions() {
     }
     OptimizedCompilationInfo* info = job->compilation_info();
     Handle<JSFunction> function(*info->closure(), isolate_);
-    if (function->HasAvailableOptimizedCode()) {
+    if (function->HasAvailableCodeKind(info->code_kind())) {
       if (FLAG_trace_concurrent_recompilation) {
         PrintF("  ** Aborting compilation for ");
         function->ShortPrint();

src/compiler/access-builder.cc

@@ -1248,6 +1248,16 @@ FieldAccess AccessBuilder::ForFeedbackVectorClosureFeedbackCellArray() {
   return access;
 }
 
+// static
+FieldAccess AccessBuilder::ForFeedbackVectorOptimizedCodeWeakOrSmi() {
+  FieldAccess access = {
+      kTaggedBase,      FeedbackVector::kOptimizedCodeWeakOrSmiOffset,
+      Handle<Name>(),   MaybeHandle<Map>(),
+      Type::Any(),      MachineType::AnyTagged(),
+      kFullWriteBarrier};
+  return access;
+}
+
 }  // namespace compiler
 }  // namespace internal
 }  // namespace v8

src/compiler/access-builder.h

@@ -345,6 +345,7 @@ class V8_EXPORT_PRIVATE AccessBuilder final
 
   // Provides access to a FeedbackVector fields.
   static FieldAccess ForFeedbackVectorClosureFeedbackCellArray();
+  static FieldAccess ForFeedbackVectorOptimizedCodeWeakOrSmi();
 
  private:
   DISALLOW_IMPLICIT_CONSTRUCTORS(AccessBuilder);

src/compiler/backend/instruction-selector.cc

@@ -2923,9 +2923,9 @@ void InstructionSelector::VisitTailCall(Node* node) {
   auto call_descriptor = CallDescriptorOf(node->op());
 
   CallDescriptor* caller = linkage()->GetIncomingDescriptor();
-  DCHECK(caller->CanTailCall(CallDescriptorOf(node->op())));
   const CallDescriptor* callee = CallDescriptorOf(node->op());
-  int stack_param_delta = callee->GetStackParameterDelta(caller);
+  DCHECK(caller->CanTailCall(callee));
+  const int stack_param_delta = callee->GetStackParameterDelta(caller);
   CallBuffer buffer(zone(), call_descriptor, nullptr);
 
   // Compute InstructionOperands for inputs and outputs.
@@ -2942,7 +2942,7 @@ void InstructionSelector::VisitTailCall(Node* node) {
   // Select the appropriate opcode based on the call type.
   InstructionCode opcode;
   InstructionOperandVector temps(zone());
-  if (linkage()->GetIncomingDescriptor()->IsJSFunctionCall()) {
+  if (caller->IsJSFunctionCall()) {
     switch (call_descriptor->kind()) {
       case CallDescriptor::kCallCodeObject:
         opcode = kArchTailCallCodeObjectFromJSFunction;
@@ -2980,7 +2980,7 @@ void InstructionSelector::VisitTailCall(Node* node) {
   // instruction. This is used by backends that need to pad arguments for stack
   // alignment, in order to store an optional slot of padding above the
   // arguments.
-  int optional_padding_slot = callee->GetFirstUnusedStackSlot();
+  const int optional_padding_slot = callee->GetFirstUnusedStackSlot();
   buffer.instruction_args.push_back(g.TempImmediate(optional_padding_slot));
 
   const int first_unused_stack_slot =

src/compiler/backend/x64/code-generator-x64.cc

@@ -657,13 +657,33 @@ void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
 
 namespace {
 
-void AdjustStackPointerForTailCall(TurboAssembler* assembler,
+void AdjustStackPointerForTailCall(Instruction* instr,
+                                   TurboAssembler* assembler, Linkage* linkage,
+                                   OptimizedCompilationInfo* info,
                                    FrameAccessState* state,
                                    int new_slot_above_sp,
                                    bool allow_shrinkage = true) {
-  int current_sp_offset = state->GetSPToFPSlotCount() +
-                          StandardFrameConstants::kFixedSlotCountAboveFp;
-  int stack_slot_delta = new_slot_above_sp - current_sp_offset;
+  int stack_slot_delta;
+  if (HasCallDescriptorFlag(instr, CallDescriptor::kIsTailCallForTierUp)) {
+    // For this special tail-call mode, the callee has the same arguments and
+    // linkage as the caller, and arguments adapter frames must be preserved.
+    // Thus we simply have reset the stack pointer register to its original
+    // value before frame construction.
+    // See also: AssembleConstructFrame.
+    DCHECK(!info->is_osr());
+    DCHECK_EQ(linkage->GetIncomingDescriptor()->CalleeSavedRegisters(), 0);
+    DCHECK_EQ(linkage->GetIncomingDescriptor()->CalleeSavedFPRegisters(), 0);
+    DCHECK_EQ(state->frame()->GetReturnSlotCount(), 0);
+    stack_slot_delta = (state->frame()->GetTotalFrameSlotCount() -
+                        kReturnAddressStackSlotCount) *
+                       -1;
+    DCHECK_LE(stack_slot_delta, 0);
+  } else {
+    int current_sp_offset = state->GetSPToFPSlotCount() +
+                            StandardFrameConstants::kFixedSlotCountAboveFp;
+    stack_slot_delta = new_slot_above_sp - current_sp_offset;
+  }
+
   if (stack_slot_delta > 0) {
     assembler->AllocateStackSpace(stack_slot_delta * kSystemPointerSize);
     state->IncreaseSPDelta(stack_slot_delta);
@@ -690,12 +710,14 @@ void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
   if (!pushes.empty() &&
       (LocationOperand::cast(pushes.back()->destination()).index() + 1 ==
        first_unused_stack_slot)) {
+    DCHECK(!HasCallDescriptorFlag(instr, CallDescriptor::kIsTailCallForTierUp));
     X64OperandConverter g(this, instr);
     for (auto move : pushes) {
       LocationOperand destination_location(
           LocationOperand::cast(move->destination()));
       InstructionOperand source(move->source());
-      AdjustStackPointerForTailCall(tasm(), frame_access_state(),
+      AdjustStackPointerForTailCall(instr, tasm(), linkage(), info(),
+                                    frame_access_state(),
                                     destination_location.index());
       if (source.IsStackSlot()) {
         LocationOperand source_location(LocationOperand::cast(source));
@@ -713,14 +735,15 @@ void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
       move->Eliminate();
     }
   }
-  AdjustStackPointerForTailCall(tasm(), frame_access_state(),
-                                first_unused_stack_slot, false);
+  AdjustStackPointerForTailCall(instr, tasm(), linkage(), info(),
+                                frame_access_state(), first_unused_stack_slot,
+                                false);
 }
 
 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
                                              int first_unused_stack_slot) {
-  AdjustStackPointerForTailCall(tasm(), frame_access_state(),
-                                first_unused_stack_slot);
+  AdjustStackPointerForTailCall(instr, tasm(), linkage(), info(),
+                                frame_access_state(), first_unused_stack_slot);
 }
 
 // Check that {kJavaScriptCallCodeStartRegister} is correct.
@@ -824,12 +847,13 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       break;
     }
     case kArchTailCallCodeObjectFromJSFunction:
-    case kArchTailCallCodeObject: {
-      if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
+      if (!HasCallDescriptorFlag(instr, CallDescriptor::kIsTailCallForTierUp)) {
         AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
                                          i.TempRegister(0), i.TempRegister(1),
                                          i.TempRegister(2));
       }
+      V8_FALLTHROUGH;
+    case kArchTailCallCodeObject: {
       if (HasImmediateInput(instr, 0)) {
         Handle<Code> code = i.InputCode(0);
         __ Jump(code, RelocInfo::CODE_TARGET);
@@ -4372,7 +4396,7 @@ static const int kQuadWordSize = 16;
 }  // namespace
 
 void CodeGenerator::FinishFrame(Frame* frame) {
-  auto call_descriptor = linkage()->GetIncomingDescriptor();
+  CallDescriptor* call_descriptor = linkage()->GetIncomingDescriptor();
 
   const RegList saves_fp = call_descriptor->CalleeSavedFPRegisters();
   if (saves_fp != 0) {

src/compiler/bytecode-graph-builder.cc

@@ -39,7 +39,7 @@ class BytecodeGraphBuilder {
                        BailoutId osr_offset, JSGraph* jsgraph,
                        CallFrequency const& invocation_frequency,
                        SourcePositionTable* source_positions, int inlining_id,
-                       BytecodeGraphBuilderFlags flags,
+                       CodeKind code_kind, BytecodeGraphBuilderFlags flags,
                        TickCounter* tick_counter);
 
   // Creates a graph by visiting bytecodes.
@@ -63,8 +63,9 @@ class BytecodeGraphBuilder {
   // Get or create the node that represents the outer function closure.
   Node* GetFunctionClosure();
 
+  CodeKind code_kind() const { return code_kind_; }
   bool native_context_independent() const {
-    return native_context_independent_;
+    return CodeKindIsNativeContextIndependentJSFunction(code_kind_);
   }
 
   // The node representing the current feedback vector is generated once prior
@@ -97,6 +98,12 @@ class BytecodeGraphBuilder {
 
   Node* BuildLoadFeedbackCell(int index);
 
+  // Checks the optimization marker and potentially triggers compilation or
+  // installs the finished code object.
+  // Only relevant for specific code kinds (see
+  // CodeKindChecksOptimizationMarker).
+  void MaybeBuildTierUpCheck();
+
   // Builder for loading the a native context field.
   Node* BuildLoadNativeContextField(int index);
 
@@ -426,7 +433,7 @@ class BytecodeGraphBuilder {
   int input_buffer_size_;
   Node** input_buffer_;
 
-  const bool native_context_independent_;
+  const CodeKind code_kind_;
   Node* feedback_cell_node_;
   Node* feedback_vector_node_;
   Node* native_context_node_;
@@ -958,7 +965,7 @@ BytecodeGraphBuilder::BytecodeGraphBuilder(
     SharedFunctionInfoRef const& shared_info,
     FeedbackVectorRef const& feedback_vector, BailoutId osr_offset,
     JSGraph* jsgraph, CallFrequency const& invocation_frequency,
-    SourcePositionTable* source_positions, int inlining_id,
+    SourcePositionTable* source_positions, int inlining_id, CodeKind code_kind,
     BytecodeGraphBuilderFlags flags, TickCounter* tick_counter)
     : broker_(broker),
       local_zone_(local_zone),
@@ -997,8 +1004,7 @@ BytecodeGraphBuilder::BytecodeGraphBuilder(
       current_exception_handler_(0),
       input_buffer_size_(0),
       input_buffer_(nullptr),
-      native_context_independent_(
-          flags & BytecodeGraphBuilderFlag::kNativeContextIndependent),
+      code_kind_(code_kind),
       feedback_cell_node_(nullptr),
       feedback_vector_node_(nullptr),
       native_context_node_(nullptr),
@@ -1120,6 +1126,19 @@ Node* BytecodeGraphBuilder::BuildLoadNativeContext() {
   return native_context;
 }
 
+void BytecodeGraphBuilder::MaybeBuildTierUpCheck() {
+  if (!CodeKindChecksOptimizationMarker(code_kind())) return;
+
+  Environment* env = environment();
+  Node* control = env->GetControlDependency();
+  Node* effect = env->GetEffectDependency();
+
+  effect = graph()->NewNode(simplified()->TierUpCheck(), feedback_vector_node(),
+                            effect, control);
+
+  env->UpdateEffectDependency(effect);
+}
+
 Node* BytecodeGraphBuilder::BuildLoadNativeContextField(int index) {
   Node* result = NewNode(javascript()->LoadContext(0, index, true));
   NodeProperties::ReplaceContextInput(result, native_context_node());
@@ -1141,9 +1160,9 @@ void BytecodeGraphBuilder::CreateGraph() {
   // Set up the basic structure of the graph. Outputs for {Start} are the formal
   // parameters (including the receiver) plus new target, number of arguments,
   // context and closure.
-  int actual_parameter_count = StartNode::OutputArityForFormalParameterCount(
+  int start_output_arity = StartNode::OutputArityForFormalParameterCount(
       bytecode_array().parameter_count());
-  graph()->SetStart(graph()->NewNode(common()->Start(actual_parameter_count)));
+  graph()->SetStart(graph()->NewNode(common()->Start(start_output_arity)));
 
   Environment env(this, bytecode_array().register_count(),
                   bytecode_array().parameter_count(),
@@ -1153,7 +1172,9 @@ void BytecodeGraphBuilder::CreateGraph() {
 
   CreateFeedbackCellNode();
   CreateFeedbackVectorNode();
+  MaybeBuildTierUpCheck();
   CreateNativeContextNode();
+
   VisitBytecodes();
 
   // Finish the basic structure of the graph.
@@ -4402,13 +4423,14 @@ void BuildGraphFromBytecode(JSHeapBroker* broker, Zone* local_zone,
                             BailoutId osr_offset, JSGraph* jsgraph,
                             CallFrequency const& invocation_frequency,
                             SourcePositionTable* source_positions,
-                            int inlining_id, BytecodeGraphBuilderFlags flags,
+                            int inlining_id, CodeKind code_kind,
+                            BytecodeGraphBuilderFlags flags,
                             TickCounter* tick_counter) {
   DCHECK(broker->IsSerializedForCompilation(shared_info, feedback_vector));
   BytecodeGraphBuilder builder(
       broker, local_zone, broker->target_native_context(), shared_info,
       feedback_vector, osr_offset, jsgraph, invocation_frequency,
-      source_positions, inlining_id, flags, tick_counter);
+      source_positions, inlining_id, code_kind, flags, tick_counter);
   builder.CreateGraph();
 }
 

src/compiler/bytecode-graph-builder.h

@@ -7,8 +7,9 @@
 
 #include "src/compiler/js-operator.h"
 #include "src/compiler/js-type-hint-lowering.h"
-#include "src/utils/utils.h"
 #include "src/handles/handles.h"
+#include "src/objects/code-kind.h"
+#include "src/utils/utils.h"
 
 namespace v8 {
 
@@ -33,7 +34,6 @@ enum class BytecodeGraphBuilderFlag : uint8_t {
   // bytecode analysis.
   kAnalyzeEnvironmentLiveness = 1 << 1,
   kBailoutOnUninitialized = 1 << 2,
-  kNativeContextIndependent = 1 << 3,
 };
 using BytecodeGraphBuilderFlags = base::Flags<BytecodeGraphBuilderFlag>;
 
@@ -45,7 +45,8 @@ void BuildGraphFromBytecode(JSHeapBroker* broker, Zone* local_zone,
                             BailoutId osr_offset, JSGraph* jsgraph,
                             CallFrequency const& invocation_frequency,
                             SourcePositionTable* source_positions,
-                            int inlining_id, BytecodeGraphBuilderFlags flags,
+                            int inlining_id, CodeKind code_kind,
+                            BytecodeGraphBuilderFlags flags,
                             TickCounter* tick_counter);
 
 }  // namespace compiler

src/compiler/effect-control-linearizer.cc

@@ -179,7 +179,8 @@ class EffectControlLinearizer {
   void LowerCheckEqualsInternalizedString(Node* node, Node* frame_state);
   void LowerCheckEqualsSymbol(Node* node, Node* frame_state);
   Node* LowerTypeOf(Node* node);
-  Node* LowerUpdateInterruptBudget(Node* node);
+  void LowerTierUpCheck(Node* node);
+  void LowerUpdateInterruptBudget(Node* node);
   Node* LowerToBoolean(Node* node);
   Node* LowerPlainPrimitiveToNumber(Node* node);
   Node* LowerPlainPrimitiveToWord32(Node* node);
@@ -1140,8 +1141,11 @@ bool EffectControlLinearizer::TryWireInStateEffect(Node* node,
     case IrOpcode::kTypeOf:
       result = LowerTypeOf(node);
       break;
+    case IrOpcode::kTierUpCheck:
+      LowerTierUpCheck(node);
+      break;
     case IrOpcode::kUpdateInterruptBudget:
-      result = LowerUpdateInterruptBudget(node);
+      LowerUpdateInterruptBudget(node);
       break;
     case IrOpcode::kNewDoubleElements:
       result = LowerNewDoubleElements(node);
@@ -3734,7 +3738,82 @@ Node* EffectControlLinearizer::LowerTypeOf(Node* node) {
                  __ NoContextConstant());
 }
 
-Node* EffectControlLinearizer::LowerUpdateInterruptBudget(Node* node) {
+void EffectControlLinearizer::LowerTierUpCheck(Node* node) {
+  TierUpCheckNode n(node);
+  TNode<FeedbackVector> vector = n.feedback_vector();
+
+  Node* optimization_marker = __ LoadField(
+      AccessBuilder::ForFeedbackVectorOptimizedCodeWeakOrSmi(), vector);
+
+  // TODO(jgruber): The branch introduces a sequence of spills before the
+  // branch (and restores at `fallthrough`) that are completely unnecessary
+  // since the IfFalse continuation ends in a tail call. Investigate how to
+  // avoid these and fix it.
+
+  // TODO(jgruber): Combine the checks below for none/queued, e.g. by
+  // reorganizing OptimizationMarker values such that the least significant bit
+  // says whether the value is interesting or not. Also update the related
+  // check in the InterpreterEntryTrampoline.
+
+  auto fallthrough = __ MakeLabel();
+  auto optimization_marker_is_not_none = __ MakeDeferredLabel();
+  auto optimization_marker_is_neither_none_nor_queued = __ MakeDeferredLabel();
+  __ BranchWithHint(
+      __ TaggedEqual(optimization_marker, __ SmiConstant(static_cast<int>(
+                                              OptimizationMarker::kNone))),
+      &fallthrough, &optimization_marker_is_not_none, BranchHint::kTrue);
+
+  __ Bind(&optimization_marker_is_not_none);
+  __ BranchWithHint(
+      __ TaggedEqual(optimization_marker,
+                     __ SmiConstant(static_cast<int>(
+                         OptimizationMarker::kInOptimizationQueue))),
+      &fallthrough, &optimization_marker_is_neither_none_nor_queued,
+      BranchHint::kNone);
+
+  __ Bind(&optimization_marker_is_neither_none_nor_queued);
+
+  // The optimization marker field contains a non-trivial value, and some
+  // action has to be taken. For example, perhaps tier-up has been requested
+  // and we need to kick off a compilation job; or optimized code is available
+  // and should be tail-called.
+  //
+  // Currently we delegate these tasks to the InterpreterEntryTrampoline.
+  // TODO(jgruber,v8:8888): Consider a dedicated builtin instead.
+
+  const int parameter_count =
+      StartNode{graph()->start()}.FormalParameterCount();
+  TNode<HeapObject> code =
+      __ HeapConstant(BUILTIN_CODE(isolate(), InterpreterEntryTrampoline));
+  Node* target = __ Parameter(Linkage::kJSCallClosureParamIndex);
+  Node* new_target =
+      __ Parameter(Linkage::GetJSCallNewTargetParamIndex(parameter_count));
+  Node* argc =
+      __ Parameter(Linkage::GetJSCallArgCountParamIndex(parameter_count));
+  Node* context =
+      __ Parameter(Linkage::GetJSCallContextParamIndex(parameter_count));
+
+  JSTrampolineDescriptor descriptor;
+  CallDescriptor::Flags flags = CallDescriptor::kFixedTargetRegister |
+                                CallDescriptor::kIsTailCallForTierUp;
+  auto call_descriptor = Linkage::GetStubCallDescriptor(
+      graph()->zone(), descriptor, descriptor.GetStackParameterCount(), flags,
+      Operator::kNoProperties);
+  Node* nodes[] = {code,    target,      new_target,  argc,
+                   context, __ effect(), __ control()};
+
+#ifdef DEBUG
+  static constexpr int kCodeContextEffectControl = 4;
+  DCHECK_EQ(arraysize(nodes),
+            descriptor.GetParameterCount() + kCodeContextEffectControl);
+#endif  // DEBUG
+
+  __ TailCall(call_descriptor, arraysize(nodes), nodes);
+
+  __ Bind(&fallthrough);
+}
+
+void EffectControlLinearizer::LowerUpdateInterruptBudget(Node* node) {
   UpdateInterruptBudgetNode n(node);
   TNode<FeedbackCell> feedback_cell = n.feedback_cell();
   TNode<Int32T> budget = __ LoadField<Int32T>(
@@ -3755,7 +3834,6 @@ Node* EffectControlLinearizer::LowerUpdateInterruptBudget(Node* node) {
 
     __ Bind(&next);
   }
-  return nullptr;
 }
 
 Node* EffectControlLinearizer::LowerToBoolean(Node* node) {

src/compiler/graph-assembler.cc

@@ -32,6 +32,7 @@ class GraphAssembler::BasicBlockUpdater {
   void AddBranch(Node* branch, BasicBlock* tblock, BasicBlock* fblock);
   void AddGoto(BasicBlock* to);
   void AddGoto(BasicBlock* from, BasicBlock* to);
+  void AddTailCall(Node* node);
 
   void StartBlock(BasicBlock* block);
   BasicBlock* Finalize(BasicBlock* original);
@@ -267,6 +268,18 @@ void GraphAssembler::BasicBlockUpdater::AddGoto(BasicBlock* from,
   current_block_ = nullptr;
 }
 
+void GraphAssembler::BasicBlockUpdater::AddTailCall(Node* node) {
+  DCHECK_EQ(node->opcode(), IrOpcode::kTailCall);
+  DCHECK_NOT_NULL(current_block_);
+
+  if (state_ == kUnchanged) {
+    CopyForChange();
+  }
+
+  schedule_->AddTailCall(current_block_, node);
+  current_block_ = nullptr;
+}
+
 void GraphAssembler::BasicBlockUpdater::UpdateSuccessors(BasicBlock* block) {
   for (SuccessorInfo succ : saved_successors_) {
     (succ.block->predecessors())[succ.index] = block;
@@ -380,6 +393,11 @@ Node* GraphAssembler::ExternalConstant(ExternalReference ref) {
   return AddClonedNode(mcgraph()->ExternalConstant(ref));
 }
 
+Node* GraphAssembler::Parameter(int index) {
+  return AddNode(
+      graph()->NewNode(common()->Parameter(index), graph()->start()));
+}
+
 Node* JSGraphAssembler::CEntryStubConstant(int result_size) {
   return AddClonedNode(jsgraph()->CEntryStubConstant(result_size));
 }
@@ -781,6 +799,28 @@ TNode<Object> GraphAssembler::Call(const Operator* op, int inputs_size,
   return AddNode<Object>(graph()->NewNode(op, inputs_size, inputs));
 }
 
+void GraphAssembler::TailCall(const CallDescriptor* call_descriptor,
+                              int inputs_size, Node** inputs) {
+#ifdef DEBUG
+  static constexpr int kTargetEffectControl = 3;
+  DCHECK_EQ(inputs_size,
+            call_descriptor->ParameterCount() + kTargetEffectControl);
+#endif  // DEBUG
+
+  Node* node = AddNode(graph()->NewNode(common()->TailCall(call_descriptor),
+                                        inputs_size, inputs));
+
+  if (block_updater_) block_updater_->AddTailCall(node);
+
+  // Unlike ConnectUnreachableToEnd, the TailCall node terminates a block; to
+  // keep it live, it *must* be connected to End (also in Turboprop schedules).
+  NodeProperties::MergeControlToEnd(graph(), common(), node);
+
+  // Setting effect, control to nullptr effectively terminates the current block
+  // by disallowing the addition of new nodes until a new label has been bound.
+  InitializeEffectControl(nullptr, nullptr);
+}
+
 void GraphAssembler::BranchWithCriticalSafetyCheck(
     Node* condition, GraphAssemblerLabel<0u>* if_true,
     GraphAssemblerLabel<0u>* if_false) {

src/compiler/graph-assembler.h

@@ -240,6 +240,8 @@ class V8_EXPORT_PRIVATE GraphAssembler {
   Node* Projection(int index, Node* value);
   Node* ExternalConstant(ExternalReference ref);
 
+  Node* Parameter(int index);
+
   Node* LoadFramePointer();
 
   Node* LoadHeapNumberValue(Node* heap_number);
@@ -326,6 +328,8 @@ class V8_EXPORT_PRIVATE GraphAssembler {
                      Args... args);
   template <typename... Args>
   TNode<Object> Call(const Operator* op, Node* first_arg, Args... args);
+  void TailCall(const CallDescriptor* call_descriptor, int inputs_size,
+                Node** inputs);
 
   // Basic control operations.
   template <size_t VarCount>

src/compiler/js-inlining.cc

@@ -470,8 +470,8 @@ Reduction JSInliner::ReduceJSCall(Node* node) {
       CallFrequency frequency = call.frequency();
       BuildGraphFromBytecode(broker(), zone(), *shared_info, feedback_vector,
                              BailoutId::None(), jsgraph(), frequency,
-                             source_positions_, inlining_id, flags,
-                             &info_->tick_counter());
+                             source_positions_, inlining_id, info_->code_kind(),
+                             flags, &info_->tick_counter());
     }
 
     // Extract the inlinee start/end nodes.

src/compiler/linkage.cc

@@ -94,6 +94,12 @@ int CallDescriptor::GetFirstUnusedStackSlot() const {
 
 int CallDescriptor::GetStackParameterDelta(
     CallDescriptor const* tail_caller) const {
+  // In the IsTailCallForTierUp case, the callee has
+  // identical linkage and runtime arguments to the caller, thus the stack
+  // parameter delta is 0. We don't explicitly pass the runtime arguments as
+  // inputs to the TailCall node, since they already exist on the stack.
+  if (IsTailCallForTierUp()) return 0;
+
   int callee_slots_above_sp = GetFirstUnusedStackSlot();
   int tail_caller_slots_above_sp = tail_caller->GetFirstUnusedStackSlot();
   int stack_param_delta = callee_slots_above_sp - tail_caller_slots_above_sp;

src/compiler/linkage.h

@@ -226,13 +226,30 @@ class V8_EXPORT_PRIVATE CallDescriptor final
     // The kCallerSavedFPRegisters only matters (and set) when the more general
     // flag for kCallerSavedRegisters above is also set.
     kCallerSavedFPRegisters = 1u << 8,
-    // AIX has a function descriptor by default but it can be disabled for a
-    // certain CFunction call (only used for Kind::kCallAddress).
-    kNoFunctionDescriptor = 1u << 9,
+    // Tail calls for tier up are special (in fact they are different enough
+    // from normal tail calls to warrant a dedicated opcode; but they also have
+    // enough similar aspects that reusing the TailCall opcode is pragmatic).
+    // Specifically:
+    //
+    // 1. Caller and callee are both JS-linkage Code objects.
+    // 2. JS runtime arguments are passed unchanged from caller to callee.
+    // 3. JS runtime arguments are not attached as inputs to the TailCall node.
+    // 4. Prior to the tail call, frame and register state is torn down to just
+    //    before the caller frame was constructed.
+    // 5. Unlike normal tail calls, arguments adaptor frames (if present) are
+    //    *not* torn down.
+    //
+    // In other words, behavior is identical to a jmp instruction prior caller
+    // frame construction.
+    kIsTailCallForTierUp = 1u << 9,
 
     // Flags past here are *not* encoded in InstructionCode and are thus not
     // accessible from the code generator. See also
     // kFlagsBitsEncodedInInstructionCode.
+
+    // AIX has a function descriptor by default but it can be disabled for a
+    // certain CFunction call (only used for Kind::kCallAddress).
+    kNoFunctionDescriptor = 1u << 10,
   };
   using Flags = base::Flags<Flag>;
 
@@ -331,6 +348,7 @@ class V8_EXPORT_PRIVATE CallDescriptor final
   bool NeedsCallerSavedFPRegisters() const {
     return flags() & kCallerSavedFPRegisters;
   }
+  bool IsTailCallForTierUp() const { return flags() & kIsTailCallForTierUp; }
   bool NoFunctionDescriptor() const { return flags() & kNoFunctionDescriptor; }
 
   LinkageLocation GetReturnLocation(size_t index) const {

src/compiler/opcodes.h

@@ -483,6 +483,7 @@
   V(StringToLowerCaseIntl)              \
   V(StringToNumber)                     \
   V(StringToUpperCaseIntl)              \
+  V(TierUpCheck)                        \
   V(ToBoolean)                          \
   V(TransitionAndStoreElement)          \
   V(TransitionAndStoreNonNumberElement) \

src/compiler/pipeline.cc

@@ -1423,17 +1423,14 @@ struct GraphBuilderPhase {
     if (data->info()->bailout_on_uninitialized()) {
       flags |= BytecodeGraphBuilderFlag::kBailoutOnUninitialized;
     }
-    if (data->info()->IsNativeContextIndependent()) {
-      flags |= BytecodeGraphBuilderFlag::kNativeContextIndependent;
-    }
 
     JSFunctionRef closure(data->broker(), data->info()->closure());
     CallFrequency frequency(1.0f);
     BuildGraphFromBytecode(
         data->broker(), temp_zone, closure.shared(), closure.feedback_vector(),
         data->info()->osr_offset(), data->jsgraph(), frequency,
-        data->source_positions(), SourcePosition::kNotInlined, flags,
-        &data->info()->tick_counter());
+        data->source_positions(), SourcePosition::kNotInlined,
+        data->info()->code_kind(), flags, &data->info()->tick_counter());
   }
 };
 

src/compiler/simplified-lowering.cc

@@ -2821,6 +2821,7 @@ class RepresentationSelector {
         return VisitUnop<T>(node, UseInfo::AnyTagged(),
                             MachineRepresentation::kTaggedPointer);
       }
+      case IrOpcode::kTierUpCheck:
       case IrOpcode::kUpdateInterruptBudget: {
         ProcessInput<T>(node, 0, UseInfo::AnyTagged());
         ProcessRemainingInputs<T>(node, 1);

src/compiler/simplified-operator.cc

@@ -1322,6 +1322,12 @@ const Operator* SimplifiedOperatorBuilder::UpdateInterruptBudget(int delta) {
       "UpdateInterruptBudget", 1, 1, 1, 0, 1, 0, delta);
 }
 
+const Operator* SimplifiedOperatorBuilder::TierUpCheck() {
+  return zone()->New<Operator>(IrOpcode::kTierUpCheck,
+                               Operator::kNoThrow | Operator::kNoDeopt,
+                               "TierUpCheck", 1, 1, 1, 0, 1, 0);
+}
+
 const Operator* SimplifiedOperatorBuilder::AssertType(Type type) {
   DCHECK(type.IsRange());
   return zone()->New<Operator1<Type>>(IrOpcode::kAssertType,

src/compiler/simplified-operator.h

@@ -810,6 +810,11 @@ class V8_EXPORT_PRIVATE SimplifiedOperatorBuilder final
   // delta parameter represents the executed bytecodes since the last update.
   const Operator* UpdateInterruptBudget(int delta);
 
+  // Takes the current feedback vector as input 0, and generates a check of the
+  // vector's marker. Depending on the marker's value, we either do nothing,
+  // trigger optimized compilation, or install a finished code object.
+  const Operator* TierUpCheck();
+
   const Operator* ToBoolean();
 
   const Operator* StringConcat();
@@ -1165,6 +1170,18 @@ class FastApiCallNode final : public SimplifiedNodeWrapperBase {
   }
 };
 
+class TierUpCheckNode final : public SimplifiedNodeWrapperBase {
+ public:
+  explicit constexpr TierUpCheckNode(Node* node)
+      : SimplifiedNodeWrapperBase(node) {
+    CONSTEXPR_DCHECK(node->opcode() == IrOpcode::kTierUpCheck);
+  }
+
+#define INPUTS(V) V(FeedbackVector, feedback_vector, 0, FeedbackVector)
+  INPUTS(DEFINE_INPUT_ACCESSORS)
+#undef INPUTS
+};
+
 class UpdateInterruptBudgetNode final : public SimplifiedNodeWrapperBase {
  public:
   explicit constexpr UpdateInterruptBudgetNode(Node* node)

src/compiler/store-store-elimination.cc

@@ -396,7 +396,8 @@ UnobservablesSet RedundantStoreFinder::RecomputeUseIntersection(Node* node) {
     // Everything is observable after these opcodes; return the empty set.
     DCHECK_EXTRA(
         opcode == IrOpcode::kReturn || opcode == IrOpcode::kTerminate ||
-            opcode == IrOpcode::kDeoptimize || opcode == IrOpcode::kThrow,
+            opcode == IrOpcode::kDeoptimize || opcode == IrOpcode::kThrow ||
+            opcode == IrOpcode::kTailCall,
         "for #%d:%s", node->id(), node->op()->mnemonic());
     USE(opcode);
 

src/compiler/typer.cc

@@ -1196,6 +1196,7 @@ Type Typer::Visitor::TypeTypeOf(Node* node) {
   return Type::InternalizedString();
 }
 
+Type Typer::Visitor::TypeTierUpCheck(Node* node) { UNREACHABLE(); }
 Type Typer::Visitor::TypeUpdateInterruptBudget(Node* node) { UNREACHABLE(); }
 
 // JS conversion operators.

src/compiler/verifier.cc

@@ -763,6 +763,7 @@ void Verifier::Visitor::Check(Node* node, const AllNodes& all) {
     case IrOpcode::kTypeOf:
       CheckTypeIs(node, Type::InternalizedString());
       break;
+    case IrOpcode::kTierUpCheck:
     case IrOpcode::kUpdateInterruptBudget:
       CheckValueInputIs(node, 0, Type::Any());
       CheckNotTyped(node);

src/objects/code-kind.h

@@ -80,12 +80,9 @@ inline constexpr bool CodeKindChecksOptimizationMarker(CodeKind kind) {
 // The optimization marker field on the feedback vector has a dual purpose of
 // controlling the tier-up workflow, and caching the produced code object for
 // access from multiple closures. The marker is not used for all code kinds
-// though, in particular it is not used when generating NCI code for caching
-// only.
+// though, in particular it is not used when generating NCI code.
 inline constexpr bool CodeKindIsStoredInOptimizedCodeCache(CodeKind kind) {
-  return kind == CodeKind::OPTIMIZED_FUNCTION ||
-         (FLAG_turbo_nci_as_midtier &&
-          kind == CodeKind::NATIVE_CONTEXT_INDEPENDENT);
+  return kind == CodeKind::OPTIMIZED_FUNCTION;
 }
 
 inline CodeKind CodeKindForTopTier() { return CodeKind::OPTIMIZED_FUNCTION; }

src/objects/js-function.cc

@@ -94,6 +94,11 @@ bool JSFunction::HasAvailableOptimizedCode() const {
   return (result & kOptimizedJSFunctionCodeKindsMask) != 0;
 }
 
+bool JSFunction::HasAvailableCodeKind(CodeKind kind) const {
+  CodeKinds result = GetAvailableCodeKinds();
+  return (result & CodeKindToCodeKindFlag(kind)) != 0;
+}
+
 namespace {
 
 // Returns false if no highest tier exists (i.e. the function is not compiled),

src/objects/js-function.h

@@ -110,6 +110,8 @@ class JSFunction : public JSFunctionOrBoundFunction {
   V8_EXPORT_PRIVATE bool HasAttachedOptimizedCode() const;
   bool HasAvailableOptimizedCode() const;
 
+  bool HasAvailableCodeKind(CodeKind kind) const;
+
   V8_EXPORT_PRIVATE bool ActiveTierIsIgnition() const;
   bool ActiveTierIsTurbofan() const;
   bool ActiveTierIsNCI() const;

src/runtime/runtime-compiler.cc

@@ -66,6 +66,8 @@ Object CompileOptimized(Isolate* isolate, Handle<JSFunction> function,
     return isolate->StackOverflow();
   }
 
+  if (function->HasOptimizationMarker()) function->ClearOptimizationMarker();
+
   if (!Compiler::CompileOptimized(function, mode, function->NextTier())) {
     return ReadOnlyRoots(isolate).exception();
   }