[turbofan] Disable concurrent inlining for OSR

Bug: v8:7790
Change-Id: Idf066adcd5c3dca3004e2eaa0d8fa389755720af
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1991490Reviewed-by: Michael Stanton <mvstanton@chromium.org>
Commit-Queue: Maya Lekova <mslekova@chromium.org>
Cr-Commit-Position: refs/heads/master@{#65671}

src/codegen/compiler.cc

@@ -865,7 +865,9 @@ MaybeHandle<Code> GetOptimizedCode(Handle<JSFunction> function,
   // tolerate the lack of a script without bytecode.
   DCHECK_IMPLIES(!has_script, shared->HasBytecodeArray());
   std::unique_ptr<OptimizedCompilationJob> job(
-      compiler::Pipeline::NewCompilationJob(isolate, function, has_script));
+      compiler::Pipeline::NewCompilationJob(
+          isolate, function, has_script,
+          FLAG_concurrent_inlining && osr_offset.IsNone()));
   OptimizedCompilationInfo* compilation_info = job->compilation_info();
 
   compilation_info->SetOptimizingForOsr(osr_offset, osr_frame);

src/codegen/optimized-compilation-info.h

@@ -64,7 +64,8 @@ class V8_EXPORT_PRIVATE OptimizedCompilationInfo final {
     kTraceHeapBroker = 1 << 17,
     kWasmRuntimeExceptionSupport = 1 << 18,
     kTurboControlFlowAwareAllocation = 1 << 19,
-    kTurboPreprocessRanges = 1 << 20
+    kTurboPreprocessRanges = 1 << 20,
+    kConcurrentInlining = 1 << 21,
   };
 
   // Construct a compilation info for optimized compilation.
@@ -93,6 +94,9 @@ class V8_EXPORT_PRIVATE OptimizedCompilationInfo final {
 
   // Flags used by optimized compilation.
 
+  void MarkAsConcurrentInlining() { SetFlag(kConcurrentInlining); }
+  bool is_concurrent_inlining() const { return GetFlag(kConcurrentInlining); }
+
   void MarkAsTurboControlFlowAwareAllocation() {
     SetFlag(kTurboControlFlowAwareAllocation);
   }

src/compiler/bytecode-graph-builder.cc

@@ -40,6 +40,7 @@ class BytecodeGraphBuilder {
                        CallFrequency const& invocation_frequency,
                        SourcePositionTable* source_positions, int inlining_id,
                        BytecodeGraphBuilderFlags flags,
+                       JSTypeHintLowering::Flags type_hint_lowering_flags,
                        TickCounter* tick_counter);
 
   // Creates a graph by visiting bytecodes.
@@ -369,6 +370,10 @@ class BytecodeGraphBuilder {
   NativeContextRef native_context() const { return native_context_; }
   SharedFunctionInfoRef shared_info() const { return shared_info_; }
 
+  bool should_disallow_heap_access() const {
+    return flags_ & BytecodeGraphBuilderFlag::kConcurrentInlining;
+  }
+
 #define DECLARE_VISIT_BYTECODE(name, ...) void Visit##name();
   BYTECODE_LIST(DECLARE_VISIT_BYTECODE)
 #undef DECLARE_VISIT_BYTECODE
@@ -433,6 +438,8 @@ class BytecodeGraphBuilder {
 
   SourcePosition const start_position_;
 
+  BytecodeGraphBuilderFlags const flags_;
+
   TickCounter* const tick_counter_;
 
   static int const kBinaryOperationHintIndex = 1;
@@ -940,7 +947,9 @@ BytecodeGraphBuilder::BytecodeGraphBuilder(
     FeedbackVectorRef const& feedback_vector, BailoutId osr_offset,
     JSGraph* jsgraph, CallFrequency const& invocation_frequency,
     SourcePositionTable* source_positions, int inlining_id,
-    BytecodeGraphBuilderFlags flags, TickCounter* tick_counter)
+    BytecodeGraphBuilderFlags flags,
+    JSTypeHintLowering::Flags type_hint_lowering_flags,
+    TickCounter* tick_counter)
     : broker_(broker),
       local_zone_(local_zone),
       jsgraph_(jsgraph),
@@ -948,11 +957,8 @@ BytecodeGraphBuilder::BytecodeGraphBuilder(
       shared_info_(shared_info),
       feedback_vector_(feedback_vector),
       invocation_frequency_(invocation_frequency),
-      type_hint_lowering_(
-          broker, jsgraph, feedback_vector,
-          (flags & BytecodeGraphBuilderFlag::kBailoutOnUninitialized)
-              ? JSTypeHintLowering::kBailoutOnUninitialized
-              : JSTypeHintLowering::kNoFlags),
+      type_hint_lowering_(broker, jsgraph, feedback_vector,
+                          type_hint_lowering_flags),
       frame_state_function_info_(common()->CreateFrameStateFunctionInfo(
           FrameStateType::kInterpretedFunction,
           bytecode_array().parameter_count(), bytecode_array().register_count(),
@@ -962,8 +968,9 @@ BytecodeGraphBuilder::BytecodeGraphBuilder(
       bytecode_analysis_(broker_->GetBytecodeAnalysis(
           bytecode_array().object(), osr_offset,
           flags & BytecodeGraphBuilderFlag::kAnalyzeEnvironmentLiveness,
-          FLAG_concurrent_inlining ? SerializationPolicy::kAssumeSerialized
-                                   : SerializationPolicy::kSerializeIfNeeded)),
+          (flags & BytecodeGraphBuilderFlag::kConcurrentInlining)
+              ? SerializationPolicy::kAssumeSerialized
+              : SerializationPolicy::kSerializeIfNeeded)),
       environment_(nullptr),
       osr_(!osr_offset.IsNone()),
       currently_peeled_loop_offset_(-1),
@@ -980,8 +987,9 @@ BytecodeGraphBuilder::BytecodeGraphBuilder(
       state_values_cache_(jsgraph),
       source_positions_(source_positions),
       start_position_(shared_info.StartPosition(), inlining_id),
+      flags_(flags),
       tick_counter_(tick_counter) {
-  if (FLAG_concurrent_inlining) {
+  if (flags & BytecodeGraphBuilderFlag::kConcurrentInlining) {
     // With concurrent inlining on, the source position address doesn't change
     // because it's been copied from the heap.
     source_position_iterator_ = std::make_unique<SourcePositionTableIterator>(
@@ -1018,7 +1026,7 @@ FeedbackSource BytecodeGraphBuilder::CreateFeedbackSource(int slot_id) {
 }
 
 void BytecodeGraphBuilder::CreateGraph() {
-  DisallowHeapAccessIf disallow_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf disallow_heap_access(should_disallow_heap_access());
   SourcePositionTable::Scope pos_scope(source_positions_, start_position_);
 
   // Set up the basic structure of the graph. Outputs for {Start} are the formal
@@ -1313,7 +1321,7 @@ void BytecodeGraphBuilder::VisitBytecodes() {
     VisitSingleBytecode();
   }
 
-  if (!FLAG_concurrent_inlining && has_one_shot_bytecode) {
+  if (!should_disallow_heap_access() && has_one_shot_bytecode) {
     // (For concurrent inlining this is done in the serializer instead.)
     isolate()->CountUsage(
         v8::Isolate::UseCounterFeature::kOptimizedFunctionWithOneShotBytecode);
@@ -2007,7 +2015,7 @@ void BytecodeGraphBuilder::VisitCreateClosure() {
 }
 
 void BytecodeGraphBuilder::VisitCreateBlockContext() {
-  DisallowHeapAccessIf no_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf no_heap_access(should_disallow_heap_access());
   ScopeInfoRef scope_info(
       broker(), bytecode_iterator().GetConstantForIndexOperand(0, isolate()));
   const Operator* op = javascript()->CreateBlockContext(scope_info.object());
@@ -2016,7 +2024,7 @@ void BytecodeGraphBuilder::VisitCreateBlockContext() {
 }
 
 void BytecodeGraphBuilder::VisitCreateFunctionContext() {
-  DisallowHeapAccessIf no_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf no_heap_access(should_disallow_heap_access());
   ScopeInfoRef scope_info(
       broker(), bytecode_iterator().GetConstantForIndexOperand(0, isolate()));
   uint32_t slots = bytecode_iterator().GetUnsignedImmediateOperand(1);
@@ -2027,7 +2035,7 @@ void BytecodeGraphBuilder::VisitCreateFunctionContext() {
 }
 
 void BytecodeGraphBuilder::VisitCreateEvalContext() {
-  DisallowHeapAccessIf no_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf no_heap_access(should_disallow_heap_access());
   ScopeInfoRef scope_info(
       broker(), bytecode_iterator().GetConstantForIndexOperand(0, isolate()));
   uint32_t slots = bytecode_iterator().GetUnsignedImmediateOperand(1);
@@ -2038,7 +2046,7 @@ void BytecodeGraphBuilder::VisitCreateEvalContext() {
 }
 
 void BytecodeGraphBuilder::VisitCreateCatchContext() {
-  DisallowHeapAccessIf no_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf no_heap_access(should_disallow_heap_access());
   interpreter::Register reg = bytecode_iterator().GetRegisterOperand(0);
   Node* exception = environment()->LookupRegister(reg);
   ScopeInfoRef scope_info(
@@ -2050,7 +2058,7 @@ void BytecodeGraphBuilder::VisitCreateCatchContext() {
 }
 
 void BytecodeGraphBuilder::VisitCreateWithContext() {
-  DisallowHeapAccessIf no_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf no_heap_access(should_disallow_heap_access());
   Node* object =
       environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
   ScopeInfoRef scope_info(
@@ -2157,7 +2165,7 @@ void BytecodeGraphBuilder::VisitCloneObject() {
 }
 
 void BytecodeGraphBuilder::VisitGetTemplateObject() {
-  DisallowHeapAccessIf no_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf no_heap_access(should_disallow_heap_access());
   FeedbackSource source =
       CreateFeedbackSource(bytecode_iterator().GetIndexOperand(1));
   TemplateObjectDescriptionRef description(
@@ -4160,10 +4168,17 @@ void BuildGraphFromBytecode(JSHeapBroker* broker, Zone* local_zone,
                             int inlining_id, BytecodeGraphBuilderFlags flags,
                             TickCounter* tick_counter) {
   DCHECK(broker->IsSerializedForCompilation(shared_info, feedback_vector));
+  JSTypeHintLowering::Flags type_hint_lowering_flags =
+      JSTypeHintLowering::kNoFlags;
+  if (flags & BytecodeGraphBuilderFlag::kBailoutOnUninitialized) {
+    type_hint_lowering_flags |= JSTypeHintLowering::kBailoutOnUninitialized;
+  }
+
   BytecodeGraphBuilder builder(
       broker, local_zone, broker->target_native_context(), shared_info,
       feedback_vector, osr_offset, jsgraph, invocation_frequency,
-      source_positions, inlining_id, flags, tick_counter);
+      source_positions, inlining_id, flags, type_hint_lowering_flags,
+      tick_counter);
   builder.CreateGraph();
 }
 

src/compiler/bytecode-graph-builder.h

@@ -33,6 +33,7 @@ enum class BytecodeGraphBuilderFlag : uint8_t {
   // bytecode analysis.
   kAnalyzeEnvironmentLiveness = 1 << 1,
   kBailoutOnUninitialized = 1 << 2,
+  kConcurrentInlining = 1 << 3,
 };
 using BytecodeGraphBuilderFlags = base::Flags<BytecodeGraphBuilderFlag>;
 

src/compiler/js-call-reducer.h

@@ -40,7 +40,11 @@ class SimplifiedOperatorBuilder;
 class V8_EXPORT_PRIVATE JSCallReducer final : public AdvancedReducer {
  public:
   // Flags that control the mode of operation.
-  enum Flag { kNoFlags = 0u, kBailoutOnUninitialized = 1u << 0 };
+  enum Flag {
+    kNoFlags = 0u,
+    kBailoutOnUninitialized = 1u << 0,
+    kConcurrentInlining = 1u << 1
+  };
   using Flags = base::Flags<Flag>;
 
   JSCallReducer(Editor* editor, JSGraph* jsgraph, JSHeapBroker* broker,
@@ -246,6 +250,10 @@ class V8_EXPORT_PRIVATE JSCallReducer final : public AdvancedReducer {
   Flags flags() const { return flags_; }
   CompilationDependencies* dependencies() const { return dependencies_; }
 
+  bool should_disallow_heap_access() const {
+    return flags() & kConcurrentInlining;
+  }
+
   JSGraph* const jsgraph_;
   JSHeapBroker* const broker_;
   Zone* const temp_zone_;

src/compiler/js-heap-broker.cc

@@ -2343,7 +2343,7 @@ base::Optional<ObjectRef> ContextRef::get(int index,
 }
 
 JSHeapBroker::JSHeapBroker(Isolate* isolate, Zone* broker_zone,
-                           bool tracing_enabled)
+                           bool tracing_enabled, bool is_concurrent_inlining)
     : isolate_(isolate),
       zone_(broker_zone),
       refs_(new (zone())
@@ -2351,6 +2351,7 @@ JSHeapBroker::JSHeapBroker(Isolate* isolate, Zone* broker_zone,
       root_index_map_(isolate),
       array_and_object_prototypes_(zone()),
       tracing_enabled_(tracing_enabled),
+      is_concurrent_inlining_(is_concurrent_inlining),
       feedback_(zone()),
       bytecode_analyses_(zone()),
       property_access_infos_(zone()),
@@ -4598,7 +4599,7 @@ ProcessedFeedback const& JSHeapBroker::GetFeedback(
 
 FeedbackSlotKind JSHeapBroker::GetFeedbackSlotKind(
     FeedbackSource const& source) const {
-  if (FLAG_concurrent_inlining) {
+  if (is_concurrent_inlining_) {
     ProcessedFeedback const& processed = GetFeedback(source);
     return processed.slot_kind();
   }
@@ -4607,7 +4608,7 @@ FeedbackSlotKind JSHeapBroker::GetFeedbackSlotKind(
 }
 
 bool JSHeapBroker::FeedbackIsInsufficient(FeedbackSource const& source) const {
-  return FLAG_concurrent_inlining
+  return (is_concurrent_inlining_)
              ? GetFeedback(source).IsInsufficient()
              : FeedbackNexus(source.vector, source.slot).IsUninitialized();
 }
@@ -4806,8 +4807,8 @@ ProcessedFeedback const& JSHeapBroker::ReadFeedbackForCall(
 BinaryOperationHint JSHeapBroker::GetFeedbackForBinaryOperation(
     FeedbackSource const& source) {
   ProcessedFeedback const& feedback =
-      FLAG_concurrent_inlining ? GetFeedback(source)
-                               : ProcessFeedbackForBinaryOperation(source);
+      (is_concurrent_inlining_) ? GetFeedback(source)
+                                : ProcessFeedbackForBinaryOperation(source);
   return feedback.IsInsufficient() ? BinaryOperationHint::kNone
                                    : feedback.AsBinaryOperation().value();
 }
@@ -4815,14 +4816,14 @@ BinaryOperationHint JSHeapBroker::GetFeedbackForBinaryOperation(
 CompareOperationHint JSHeapBroker::GetFeedbackForCompareOperation(
     FeedbackSource const& source) {
   ProcessedFeedback const& feedback =
-      FLAG_concurrent_inlining ? GetFeedback(source)
-                               : ProcessFeedbackForCompareOperation(source);
+      (is_concurrent_inlining_) ? GetFeedback(source)
+                                : ProcessFeedbackForCompareOperation(source);
   return feedback.IsInsufficient() ? CompareOperationHint::kNone
                                    : feedback.AsCompareOperation().value();
 }
 
 ForInHint JSHeapBroker::GetFeedbackForForIn(FeedbackSource const& source) {
-  ProcessedFeedback const& feedback = FLAG_concurrent_inlining
+  ProcessedFeedback const& feedback = (is_concurrent_inlining_)
                                           ? GetFeedback(source)
                                           : ProcessFeedbackForForIn(source);
   return feedback.IsInsufficient() ? ForInHint::kNone
@@ -4832,46 +4833,46 @@ ForInHint JSHeapBroker::GetFeedbackForForIn(FeedbackSource const& source) {
 ProcessedFeedback const& JSHeapBroker::GetFeedbackForPropertyAccess(
     FeedbackSource const& source, AccessMode mode,
     base::Optional<NameRef> static_name) {
-  return FLAG_concurrent_inlining
+  return (is_concurrent_inlining_)
              ? GetFeedback(source)
              : ProcessFeedbackForPropertyAccess(source, mode, static_name);
 }
 
 ProcessedFeedback const& JSHeapBroker::GetFeedbackForInstanceOf(
     FeedbackSource const& source) {
-  return FLAG_concurrent_inlining ? GetFeedback(source)
-                                  : ProcessFeedbackForInstanceOf(source);
+  return (is_concurrent_inlining_) ? GetFeedback(source)
+                                   : ProcessFeedbackForInstanceOf(source);
 }
 
 ProcessedFeedback const& JSHeapBroker::GetFeedbackForCall(
     FeedbackSource const& source) {
-  return FLAG_concurrent_inlining ? GetFeedback(source)
-                                  : ProcessFeedbackForCall(source);
+  return (is_concurrent_inlining_) ? GetFeedback(source)
+                                   : ProcessFeedbackForCall(source);
 }
 
 ProcessedFeedback const& JSHeapBroker::GetFeedbackForGlobalAccess(
     FeedbackSource const& source) {
-  return FLAG_concurrent_inlining ? GetFeedback(source)
-                                  : ProcessFeedbackForGlobalAccess(source);
+  return (is_concurrent_inlining_) ? GetFeedback(source)
+                                   : ProcessFeedbackForGlobalAccess(source);
 }
 
 ProcessedFeedback const& JSHeapBroker::GetFeedbackForArrayOrObjectLiteral(
     FeedbackSource const& source) {
-  return FLAG_concurrent_inlining
+  return (is_concurrent_inlining_)
              ? GetFeedback(source)
              : ProcessFeedbackForArrayOrObjectLiteral(source);
 }
 
 ProcessedFeedback const& JSHeapBroker::GetFeedbackForRegExpLiteral(
     FeedbackSource const& source) {
-  return FLAG_concurrent_inlining ? GetFeedback(source)
-                                  : ProcessFeedbackForRegExpLiteral(source);
+  return (is_concurrent_inlining_) ? GetFeedback(source)
+                                   : ProcessFeedbackForRegExpLiteral(source);
 }
 
 ProcessedFeedback const& JSHeapBroker::GetFeedbackForTemplateObject(
     FeedbackSource const& source) {
-  return FLAG_concurrent_inlining ? GetFeedback(source)
-                                  : ProcessFeedbackForTemplateObject(source);
+  return (is_concurrent_inlining_) ? GetFeedback(source)
+                                   : ProcessFeedbackForTemplateObject(source);
 }
 
 ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForArrayOrObjectLiteral(
@@ -5094,7 +5095,7 @@ PropertyAccessInfo JSHeapBroker::GetPropertyAccessInfo(
   AccessInfoFactory factory(this, dependencies, zone());
   PropertyAccessInfo access_info = factory.ComputePropertyAccessInfo(
       map.object(), name.object(), access_mode);
-  if (FLAG_concurrent_inlining) {
+  if (is_concurrent_inlining_) {
     CHECK(SerializingAllowed());
     TRACE(this, "Storing PropertyAccessInfo for "
                     << access_mode << " of property " << name << " on map "

src/compiler/js-heap-broker.h

@@ -73,7 +73,8 @@ struct PropertyAccessTarget {
 
 class V8_EXPORT_PRIVATE JSHeapBroker {
  public:
-  JSHeapBroker(Isolate* isolate, Zone* broker_zone, bool tracing_enabled);
+  JSHeapBroker(Isolate* isolate, Zone* broker_zone, bool tracing_enabled,
+               bool is_concurrent_inlining);
 
   // The compilation target's native context. We need the setter because at
   // broker construction time we don't yet have the canonical handle.
@@ -242,6 +243,7 @@ class V8_EXPORT_PRIVATE JSHeapBroker {
       array_and_object_prototypes_;
   BrokerMode mode_ = kDisabled;
   bool const tracing_enabled_;
+  bool const is_concurrent_inlining_;
   mutable StdoutStream trace_out_;
   unsigned trace_indentation_ = 0;
   PerIsolateCompilerCache* compiler_cache_ = nullptr;

src/compiler/js-inlining-heuristic.cc

@@ -127,7 +127,7 @@ JSInliningHeuristic::Candidate JSInliningHeuristic::CollectFunctions(
 }
 
 Reduction JSInliningHeuristic::Reduce(Node* node) {
-  DisallowHeapAccessIf no_heap_acess(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf no_heap_acess(info_->is_concurrent_inlining());
 
   if (!IrOpcode::IsInlineeOpcode(node->opcode())) return NoChange();
 
@@ -222,7 +222,7 @@ Reduction JSInliningHeuristic::Reduce(Node* node) {
 }
 
 void JSInliningHeuristic::Finalize() {
-  DisallowHeapAccessIf no_heap_acess(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf no_heap_acess(info_->is_concurrent_inlining());
 
   if (candidates_.empty()) return;  // Nothing to do without candidates.
   if (FLAG_trace_turbo_inlining) PrintCandidates();

src/compiler/js-inlining-heuristic.h

@@ -22,6 +22,7 @@ class JSInliningHeuristic final : public AdvancedReducer {
         candidates_(local_zone),
         seen_(local_zone),
         source_positions_(source_positions),
+        info_(info),
         jsgraph_(jsgraph),
         broker_(broker) {}
 
@@ -92,6 +93,7 @@ class JSInliningHeuristic final : public AdvancedReducer {
   Candidates candidates_;
   ZoneSet<NodeId> seen_;
   SourcePositionTable* source_positions_;
+  OptimizedCompilationInfo* info_;
   JSGraph* const jsgraph_;
   JSHeapBroker* const broker_;
   int total_inlined_bytecode_size_ = 0;

src/compiler/js-inlining.cc

@@ -419,7 +419,8 @@ Reduction JSInliner::ReduceJSCall(Node* node) {
   // always hold true.
   CHECK(shared_info->is_compiled());
 
-  if (!FLAG_concurrent_inlining && info_->is_source_positions_enabled()) {
+  if (!info_->is_concurrent_inlining() &&
+      info_->is_source_positions_enabled()) {
     SharedFunctionInfo::EnsureSourcePositionsAvailable(isolate(),
                                                        shared_info->object());
   }
@@ -457,6 +458,9 @@ Reduction JSInliner::ReduceJSCall(Node* node) {
     if (info_->is_bailout_on_uninitialized()) {
       flags |= BytecodeGraphBuilderFlag::kBailoutOnUninitialized;
     }
+    if (info_->is_concurrent_inlining()) {
+      flags |= BytecodeGraphBuilderFlag::kConcurrentInlining;
+    }
     {
       CallFrequency frequency = call.frequency();
       BuildGraphFromBytecode(broker(), zone(), *shared_info, feedback_vector,

src/compiler/js-intrinsic-lowering.cc

@@ -22,11 +22,14 @@ namespace internal {
 namespace compiler {
 
 JSIntrinsicLowering::JSIntrinsicLowering(Editor* editor, JSGraph* jsgraph,
-                                         JSHeapBroker* broker)
-    : AdvancedReducer(editor), jsgraph_(jsgraph), broker_(broker) {}
+                                         JSHeapBroker* broker, Flags flags)
+    : AdvancedReducer(editor),
+      jsgraph_(jsgraph),
+      broker_(broker),
+      flags_(flags) {}
 
 Reduction JSIntrinsicLowering::Reduce(Node* node) {
-  DisallowHeapAccessIf no_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf no_heap_access(flags_ & kConcurrentInlining);
 
   if (node->opcode() != IrOpcode::kJSCallRuntime) return NoChange();
   const Runtime::Function* const f =

src/compiler/js-intrinsic-lowering.h

@@ -31,7 +31,12 @@ class SimplifiedOperatorBuilder;
 class V8_EXPORT_PRIVATE JSIntrinsicLowering final
     : public NON_EXPORTED_BASE(AdvancedReducer) {
  public:
-  JSIntrinsicLowering(Editor* editor, JSGraph* jsgraph, JSHeapBroker* broker);
+  // Flags that control the mode of operation.
+  enum Flag { kNoFlags = 0u, kConcurrentInlining = 1u << 0 };
+  using Flags = base::Flags<Flag>;
+
+  JSIntrinsicLowering(Editor* editor, JSGraph* jsgraph, JSHeapBroker* broker,
+                      Flags flags);
   ~JSIntrinsicLowering() final = default;
 
   const char* reducer_name() const override { return "JSIntrinsicLowering"; }
@@ -90,6 +95,7 @@ class V8_EXPORT_PRIVATE JSIntrinsicLowering final
 
   JSGraph* const jsgraph_;
   JSHeapBroker* const broker_;
+  Flags const flags_;
 };
 
 }  // namespace compiler

src/compiler/js-native-context-specialization.cc

@@ -69,7 +69,7 @@ JSNativeContextSpecialization::JSNativeContextSpecialization(
       type_cache_(TypeCache::Get()) {}
 
 Reduction JSNativeContextSpecialization::Reduce(Node* node) {
-  DisallowHeapAccessIf disallow_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf disallow_heap_access(should_disallow_heap_access());
 
   switch (node->opcode()) {
     case IrOpcode::kJSAdd:
@@ -351,7 +351,7 @@ Reduction JSNativeContextSpecialization::ReduceJSGetSuperConstructor(
   if (!m.HasValue()) return NoChange();
   JSFunctionRef function = m.Ref(broker()).AsJSFunction();
   MapRef function_map = function.map();
-  if (FLAG_concurrent_inlining && !function_map.serialized_prototype()) {
+  if (should_disallow_heap_access() && !function_map.serialized_prototype()) {
     TRACE_BROKER_MISSING(broker(), "data for map " << function_map);
     return NoChange();
   }
@@ -403,7 +403,7 @@ Reduction JSNativeContextSpecialization::ReduceJSInstanceOf(Node* node) {
   MapRef receiver_map = receiver_ref.map();
 
   PropertyAccessInfo access_info = PropertyAccessInfo::Invalid(graph()->zone());
-  if (FLAG_concurrent_inlining) {
+  if (should_disallow_heap_access()) {
     access_info = broker()->GetPropertyAccessInfo(
         receiver_map,
         NameRef(broker(), isolate()->factory()->has_instance_symbol()),
@@ -529,7 +529,7 @@ JSNativeContextSpecialization::InferHasInPrototypeChain(
         all = false;
         break;
       }
-      if (FLAG_concurrent_inlining && !map.serialized_prototype()) {
+      if (should_disallow_heap_access() && !map.serialized_prototype()) {
         TRACE_BROKER_MISSING(broker(), "prototype data for map " << map);
         return kMayBeInPrototypeChain;
       }
@@ -608,7 +608,7 @@ Reduction JSNativeContextSpecialization::ReduceJSOrdinaryHasInstance(
     // OrdinaryHasInstance on bound functions turns into a recursive invocation
     // of the instanceof operator again.
     JSBoundFunctionRef function = m.Ref(broker()).AsJSBoundFunction();
-    if (FLAG_concurrent_inlining && !function.serialized()) {
+    if (should_disallow_heap_access() && !function.serialized()) {
       TRACE_BROKER_MISSING(broker(), "data for JSBoundFunction " << function);
       return NoChange();
     }
@@ -627,7 +627,7 @@ Reduction JSNativeContextSpecialization::ReduceJSOrdinaryHasInstance(
     // Optimize if we currently know the "prototype" property.
 
     JSFunctionRef function = m.Ref(broker()).AsJSFunction();
-    if (FLAG_concurrent_inlining && !function.serialized()) {
+    if (should_disallow_heap_access() && !function.serialized()) {
       TRACE_BROKER_MISSING(broker(), "data for JSFunction " << function);
       return NoChange();
     }
@@ -706,7 +706,7 @@ Reduction JSNativeContextSpecialization::ReduceJSResolvePromise(Node* node) {
   ZoneVector<PropertyAccessInfo> access_infos(graph()->zone());
   AccessInfoFactory access_info_factory(broker(), dependencies(),
                                         graph()->zone());
-  if (!FLAG_concurrent_inlining) {
+  if (!should_disallow_heap_access()) {
     access_info_factory.ComputePropertyAccessInfos(
         resolution_maps, factory()->then_string(), AccessMode::kLoad,
         &access_infos);
@@ -1078,8 +1078,9 @@ Reduction JSNativeContextSpecialization::ReduceNamedAccess(
       if (map.is_deprecated()) continue;
       PropertyAccessInfo access_info = broker()->GetPropertyAccessInfo(
           map, feedback.name(), access_mode, dependencies(),
-          FLAG_concurrent_inlining ? SerializationPolicy::kAssumeSerialized
-                                   : SerializationPolicy::kSerializeIfNeeded);
+          should_disallow_heap_access()
+              ? SerializationPolicy::kAssumeSerialized
+              : SerializationPolicy::kSerializeIfNeeded);
       access_infos_for_feedback.push_back(access_info);
     }
 
@@ -1326,7 +1327,7 @@ Reduction JSNativeContextSpecialization::ReduceJSLoadNamed(Node* node) {
         name.equals(ObjectRef(broker(), factory()->prototype_string()))) {
       // Optimize "prototype" property of functions.
       JSFunctionRef function = object.AsJSFunction();
-      if (FLAG_concurrent_inlining && !function.serialized()) {
+      if (should_disallow_heap_access() && !function.serialized()) {
         TRACE_BROKER_MISSING(broker(), "data for function " << function);
         return NoChange();
       }
@@ -1622,7 +1623,7 @@ Reduction JSNativeContextSpecialization::ReduceElementAccess(
     base::Optional<JSTypedArrayRef> typed_array =
         GetTypedArrayConstant(broker(), receiver);
     if (typed_array.has_value()) {
-      if (FLAG_concurrent_inlining && !typed_array->serialized()) {
+      if (should_disallow_heap_access() && !typed_array->serialized()) {
         TRACE_BROKER_MISSING(broker(), "data for typed array " << *typed_array);
         return NoChange();
       }
@@ -1839,7 +1840,7 @@ Reduction JSNativeContextSpecialization::ReduceElementLoadFromHeapConstant(
 Reduction JSNativeContextSpecialization::ReducePropertyAccess(
     Node* node, Node* key, base::Optional<NameRef> static_name, Node* value,
     FeedbackSource const& source, AccessMode access_mode) {
-  DisallowHeapAccessIf disallow_heap_access(FLAG_concurrent_inlining);
+  DisallowHeapAccessIf disallow_heap_access(should_disallow_heap_access());
 
   DCHECK_EQ(key == nullptr, static_name.has_value());
   DCHECK(node->opcode() == IrOpcode::kJSLoadProperty ||

src/compiler/js-native-context-specialization.h

@@ -44,7 +44,11 @@ class V8_EXPORT_PRIVATE JSNativeContextSpecialization final
     : public AdvancedReducer {
  public:
   // Flags that control the mode of operation.
-  enum Flag { kNoFlags = 0u, kBailoutOnUninitialized = 1u << 0 };
+  enum Flag {
+    kNoFlags = 0u,
+    kBailoutOnUninitialized = 1u << 0,
+    kConcurrentInlining = 1u << 1
+  };
   using Flags = base::Flags<Flag>;
 
   JSNativeContextSpecialization(Editor* editor, JSGraph* jsgraph,
@@ -248,6 +252,10 @@ class V8_EXPORT_PRIVATE JSNativeContextSpecialization final
   Zone* zone() const { return zone_; }
   Zone* shared_zone() const { return shared_zone_; }
 
+  bool should_disallow_heap_access() const {
+    return flags() & kConcurrentInlining;
+  }
+
   JSGraph* const jsgraph_;
   JSHeapBroker* const broker_;
   Flags const flags_;

src/compiler/pipeline.cc

@@ -132,7 +132,8 @@ class PipelineData {
   // For main entry point.
   PipelineData(ZoneStats* zone_stats, Isolate* isolate,
                OptimizedCompilationInfo* info,
-               PipelineStatistics* pipeline_statistics)
+               PipelineStatistics* pipeline_statistics,
+               bool is_concurrent_inlining)
       : isolate_(isolate),
         allocator_(isolate->allocator()),
         info_(info),
@@ -150,7 +151,8 @@ class PipelineData {
         codegen_zone_scope_(zone_stats_, kCodegenZoneName),
         codegen_zone_(codegen_zone_scope_.zone()),
         broker_(new JSHeapBroker(isolate_, info_->zone(),
-                                 info_->trace_heap_broker_enabled())),
+                                 info_->trace_heap_broker_enabled(),
+                                 is_concurrent_inlining)),
         register_allocation_zone_scope_(zone_stats_,
                                         kRegisterAllocationZoneName),
         register_allocation_zone_(register_allocation_zone_scope_.zone()),
@@ -968,7 +970,8 @@ class PipelineCompilationJob final : public OptimizedCompilationJob {
  public:
   PipelineCompilationJob(Isolate* isolate,
                          Handle<SharedFunctionInfo> shared_info,
-                         Handle<JSFunction> function);
+                         Handle<JSFunction> function,
+                         bool is_concurrent_inlining);
   ~PipelineCompilationJob() final;
 
  protected:
@@ -993,7 +996,7 @@ class PipelineCompilationJob final : public OptimizedCompilationJob {
 
 PipelineCompilationJob::PipelineCompilationJob(
     Isolate* isolate, Handle<SharedFunctionInfo> shared_info,
-    Handle<JSFunction> function)
+    Handle<JSFunction> function, bool is_concurrent_inlining)
     // Note that the OptimizedCompilationInfo is not initialized at the time
     // we pass it to the CompilationJob constructor, but it is not
     // dereferenced there.
@@ -1006,10 +1009,9 @@ PipelineCompilationJob::PipelineCompilationJob(
           handle(Script::cast(shared_info->script()), isolate),
           compilation_info(), function->GetIsolate(), &zone_stats_)),
       data_(&zone_stats_, function->GetIsolate(), compilation_info(),
-            pipeline_statistics_.get()),
+            pipeline_statistics_.get(), is_concurrent_inlining),
       pipeline_(&data_),
-      linkage_(nullptr) {
-}
+      linkage_(nullptr) {}
 
 PipelineCompilationJob::~PipelineCompilationJob() {
 }
@@ -1030,6 +1032,9 @@ PipelineCompilationJob::Status PipelineCompilationJob::PrepareJobImpl(
   if (FLAG_turbo_inlining) {
     compilation_info()->MarkAsInliningEnabled();
   }
+  if (FLAG_concurrent_inlining && !compilation_info()->is_osr()) {
+    compilation_info()->MarkAsConcurrentInlining();
+  }
 
   // This is the bottleneck for computing and setting poisoning level in the
   // optimizing compiler.
@@ -1078,7 +1083,7 @@ PipelineCompilationJob::Status PipelineCompilationJob::PrepareJobImpl(
 
   pipeline_.Serialize();
 
-  if (!FLAG_concurrent_inlining) {
+  if (!compilation_info()->is_concurrent_inlining()) {
     if (!pipeline_.CreateGraph()) {
       CHECK(!isolate->has_pending_exception());
       return AbortOptimization(BailoutReason::kGraphBuildingFailed);
@@ -1110,7 +1115,7 @@ PipelineCompilationJob::Status PipelineCompilationJob::ExecuteJobImpl(
   // Ensure that the RuntimeCallStats table is only available during execution
   // and not during finalization as that might be on a different thread.
   PipelineExecuteJobScope scope(&data_, stats);
-  if (FLAG_concurrent_inlining) {
+  if (compilation_info()->is_concurrent_inlining()) {
     if (!pipeline_.CreateGraph()) {
       return AbortOptimization(BailoutReason::kGraphBuildingFailed);
     }
@@ -1323,6 +1328,9 @@ struct GraphBuilderPhase {
     if (data->info()->is_bailout_on_uninitialized()) {
       flags |= BytecodeGraphBuilderFlag::kBailoutOnUninitialized;
     }
+    if (data->info()->is_concurrent_inlining()) {
+      flags |= BytecodeGraphBuilderFlag::kConcurrentInlining;
+    }
 
     JSFunctionRef closure(data->broker(), data->info()->closure());
     CallFrequency frequency(1.0f);
@@ -1347,11 +1355,15 @@ struct InliningPhase {
     CommonOperatorReducer common_reducer(&graph_reducer, data->graph(),
                                          data->broker(), data->common(),
                                          data->machine(), temp_zone);
+    JSCallReducer::Flags call_reducer_flags = JSCallReducer::kNoFlags;
+    if (data->info()->is_bailout_on_uninitialized()) {
+      call_reducer_flags |= JSCallReducer::kBailoutOnUninitialized;
+    }
+    if (data->info()->is_concurrent_inlining()) {
+      call_reducer_flags |= JSCallReducer::kConcurrentInlining;
+    }
     JSCallReducer call_reducer(&graph_reducer, data->jsgraph(), data->broker(),
-                               temp_zone,
-                               data->info()->is_bailout_on_uninitialized()
-                                   ? JSCallReducer::kBailoutOnUninitialized
-                                   : JSCallReducer::kNoFlags,
+                               temp_zone, call_reducer_flags,
                                data->dependencies());
     JSContextSpecialization context_specialization(
         &graph_reducer, data->jsgraph(), data->broker(),
@@ -1364,6 +1376,9 @@ struct InliningPhase {
     if (data->info()->is_bailout_on_uninitialized()) {
       flags |= JSNativeContextSpecialization::kBailoutOnUninitialized;
     }
+    if (data->info()->is_concurrent_inlining()) {
+      flags |= JSNativeContextSpecialization::kConcurrentInlining;
+    }
     // Passing the OptimizedCompilationInfo's shared zone here as
     // JSNativeContextSpecialization allocates out-of-heap objects
     // that need to live until code generation.
@@ -1373,8 +1388,15 @@ struct InliningPhase {
     JSInliningHeuristic inlining(&graph_reducer,
                                  temp_zone, data->info(), data->jsgraph(),
                                  data->broker(), data->source_positions());
+
+    JSIntrinsicLowering::Flags intrinsic_lowering_flags =
+        JSIntrinsicLowering::kNoFlags;
+    if (data->info()->is_concurrent_inlining()) {
+      intrinsic_lowering_flags |= JSIntrinsicLowering::kConcurrentInlining;
+    }
     JSIntrinsicLowering intrinsic_lowering(&graph_reducer, data->jsgraph(),
-                                           data->broker());
+                                           data->broker(),
+                                           intrinsic_lowering_flags);
     AddReducer(data, &graph_reducer, &dead_code_elimination);
     AddReducer(data, &graph_reducer, &checkpoint_elimination);
     AddReducer(data, &graph_reducer, &common_reducer);
@@ -2349,7 +2371,7 @@ void PipelineImpl::Serialize() {
   }
 
   data->broker()->SetTargetNativeContextRef(data->native_context());
-  if (FLAG_concurrent_inlining) {
+  if (data->info()->is_concurrent_inlining()) {
     Run<HeapBrokerInitializationPhase>();
     Run<SerializationPhase>();
     data->broker()->StopSerializing();
@@ -2389,7 +2411,7 @@ bool PipelineImpl::CreateGraph() {
 
   // Run the type-sensitive lowerings and optimizations on the graph.
   {
-    if (!FLAG_concurrent_inlining) {
+    if (!data->info()->is_concurrent_inlining()) {
       Run<HeapBrokerInitializationPhase>();
       Run<CopyMetadataForConcurrentCompilePhase>();
       data->broker()->StopSerializing();
@@ -2786,7 +2808,12 @@ MaybeHandle<Code> Pipeline::GenerateCodeForTesting(
   std::unique_ptr<PipelineStatistics> pipeline_statistics(
       CreatePipelineStatistics(Handle<Script>::null(), info, isolate,
                                &zone_stats));
-  PipelineData data(&zone_stats, isolate, info, pipeline_statistics.get());
+  if (i::FLAG_concurrent_inlining) {
+    info->MarkAsConcurrentInlining();
+  }
+
+  PipelineData data(&zone_stats, isolate, info, pipeline_statistics.get(),
+                    info->is_concurrent_inlining());
   PipelineImpl pipeline(&data);
 
   Linkage linkage(Linkage::ComputeIncoming(data.instruction_zone(), info));
@@ -2848,10 +2875,12 @@ MaybeHandle<Code> Pipeline::GenerateCodeForTesting(
 
 // static
 std::unique_ptr<OptimizedCompilationJob> Pipeline::NewCompilationJob(
-    Isolate* isolate, Handle<JSFunction> function, bool has_script) {
+    Isolate* isolate, Handle<JSFunction> function, bool has_script,
+    bool is_concurrent_inlining) {
   Handle<SharedFunctionInfo> shared =
       handle(function->shared(), function->GetIsolate());
-  return std::make_unique<PipelineCompilationJob>(isolate, shared, function);
+  return std::make_unique<PipelineCompilationJob>(isolate, shared, function,
+                                                  is_concurrent_inlining);
 }
 
 // static

src/compiler/pipeline.h

@@ -45,7 +45,8 @@ class Pipeline : public AllStatic {
  public:
   // Returns a new compilation job for the given JavaScript function.
   static std::unique_ptr<OptimizedCompilationJob> NewCompilationJob(
-      Isolate* isolate, Handle<JSFunction> function, bool has_script);
+      Isolate* isolate, Handle<JSFunction> function, bool has_script,
+      bool is_concurrent_inlining);
 
   // Run the pipeline for the WebAssembly compilation info.
   static void GenerateCodeForWasmFunction(

test/cctest/compiler/test-js-constant-cache.cc

@@ -38,7 +38,7 @@ class JSConstantCacheTester : public HandleAndZoneScope,
         JSGraph(main_isolate(), &main_graph_, &main_common_, &main_javascript_,
                 nullptr, &main_machine_),
         canonical_(main_isolate()),
-        broker_(main_isolate(), main_zone(), false) {
+        broker_(main_isolate(), main_zone(), false, false) {
     main_graph_.SetStart(main_graph_.NewNode(common()->Start(0)));
     main_graph_.SetEnd(
         main_graph_.NewNode(common()->End(1), main_graph_.start()));

test/cctest/compiler/test-js-context-specialization.cc

@@ -33,7 +33,8 @@ class ContextSpecializationTester : public HandleAndZoneScope {
         jsgraph_(main_isolate(), graph(), common(), &javascript_, &simplified_,
                  &machine_),
         reducer_(main_zone(), graph(), &tick_counter_),
-        js_heap_broker_(main_isolate(), main_zone(), FLAG_trace_heap_broker),
+        js_heap_broker_(main_isolate(), main_zone(), FLAG_trace_heap_broker,
+                        false),
         spec_(&reducer_, jsgraph(), &js_heap_broker_, context,
               MaybeHandle<JSFunction>()) {}
 

test/cctest/compiler/test-js-typed-lowering.cc

@@ -26,7 +26,7 @@ class JSTypedLoweringTester : public HandleAndZoneScope {
   explicit JSTypedLoweringTester(int num_parameters = 0)
       : isolate(main_isolate()),
         canonical(isolate),
-        js_heap_broker(isolate, main_zone(), FLAG_trace_heap_broker),
+        js_heap_broker(isolate, main_zone(), FLAG_trace_heap_broker, false),
         binop(nullptr),
         unop(nullptr),
         javascript(main_zone()),

test/cctest/compiler/test-representation-change.cc

@@ -27,7 +27,7 @@ class RepresentationChangerTester : public HandleAndZoneScope,
         javascript_(main_zone()),
         jsgraph_(main_isolate(), main_graph_, &main_common_, &javascript_,
                  &main_simplified_, &main_machine_),
-        broker_{main_isolate(), main_zone(), FLAG_trace_heap_broker},
+        broker_{main_isolate(), main_zone(), FLAG_trace_heap_broker, false},
         changer_(&jsgraph_, &broker_) {
     Node* s = graph()->NewNode(common()->Start(num_parameters));
     graph()->SetStart(s);

test/common/types-fuzz.h

@@ -42,7 +42,7 @@ class Types {
  public:
   Types(Zone* zone, Isolate* isolate, v8::base::RandomNumberGenerator* rng)
       : zone_(zone),
-        js_heap_broker_(isolate, zone, FLAG_trace_heap_broker),
+        js_heap_broker_(isolate, zone, FLAG_trace_heap_broker, false),
         rng_(rng) {
 #define DECLARE_TYPE(name, value) \
   name = Type::name();            \

test/unittests/compiler/common-operator-reducer-unittest.cc

@@ -29,7 +29,7 @@ class CommonOperatorReducerTest : public GraphTest {
   Reduction Reduce(
       AdvancedReducer::Editor* editor, Node* node,
       MachineOperatorBuilder::Flags flags = MachineOperatorBuilder::kNoFlags) {
-    JSHeapBroker broker(isolate(), zone(), FLAG_trace_heap_broker);
+    JSHeapBroker broker(isolate(), zone(), FLAG_trace_heap_broker, false);
     MachineOperatorBuilder machine(zone(), MachineType::PointerRepresentation(),
                                    flags);
     CommonOperatorReducer reducer(editor, graph(), &broker, common(), &machine,

test/unittests/compiler/constant-folding-reducer-unittest.cc

@@ -63,7 +63,7 @@ class ConstantFoldingReducerTest : public TypedGraphTest {
  public:
   ConstantFoldingReducerTest()
       : TypedGraphTest(3),
-        broker_(isolate(), zone(), FLAG_trace_heap_broker),
+        broker_(isolate(), zone(), FLAG_trace_heap_broker, false),
         simplified_(zone()),
         deps_(&broker_, zone()) {}
   ~ConstantFoldingReducerTest() override = default;

test/unittests/compiler/graph-unittest.cc

@@ -18,7 +18,7 @@ GraphTest::GraphTest(int num_parameters)
     : canonical_(isolate()),
       common_(zone()),
       graph_(zone()),
-      broker_(isolate(), zone(), FLAG_trace_heap_broker),
+      broker_(isolate(), zone(), FLAG_trace_heap_broker, false),
       source_positions_(&graph_),
       node_origins_(&graph_) {
   graph()->SetStart(graph()->NewNode(common()->Start(num_parameters)));

test/unittests/compiler/js-intrinsic-lowering-unittest.cc

@@ -37,7 +37,8 @@ class JSIntrinsicLoweringTest : public GraphTest {
                     &machine);
     // TODO(titzer): mock the GraphReducer here for better unit testing.
     GraphReducer graph_reducer(zone(), graph(), tick_counter());
-    JSIntrinsicLowering reducer(&graph_reducer, &jsgraph, broker());
+    JSIntrinsicLowering reducer(&graph_reducer, &jsgraph, broker(),
+                                JSIntrinsicLowering::kNoFlags);
     return reducer.Reduce(node);
   }
 

test/unittests/compiler/simplified-operator-reducer-unittest.cc

@@ -30,7 +30,7 @@ class SimplifiedOperatorReducerTest : public GraphTest {
 
  protected:
   Reduction Reduce(Node* node) {
-    JSHeapBroker broker(isolate(), zone(), FLAG_trace_heap_broker);
+    JSHeapBroker broker(isolate(), zone(), FLAG_trace_heap_broker, false);
     MachineOperatorBuilder machine(zone());
     JSOperatorBuilder javascript(zone());
     JSGraph jsgraph(isolate(), graph(), common(), &javascript, simplified(),

test/unittests/compiler/typer-unittest.cc

@@ -22,7 +22,7 @@ class TyperTest : public TypedGraphTest {
  public:
   TyperTest()
       : TypedGraphTest(3),
-        broker_(isolate(), zone(), FLAG_trace_heap_broker),
+        broker_(isolate(), zone(), FLAG_trace_heap_broker, false),
         operation_typer_(&broker_, zone()),
         types_(zone(), isolate(), random_number_generator()),
         javascript_(zone()),