[turbofan] Serialize bytecode array for concurrent graph building.

We also introduce a subclass of AbstractBytecodeArray to allow access
to the off-heap copy of bytecode array from the bytecode iterator.

Note that not all bytecode iterators have been converted to iterate
the off-heap copy; in particular, bytecode analysis still iterates
the on-heap bytecode array.

Bug: v8:7790
Change-Id: Icc1b0adb5f63d7cfe94286311f64f209067bbfd5
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1672931
Commit-Queue: Jaroslav Sevcik <jarin@chromium.org>
Reviewed-by: Maya Lekova <mslekova@chromium.org>
Reviewed-by: Georg Neis <neis@chromium.org>
Reviewed-by: Ross McIlroy <rmcilroy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#62403}

src/compiler/bytecode-graph-builder.cc

@@ -32,7 +32,7 @@ namespace compiler {
 class BytecodeGraphBuilder {
  public:
   BytecodeGraphBuilder(JSHeapBroker* broker, Zone* local_zone,
-                       Handle<BytecodeArray> bytecode_array,
+                       BytecodeArrayRef bytecode_array,
                        Handle<SharedFunctionInfo> shared,
                        Handle<FeedbackVector> feedback_vector,
                        BailoutId osr_offset, JSGraph* jsgraph,
@@ -318,9 +318,7 @@ class BytecodeGraphBuilder {
     return jsgraph_->simplified();
   }
   Zone* local_zone() const { return local_zone_; }
-  const Handle<BytecodeArray>& bytecode_array() const {
-    return bytecode_array_;
-  }
+  const BytecodeArrayRef bytecode_array() const { return bytecode_array_; }
   const Handle<FeedbackVector>& feedback_vector() const {
     return feedback_vector_;
   }
@@ -382,7 +380,7 @@ class BytecodeGraphBuilder {
   Zone* const local_zone_;
   JSGraph* const jsgraph_;
   CallFrequency const invocation_frequency_;
-  Handle<BytecodeArray> const bytecode_array_;
+  BytecodeArrayRef const bytecode_array_;
   Handle<FeedbackVector> const feedback_vector_;
   JSTypeHintLowering const type_hint_lowering_;
   const FrameStateFunctionInfo* const frame_state_function_info_;
@@ -938,8 +936,7 @@ Node* BytecodeGraphBuilder::Environment::Checkpoint(
 }
 
 BytecodeGraphBuilder::BytecodeGraphBuilder(
-    JSHeapBroker* broker, Zone* local_zone,
-    Handle<BytecodeArray> bytecode_array,
+    JSHeapBroker* broker, Zone* local_zone, BytecodeArrayRef bytecode_array,
     Handle<SharedFunctionInfo> shared_info,
     Handle<FeedbackVector> feedback_vector, BailoutId osr_offset,
     JSGraph* jsgraph, CallFrequency const& invocation_frequency,
@@ -958,13 +955,15 @@ BytecodeGraphBuilder::BytecodeGraphBuilder(
               : JSTypeHintLowering::kNoFlags),
       frame_state_function_info_(common()->CreateFrameStateFunctionInfo(
           FrameStateType::kInterpretedFunction,
-          bytecode_array->parameter_count(), bytecode_array->register_count(),
+          bytecode_array.parameter_count(), bytecode_array.register_count(),
           shared_info)),
       source_position_iterator_(
-          handle(bytecode_array->SourcePositionTableIfCollected(), isolate())),
-      bytecode_iterator_(bytecode_array),
+          handle(bytecode_array.object()->SourcePositionTableIfCollected(),
+                 isolate())),
+      bytecode_iterator_(
+          base::make_unique<OffHeapBytecodeArray>(bytecode_array)),
       bytecode_analysis_(
-          bytecode_array, local_zone,
+          bytecode_array.object(), local_zone,
           flags & BytecodeGraphBuilderFlag::kAnalyzeEnvironmentLiveness),
       environment_(nullptr),
       osr_offset_(osr_offset),
@@ -1009,21 +1008,18 @@ VectorSlotPair BytecodeGraphBuilder::CreateVectorSlotPair(int slot_id) {
 }
 
 void BytecodeGraphBuilder::CreateGraph() {
-  BytecodeArrayRef bytecode_array_ref(broker(), bytecode_array());
-
   SourcePositionTable::Scope pos_scope(source_positions_, start_position_);
 
   // Set up the basic structure of the graph. Outputs for {Start} are the formal
   // parameters (including the receiver) plus new target, number of arguments,
   // context and closure.
-  int actual_parameter_count = bytecode_array_ref.parameter_count() + 4;
+  int actual_parameter_count = bytecode_array().parameter_count() + 4;
   graph()->SetStart(graph()->NewNode(common()->Start(actual_parameter_count)));
 
-  Environment env(
-      this, bytecode_array_ref.register_count(),
-      bytecode_array_ref.parameter_count(),
-      bytecode_array_ref.incoming_new_target_or_generator_register(),
-      graph()->start());
+  Environment env(this, bytecode_array().register_count(),
+                  bytecode_array().parameter_count(),
+                  bytecode_array().incoming_new_target_or_generator_register(),
+                  graph()->start());
   set_environment(&env);
 
   VisitBytecodes();
@@ -3289,8 +3285,7 @@ void BytecodeGraphBuilder::VisitSuspendGenerator() {
   CHECK_EQ(0, first_reg.index());
   int register_count =
       static_cast<int>(bytecode_iterator().GetRegisterCountOperand(2));
-  int parameter_count_without_receiver =
-      bytecode_array()->parameter_count() - 1;
+  int parameter_count_without_receiver = bytecode_array().parameter_count() - 1;
 
   Node* suspend_id = jsgraph()->SmiConstant(
       bytecode_iterator().GetUnsignedImmediateOperand(3));
@@ -3430,8 +3425,7 @@ void BytecodeGraphBuilder::VisitResumeGenerator() {
   const BytecodeLivenessState* liveness = bytecode_analysis().GetOutLivenessFor(
       bytecode_iterator().current_offset());
 
-  int parameter_count_without_receiver =
-      bytecode_array()->parameter_count() - 1;
+  int parameter_count_without_receiver = bytecode_array().parameter_count() - 1;
 
   // Mapping between registers and array indices must match that used in
   // InterpreterAssembler::ExportParametersAndRegisterFile.
@@ -3824,7 +3818,7 @@ Node** BytecodeGraphBuilder::EnsureInputBufferSize(int size) {
 }
 
 void BytecodeGraphBuilder::ExitThenEnterExceptionHandlers(int current_offset) {
-  HandlerTable table(*bytecode_array());
+  HandlerTable table(*(bytecode_array().object()));
 
   // Potentially exit exception handlers.
   while (!exception_handlers_.empty()) {
@@ -4027,7 +4021,9 @@ void BuildGraphFromBytecode(JSHeapBroker* broker, Zone* local_zone,
                             SourcePositionTable* source_positions,
                             Handle<Context> native_context, int inlining_id,
                             BytecodeGraphBuilderFlags flags) {
-  BytecodeGraphBuilder builder(broker, local_zone, bytecode_array, shared,
+  BytecodeArrayRef bytecode_array_ref(broker, bytecode_array);
+  DCHECK(bytecode_array_ref.IsSerializedForCompilation());
+  BytecodeGraphBuilder builder(broker, local_zone, bytecode_array_ref, shared,
                                feedback_vector, osr_offset, jsgraph,
                                invocation_frequency, source_positions,
                                native_context, inlining_id, flags);

src/compiler/js-heap-broker.cc

@@ -1311,18 +1311,74 @@ class BytecodeArrayData : public FixedArrayBaseData {
     return incoming_new_target_or_generator_register_;
   }
 
+  uint8_t get(int index) const {
+    DCHECK(is_serialized_for_compilation_);
+    return bytecodes_[index];
+  }
+
+  Address GetFirstBytecodeAddress() const {
+    return reinterpret_cast<Address>(bytecodes_.data());
+  }
+
+  Handle<Object> GetConstantAtIndex(int index, Isolate* isolate) const {
+    return constant_pool_[index]->object();
+  }
+
+  bool IsConstantAtIndexSmi(int index) const {
+    return constant_pool_[index]->is_smi();
+  }
+
+  Smi GetConstantAtIndexAsSmi(int index) const {
+    return *(Handle<Smi>::cast(constant_pool_[index]->object()));
+  }
+
+  bool IsSerializedForCompilation() const {
+    return is_serialized_for_compilation_;
+  }
+
+  void SerializeForCompilation(JSHeapBroker* broker) {
+    if (is_serialized_for_compilation_) return;
+
+    DCHECK(bytecodes_.empty());
+    DCHECK(constant_pool_.empty());
+
+    Handle<BytecodeArray> bytecode_array =
+        Handle<BytecodeArray>::cast(object());
+
+    bytecodes_.reserve(bytecode_array->length());
+    for (int i = 0; i < bytecode_array->length(); i++) {
+      bytecodes_.push_back(bytecode_array->get(i));
+    }
+
+    Handle<FixedArray> constant_pool(bytecode_array->constant_pool(),
+                                     broker->isolate());
+
+    constant_pool_.reserve(constant_pool->length());
+    for (int i = 0; i < constant_pool->length(); i++) {
+      constant_pool_.push_back(broker->GetOrCreateData(constant_pool->get(i)));
+    }
+
+    is_serialized_for_compilation_ = true;
+  }
+
   BytecodeArrayData(JSHeapBroker* broker, ObjectData** storage,
                     Handle<BytecodeArray> object)
       : FixedArrayBaseData(broker, storage, object),
         register_count_(object->register_count()),
         parameter_count_(object->parameter_count()),
         incoming_new_target_or_generator_register_(
-            object->incoming_new_target_or_generator_register()) {}
+            object->incoming_new_target_or_generator_register()),
+        bytecodes_(broker->zone()),
+        constant_pool_(broker->zone()) {}
 
  private:
   int const register_count_;
   int const parameter_count_;
   interpreter::Register const incoming_new_target_or_generator_register_;
+
+  bool is_serialized_for_compilation_ = false;
+  ZoneVector<uint8_t> bytecodes_;
+  ZoneVector<ObjectData*> constant_pool_;
 };
 
 class JSArrayData : public JSObjectData {
@@ -2639,6 +2695,62 @@ double FixedDoubleArrayRef::get_scalar(int i) const {
   return data()->AsFixedDoubleArray()->Get(i).get_scalar();
 }
 
+uint8_t BytecodeArrayRef::get(int index) const {
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleAllocation handle_allocation;
+    AllowHandleDereference allow_handle_dereference;
+    return object()->get(index);
+  }
+  return data()->AsBytecodeArray()->get(index);
+}
+
+Address BytecodeArrayRef::GetFirstBytecodeAddress() const {
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleAllocation handle_allocation;
+    AllowHandleDereference allow_handle_dereference;
+    return object()->GetFirstBytecodeAddress();
+  }
+  return data()->AsBytecodeArray()->GetFirstBytecodeAddress();
+}
+
+Handle<Object> BytecodeArrayRef::GetConstantAtIndex(int index) const {
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleAllocation handle_allocation;
+    AllowHandleDereference allow_handle_dereference;
+    return handle(object()->constant_pool().get(index), broker()->isolate());
+  }
+  return data()->AsBytecodeArray()->GetConstantAtIndex(index,
+                                                       broker()->isolate());
+}
+
+bool BytecodeArrayRef::IsConstantAtIndexSmi(int index) const {
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleAllocation handle_allocation;
+    AllowHandleDereference allow_handle_dereference;
+    return object()->constant_pool().get(index).IsSmi();
+  }
+  return data()->AsBytecodeArray()->IsConstantAtIndexSmi(index);
+}
+
+Smi BytecodeArrayRef::GetConstantAtIndexAsSmi(int index) const {
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleAllocation handle_allocation;
+    AllowHandleDereference allow_handle_dereference;
+    return Smi::cast(object()->constant_pool().get(index));
+  }
+  return data()->AsBytecodeArray()->GetConstantAtIndexAsSmi(index);
+}
+
+bool BytecodeArrayRef::IsSerializedForCompilation() const {
+  if (broker()->mode() == JSHeapBroker::kDisabled) return true;
+  return data()->AsBytecodeArray()->IsSerializedForCompilation();
+}
+
+void BytecodeArrayRef::SerializeForCompilation() {
+  if (broker()->mode() == JSHeapBroker::kDisabled) return;
+  data()->AsBytecodeArray()->SerializeForCompilation(broker());
+}
+
 #define IF_BROKER_DISABLED_ACCESS_HANDLE_C(holder, name) \
   if (broker()->mode() == JSHeapBroker::kDisabled) {     \
     AllowHandleAllocation handle_allocation;             \
@@ -3773,6 +3885,36 @@ NamedAccessFeedback const* ProcessedFeedback::AsNamedAccess() const {
   return static_cast<NamedAccessFeedback const*>(this);
 }
 
+OffHeapBytecodeArray::OffHeapBytecodeArray(BytecodeArrayRef bytecode_array)
+    : array_(bytecode_array) {}
+
+int OffHeapBytecodeArray::length() const { return array_.length(); }
+
+int OffHeapBytecodeArray::parameter_count() const {
+  return array_.parameter_count();
+}
+
+uint8_t OffHeapBytecodeArray::get(int index) const { return array_.get(index); }
+
+void OffHeapBytecodeArray::set(int index, uint8_t value) { UNREACHABLE(); }
+
+Address OffHeapBytecodeArray::GetFirstBytecodeAddress() const {
+  return array_.GetFirstBytecodeAddress();
+}
+
+Handle<Object> OffHeapBytecodeArray::GetConstantAtIndex(
+    int index, Isolate* isolate) const {
+  return array_.GetConstantAtIndex(index);
+}
+
+bool OffHeapBytecodeArray::IsConstantAtIndexSmi(int index) const {
+  return array_.IsConstantAtIndexSmi(index);
+}
+
+Smi OffHeapBytecodeArray::GetConstantAtIndexAsSmi(int index) const {
+  return array_.GetConstantAtIndexAsSmi(index);
+}
+
 #undef BIMODAL_ACCESSOR
 #undef BIMODAL_ACCESSOR_B
 #undef BIMODAL_ACCESSOR_C

src/compiler/js-heap-broker.h

@@ -11,6 +11,7 @@
 #include "src/compiler/refs-map.h"
 #include "src/handles/handles.h"
 #include "src/ic/call-optimization.h"
+#include "src/interpreter/bytecode-array-accessor.h"
 #include "src/objects/feedback-vector.h"
 #include "src/objects/function-kind.h"
 #include "src/objects/instance-type.h"
@@ -605,6 +606,18 @@ class BytecodeArrayRef : public FixedArrayBaseRef {
   int register_count() const;
   int parameter_count() const;
   interpreter::Register incoming_new_target_or_generator_register() const;
+
+  // Bytecode access methods.
+  uint8_t get(int index) const;
+  Address GetFirstBytecodeAddress() const;
+
+  // Constant pool access.
+  Handle<Object> GetConstantAtIndex(int index) const;
+  bool IsConstantAtIndexSmi(int index) const;
+  Smi GetConstantAtIndexAsSmi(int index) const;
+
+  bool IsSerializedForCompilation() const;
+  void SerializeForCompilation();
 };
 
 class JSArrayRef : public JSObjectRef {
@@ -1008,6 +1021,23 @@ Reduction NoChangeBecauseOfMissingData(JSHeapBroker* broker,
 // compilation is finished.
 bool CanInlineElementAccess(MapRef const& map);
 
+class OffHeapBytecodeArray final : public interpreter::AbstractBytecodeArray {
+ public:
+  explicit OffHeapBytecodeArray(BytecodeArrayRef bytecode_array);
+
+  int length() const override;
+  int parameter_count() const override;
+  uint8_t get(int index) const override;
+  void set(int index, uint8_t value) override;
+  Address GetFirstBytecodeAddress() const override;
+  Handle<Object> GetConstantAtIndex(int index, Isolate* isolate) const override;
+  bool IsConstantAtIndexSmi(int index) const override;
+  Smi GetConstantAtIndexAsSmi(int index) const override;
+
+ private:
+  BytecodeArrayRef array_;
+};
+
 }  // namespace compiler
 }  // namespace internal
 }  // namespace v8

src/compiler/serializer-for-background-compilation.cc

@@ -482,6 +482,7 @@ void SerializerForBackgroundCompilation::TraverseBytecode() {
   BytecodeArrayRef bytecode_array(
       broker(), handle(environment()->function().shared()->GetBytecodeArray(),
                        broker()->isolate()));
+  bytecode_array.SerializeForCompilation();
   BytecodeArrayIterator iterator(bytecode_array.object());
   ExceptionHandlerMatcher handler_matcher(iterator, bytecode_array.object());
 

src/interpreter/bytecode-array-accessor.cc

@@ -55,8 +55,8 @@ class OnHeapBytecodeArray final : public AbstractBytecodeArray {
 }  // namespace
 
 BytecodeArrayAccessor::BytecodeArrayAccessor(
-    AbstractBytecodeArray* bytecode_array, int initial_offset)
-    : bytecode_array_(bytecode_array),
+    std::unique_ptr<AbstractBytecodeArray> bytecode_array, int initial_offset)
+    : bytecode_array_(std::move(bytecode_array)),
       bytecode_offset_(initial_offset),
       operand_scale_(OperandScale::kSingle),
       prefix_offset_(0) {
@@ -65,8 +65,9 @@ BytecodeArrayAccessor::BytecodeArrayAccessor(
 
 BytecodeArrayAccessor::BytecodeArrayAccessor(
     Handle<BytecodeArray> bytecode_array, int initial_offset)
-    : BytecodeArrayAccessor(new OnHeapBytecodeArray(bytecode_array),
-                            initial_offset) {}
+    : BytecodeArrayAccessor(
+          base::make_unique<OnHeapBytecodeArray>(bytecode_array),
+          initial_offset) {}
 
 void BytecodeArrayAccessor::SetOffset(int offset) {
   bytecode_offset_ = offset;

src/interpreter/bytecode-array-accessor.h

@@ -83,7 +83,7 @@ class V8_EXPORT_PRIVATE AbstractBytecodeArray {
 
 class V8_EXPORT_PRIVATE BytecodeArrayAccessor {
  public:
-  BytecodeArrayAccessor(AbstractBytecodeArray* bytecode_array,
+  BytecodeArrayAccessor(std::unique_ptr<AbstractBytecodeArray> bytecode_array,
                         int initial_offset);
 
   BytecodeArrayAccessor(Handle<BytecodeArray> bytecode_array,

src/interpreter/bytecode-array-iterator.cc

@@ -11,8 +11,8 @@ namespace internal {
 namespace interpreter {
 
 BytecodeArrayIterator::BytecodeArrayIterator(
-    AbstractBytecodeArray* bytecode_array)
-    : BytecodeArrayAccessor(bytecode_array, 0) {}
+    std::unique_ptr<AbstractBytecodeArray> bytecode_array)
+    : BytecodeArrayAccessor(std::move(bytecode_array), 0) {}
 
 BytecodeArrayIterator::BytecodeArrayIterator(
     Handle<BytecodeArray> bytecode_array)

src/interpreter/bytecode-array-iterator.h

@@ -14,7 +14,8 @@ namespace interpreter {
 class V8_EXPORT_PRIVATE BytecodeArrayIterator final
     : public BytecodeArrayAccessor {
  public:
-  explicit BytecodeArrayIterator(AbstractBytecodeArray* array);
+  explicit BytecodeArrayIterator(std::unique_ptr<AbstractBytecodeArray> array);
+
   explicit BytecodeArrayIterator(Handle<BytecodeArray> array);
 
   void Advance();

src/interpreter/bytecode-array-random-iterator.cc

@@ -11,8 +11,8 @@ namespace internal {
 namespace interpreter {
 
 BytecodeArrayRandomIterator::BytecodeArrayRandomIterator(
-    AbstractBytecodeArray* bytecode_array, Zone* zone)
-    : BytecodeArrayAccessor(bytecode_array, 0), offsets_(zone) {
+    std::unique_ptr<AbstractBytecodeArray> bytecode_array, Zone* zone)
+    : BytecodeArrayAccessor(std::move(bytecode_array), 0), offsets_(zone) {
   Initialize();
 }
 

src/interpreter/bytecode-array-random-iterator.h

@@ -16,10 +16,10 @@ namespace interpreter {
 class V8_EXPORT_PRIVATE BytecodeArrayRandomIterator final
     : public BytecodeArrayAccessor {
  public:
-  explicit BytecodeArrayRandomIterator(AbstractBytecodeArray* bytecode_array,
-                                       Zone* zone);
-  explicit BytecodeArrayRandomIterator(Handle<BytecodeArray> bytecode_array,
-                                       Zone* zone);
+  BytecodeArrayRandomIterator(
+      std::unique_ptr<AbstractBytecodeArray> bytecode_array, Zone* zone);
+
+  BytecodeArrayRandomIterator(Handle<BytecodeArray> bytecode_array, Zone* zone);
 
   BytecodeArrayRandomIterator& operator++() {
     ++current_index_;