wasm-serialization.cc 32.8 KB
Newer Older
1 2 3 4 5 6
// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/wasm/wasm-serialization.h"

7
#include "src/base/platform/wrappers.h"
8
#include "src/codegen/assembler-inl.h"
9
#include "src/codegen/external-reference-table.h"
10 11
#include "src/objects/objects-inl.h"
#include "src/objects/objects.h"
12
#include "src/runtime/runtime.h"
13
#include "src/snapshot/code-serializer.h"
14 15 16
#include "src/utils/ostreams.h"
#include "src/utils/utils.h"
#include "src/utils/version.h"
17
#include "src/wasm/code-space-access.h"
18
#include "src/wasm/function-compiler.h"
19 20
#include "src/wasm/module-compiler.h"
#include "src/wasm/module-decoder.h"
21
#include "src/wasm/wasm-code-manager.h"
22
#include "src/wasm/wasm-engine.h"
23 24 25 26 27 28 29 30
#include "src/wasm/wasm-module.h"
#include "src/wasm/wasm-objects-inl.h"
#include "src/wasm/wasm-objects.h"
#include "src/wasm/wasm-result.h"

namespace v8 {
namespace internal {
namespace wasm {
31

32 33
namespace {

34
// TODO(bbudge) Try to unify the various implementations of readers and writers
35
// in Wasm, e.g. StreamProcessor and ZoneBuffer, with these.
36 37
class Writer {
 public:
38
  explicit Writer(Vector<byte> buffer)
39
      : start_(buffer.begin()), end_(buffer.end()), pos_(buffer.begin()) {}
40 41 42 43 44 45 46 47

  size_t bytes_written() const { return pos_ - start_; }
  byte* current_location() const { return pos_; }
  size_t current_size() const { return end_ - pos_; }
  Vector<byte> current_buffer() const {
    return {current_location(), current_size()};
  }

48 49
  template <typename T>
  void Write(const T& value) {
50 51 52
    DCHECK_GE(current_size(), sizeof(T));
    WriteUnalignedValue(reinterpret_cast<Address>(current_location()), value);
    pos_ += sizeof(T);
53
    if (FLAG_trace_wasm_serialization) {
54 55
      StdoutStream{} << "wrote: " << static_cast<size_t>(value)
                     << " sized: " << sizeof(T) << std::endl;
56 57 58
    }
  }

59 60 61
  void WriteVector(const Vector<const byte> v) {
    DCHECK_GE(current_size(), v.size());
    if (v.size() > 0) {
62
      memcpy(current_location(), v.begin(), v.size());
63
      pos_ += v.size();
64
    }
65
    if (FLAG_trace_wasm_serialization) {
66 67
      StdoutStream{} << "wrote vector of " << v.size() << " elements"
                     << std::endl;
68 69 70
    }
  }

71
  void Skip(size_t size) { pos_ += size; }
72

73
 private:
74
  byte* const start_;
75 76
  byte* const end_;
  byte* pos_;
77 78 79 80
};

class Reader {
 public:
81
  explicit Reader(Vector<const byte> buffer)
82
      : start_(buffer.begin()), end_(buffer.end()), pos_(buffer.begin()) {}
83 84 85 86 87 88 89 90

  size_t bytes_read() const { return pos_ - start_; }
  const byte* current_location() const { return pos_; }
  size_t current_size() const { return end_ - pos_; }
  Vector<const byte> current_buffer() const {
    return {current_location(), current_size()};
  }

91 92
  template <typename T>
  T Read() {
93 94 95 96
    DCHECK_GE(current_size(), sizeof(T));
    T value =
        ReadUnalignedValue<T>(reinterpret_cast<Address>(current_location()));
    pos_ += sizeof(T);
97
    if (FLAG_trace_wasm_serialization) {
98 99
      StdoutStream{} << "read: " << static_cast<size_t>(value)
                     << " sized: " << sizeof(T) << std::endl;
100
    }
101
    return value;
102 103
  }

104 105 106 107 108
  template <typename T>
  Vector<const T> ReadVector(size_t size) {
    DCHECK_GE(current_size(), size);
    Vector<const byte> bytes{pos_, size * sizeof(T)};
    pos_ += size * sizeof(T);
109
    if (FLAG_trace_wasm_serialization) {
110 111
      StdoutStream{} << "read vector of " << size << " elements of size "
                     << sizeof(T) << " (total size " << size * sizeof(T) << ")"
112
                     << std::endl;
113
    }
114
    return Vector<const T>::cast(bytes);
115 116
  }

117
  void Skip(size_t size) { pos_ += size; }
118

119
 private:
120
  const byte* const start_;
121 122
  const byte* const end_;
  const byte* pos_;
123 124
};

125
void WriteHeader(Writer* writer) {
126
  writer->Write(SerializedData::kMagicNumber);
127 128 129
  writer->Write(Version::Hash());
  writer->Write(static_cast<uint32_t>(CpuFeatures::SupportedFeatures()));
  writer->Write(FlagList::Hash());
130
  DCHECK_EQ(WasmSerializer::kHeaderSize, writer->bytes_written());
131 132
}

133 134
// On Intel, call sites are encoded as a displacement. For linking and for
// serialization/deserialization, we want to store/retrieve a tag (the function
135 136 137
// index). On Intel, that means accessing the raw displacement.
// On ARM64, call sites are encoded as either a literal load or a direct branch.
// Other platforms simply require accessing the target address.
138 139
void SetWasmCalleeTag(RelocInfo* rinfo, uint32_t tag) {
#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_IA32
140
  DCHECK(rinfo->HasTargetAddressAddress());
141
  DCHECK(!RelocInfo::IsCompressedEmbeddedObject(rinfo->rmode()));
142
  WriteUnalignedValue(rinfo->target_address_address(), tag);
143 144 145
#elif V8_TARGET_ARCH_ARM64
  Instruction* instr = reinterpret_cast<Instruction*>(rinfo->pc());
  if (instr->IsLdrLiteralX()) {
146 147
    WriteUnalignedValue(rinfo->constant_pool_entry_address(),
                        static_cast<Address>(tag));
148 149 150
  } else {
    DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
    instr->SetBranchImmTarget(
151
        reinterpret_cast<Instruction*>(rinfo->pc() + tag * kInstrSize));
152
  }
153
#else
154
  Address addr = static_cast<Address>(tag);
155 156
  if (rinfo->rmode() == RelocInfo::EXTERNAL_REFERENCE) {
    rinfo->set_target_external_reference(addr, SKIP_ICACHE_FLUSH);
157 158
  } else if (rinfo->rmode() == RelocInfo::WASM_STUB_CALL) {
    rinfo->set_wasm_stub_call_address(addr, SKIP_ICACHE_FLUSH);
159 160 161
  } else {
    rinfo->set_target_address(addr, SKIP_WRITE_BARRIER, SKIP_ICACHE_FLUSH);
  }
162 163 164 165 166
#endif
}

uint32_t GetWasmCalleeTag(RelocInfo* rinfo) {
#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_IA32
167
  DCHECK(!RelocInfo::IsCompressedEmbeddedObject(rinfo->rmode()));
168
  return ReadUnalignedValue<uint32_t>(rinfo->target_address_address());
169 170 171
#elif V8_TARGET_ARCH_ARM64
  Instruction* instr = reinterpret_cast<Instruction*>(rinfo->pc());
  if (instr->IsLdrLiteralX()) {
172
    return ReadUnalignedValue<uint32_t>(rinfo->constant_pool_entry_address());
173 174
  } else {
    DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
175
    return static_cast<uint32_t>(instr->ImmPCOffset() / kInstrSize);
176
  }
177
#else
178 179 180 181 182 183 184 185
  Address addr;
  if (rinfo->rmode() == RelocInfo::EXTERNAL_REFERENCE) {
    addr = rinfo->target_external_reference();
  } else if (rinfo->rmode() == RelocInfo::WASM_STUB_CALL) {
    addr = rinfo->wasm_stub_call_address();
  } else {
    addr = rinfo->target_address();
  }
186
  return static_cast<uint32_t>(addr);
187 188 189
#endif
}

190
constexpr size_t kHeaderSize = sizeof(size_t);  // total code size
191

192 193 194 195 196 197 198 199 200 201 202 203 204 205
constexpr size_t kCodeHeaderSize = sizeof(bool) +  // whether code is present
                                   sizeof(int) +   // offset of constant pool
                                   sizeof(int) +   // offset of safepoint table
                                   sizeof(int) +   // offset of handler table
                                   sizeof(int) +   // offset of code comments
                                   sizeof(int) +   // unpadded binary size
                                   sizeof(int) +   // stack slots
                                   sizeof(int) +   // tagged parameter slots
                                   sizeof(int) +   // code size
                                   sizeof(int) +   // reloc size
                                   sizeof(int) +   // source positions size
                                   sizeof(int) +  // protected instructions size
                                   sizeof(WasmCode::Kind) +  // code kind
                                   sizeof(ExecutionTier);    // tier
206

207 208 209 210
// A List of all isolate-independent external references. This is used to create
// a tag from the Address of an external reference and vice versa.
class ExternalReferenceList {
 public:
211 212 213
  ExternalReferenceList(const ExternalReferenceList&) = delete;
  ExternalReferenceList& operator=(const ExternalReferenceList&) = delete;

214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
  uint32_t tag_from_address(Address ext_ref_address) const {
    auto tag_addr_less_than = [this](uint32_t tag, Address searched_addr) {
      return external_reference_by_tag_[tag] < searched_addr;
    };
    auto it = std::lower_bound(std::begin(tags_ordered_by_address_),
                               std::end(tags_ordered_by_address_),
                               ext_ref_address, tag_addr_less_than);
    DCHECK_NE(std::end(tags_ordered_by_address_), it);
    uint32_t tag = *it;
    DCHECK_EQ(address_from_tag(tag), ext_ref_address);
    return tag;
  }

  Address address_from_tag(uint32_t tag) const {
    DCHECK_GT(kNumExternalReferences, tag);
    return external_reference_by_tag_[tag];
  }

  static const ExternalReferenceList& Get() {
    static ExternalReferenceList list;  // Lazily initialized.
    return list;
  }

 private:
  // Private constructor. There will only be a single instance of this object.
  ExternalReferenceList() {
    for (uint32_t i = 0; i < kNumExternalReferences; ++i) {
      tags_ordered_by_address_[i] = i;
    }
    auto addr_by_tag_less_than = [this](uint32_t a, uint32_t b) {
      return external_reference_by_tag_[a] < external_reference_by_tag_[b];
    };
    std::sort(std::begin(tags_ordered_by_address_),
              std::end(tags_ordered_by_address_), addr_by_tag_less_than);
  }

250 251
#define COUNT_EXTERNAL_REFERENCE(name, ...) +1
  static constexpr uint32_t kNumExternalReferencesList =
252
      EXTERNAL_REFERENCE_LIST(COUNT_EXTERNAL_REFERENCE);
253 254 255 256
  static constexpr uint32_t kNumExternalReferencesIntrinsics =
      FOR_EACH_INTRINSIC(COUNT_EXTERNAL_REFERENCE);
  static constexpr uint32_t kNumExternalReferences =
      kNumExternalReferencesList + kNumExternalReferencesIntrinsics;
257 258 259
#undef COUNT_EXTERNAL_REFERENCE

  Address external_reference_by_tag_[kNumExternalReferences] = {
260 261
#define EXT_REF_ADDR(name, desc) ExternalReference::name().address(),
      EXTERNAL_REFERENCE_LIST(EXT_REF_ADDR)
262
#undef EXT_REF_ADDR
263 264 265 266 267
#define RUNTIME_ADDR(name, ...) \
  ExternalReference::Create(Runtime::k##name).address(),
          FOR_EACH_INTRINSIC(RUNTIME_ADDR)
#undef RUNTIME_ADDR
  };
268 269 270 271 272 273
  uint32_t tags_ordered_by_address_[kNumExternalReferences];
};

static_assert(std::is_trivially_destructible<ExternalReferenceList>::value,
              "static destructors not allowed");

274
}  // namespace
275 276 277

class V8_EXPORT_PRIVATE NativeModuleSerializer {
 public:
278
  NativeModuleSerializer(const NativeModule*, Vector<WasmCode* const>);
279 280
  NativeModuleSerializer(const NativeModuleSerializer&) = delete;
  NativeModuleSerializer& operator=(const NativeModuleSerializer&) = delete;
281

282
  size_t Measure() const;
283
  bool Write(Writer* writer);
284 285

 private:
286
  size_t MeasureCode(const WasmCode*) const;
287
  void WriteHeader(Writer*, size_t total_code_size);
288
  bool WriteCode(const WasmCode*, Writer*);
289

290
  const NativeModule* const native_module_;
291 292 293
  const Vector<WasmCode* const> code_table_;
  bool write_called_ = false;
  size_t total_written_code_ = 0;
294 295
};

296
NativeModuleSerializer::NativeModuleSerializer(
297
    const NativeModule* module, Vector<WasmCode* const> code_table)
298
    : native_module_(module), code_table_(code_table) {
299 300 301 302 303 304
  DCHECK_NOT_NULL(native_module_);
  // TODO(mtrofin): persist the export wrappers. Ideally, we'd only persist
  // the unique ones, i.e. the cache.
}

size_t NativeModuleSerializer::MeasureCode(const WasmCode* code) const {
305
  if (code == nullptr) return sizeof(bool);
306
  DCHECK_EQ(WasmCode::kFunction, code->kind());
307 308 309
  if (FLAG_wasm_lazy_compilation && code->tier() != ExecutionTier::kTurbofan) {
    return sizeof(bool);
  }
310 311
  return kCodeHeaderSize + code->instructions().size() +
         code->reloc_info().size() + code->source_positions().size() +
312
         code->protected_instructions_data().size();
313 314 315
}

size_t NativeModuleSerializer::Measure() const {
316
  size_t size = kHeaderSize;
317
  for (WasmCode* code : code_table_) {
318
    size += MeasureCode(code);
319
  }
320
  return size;
321 322
}

323 324
void NativeModuleSerializer::WriteHeader(Writer* writer,
                                         size_t total_code_size) {
325 326 327
  // TODO(eholk): We need to properly preserve the flag whether the trap
  // handler was used or not when serializing.

328
  writer->Write(total_code_size);
329 330
}

331 332
bool NativeModuleSerializer::WriteCode(const WasmCode* code, Writer* writer) {
  DCHECK_IMPLIES(!FLAG_wasm_lazy_compilation, code != nullptr);
333
  if (code == nullptr) {
334
    writer->Write(false);
335
    return true;
336
  }
337
  DCHECK_EQ(WasmCode::kFunction, code->kind());
338 339 340 341 342 343 344 345 346 347
  // Only serialize TurboFan code, as Liftoff code can contain breakpoints or
  // non-relocatable constants.
  if (code->tier() != ExecutionTier::kTurbofan) {
    if (FLAG_wasm_lazy_compilation) {
      writer->Write(false);
      return true;
    }
    return false;
  }
  writer->Write(true);
348 349 350 351
  // Write the size of the entire code section, followed by the code header.
  writer->Write(code->constant_pool_offset());
  writer->Write(code->safepoint_table_offset());
  writer->Write(code->handler_table_offset());
352 353
  writer->Write(code->code_comments_offset());
  writer->Write(code->unpadded_binary_size());
354
  writer->Write(code->stack_slots());
355
  writer->Write(code->raw_tagged_parameter_slots_for_serialization());
356 357 358
  writer->Write(code->instructions().length());
  writer->Write(code->reloc_info().length());
  writer->Write(code->source_positions().length());
359
  writer->Write(code->protected_instructions_data().length());
360
  writer->Write(code->kind());
361
  writer->Write(code->tier());
362 363

  // Get a pointer to the destination buffer, to hold relocated code.
364
  byte* serialized_code_start = writer->current_buffer().begin();
365 366 367 368
  byte* code_start = serialized_code_start;
  size_t code_size = code->instructions().size();
  writer->Skip(code_size);
  // Write the reloc info, source positions, and protected code.
369 370
  writer->WriteVector(code->reloc_info());
  writer->WriteVector(code->source_positions());
371
  writer->WriteVector(code->protected_instructions_data());
372
#if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64 || V8_TARGET_ARCH_ARM || \
Brice Dobry's avatar
Brice Dobry committed
373 374
    V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64 || V8_TARGET_ARCH_S390X || \
    V8_TARGET_ARCH_RISCV64
375 376 377 378
  // On platforms that don't support misaligned word stores, copy to an aligned
  // buffer if necessary so we can relocate the serialized code.
  std::unique_ptr<byte[]> aligned_buffer;
  if (!IsAligned(reinterpret_cast<Address>(serialized_code_start),
379 380 381
                 kSystemPointerSize)) {
    // 'byte' does not guarantee an alignment but seems to work well enough in
    // practice.
382 383 384 385
    aligned_buffer.reset(new byte[code_size]);
    code_start = aligned_buffer.get();
  }
#endif
386
  memcpy(code_start, code->instructions().begin(), code_size);
387
  // Relocate the code.
388
  int mask = RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
389
             RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL) |
390 391 392
             RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
393 394
  RelocIterator orig_iter(code->instructions(), code->reloc_info(),
                          code->constant_pool(), mask);
395 396
  for (RelocIterator iter(
           {code_start, code->instructions().size()}, code->reloc_info(),
397
           reinterpret_cast<Address>(code_start) + code->constant_pool_offset(),
398
           mask);
399 400 401 402 403
       !iter.done(); iter.next(), orig_iter.next()) {
    RelocInfo::Mode mode = orig_iter.rinfo()->rmode();
    switch (mode) {
      case RelocInfo::WASM_CALL: {
        Address orig_target = orig_iter.rinfo()->wasm_call_address();
404 405
        uint32_t tag =
            native_module_->GetFunctionIndexFromJumpTableSlot(orig_target);
406
        SetWasmCalleeTag(iter.rinfo(), tag);
407
      } break;
408
      case RelocInfo::WASM_STUB_CALL: {
409 410 411
        Address target = orig_iter.rinfo()->wasm_stub_call_address();
        uint32_t tag = native_module_->GetRuntimeStubId(target);
        DCHECK_GT(WasmCode::kRuntimeStubCount, tag);
412 413
        SetWasmCalleeTag(iter.rinfo(), tag);
      } break;
414 415
      case RelocInfo::EXTERNAL_REFERENCE: {
        Address orig_target = orig_iter.rinfo()->target_external_reference();
416 417 418
        uint32_t ext_ref_tag =
            ExternalReferenceList::Get().tag_from_address(orig_target);
        SetWasmCalleeTag(iter.rinfo(), ext_ref_tag);
419
      } break;
420 421 422 423 424 425 426
      case RelocInfo::INTERNAL_REFERENCE:
      case RelocInfo::INTERNAL_REFERENCE_ENCODED: {
        Address orig_target = orig_iter.rinfo()->target_internal_reference();
        Address offset = orig_target - code->instruction_start();
        Assembler::deserialization_set_target_internal_reference_at(
            iter.rinfo()->pc(), offset, mode);
      } break;
427 428 429 430
      default:
        UNREACHABLE();
    }
  }
431 432
  // If we copied to an aligned buffer, copy code into serialized buffer.
  if (code_start != serialized_code_start) {
433
    memcpy(serialized_code_start, code_start, code_size);
434
  }
435
  total_written_code_ += code_size;
436
  return true;
437 438
}

439 440 441 442
bool NativeModuleSerializer::Write(Writer* writer) {
  DCHECK(!write_called_);
  write_called_ = true;

443 444 445 446 447 448 449 450
  size_t total_code_size = 0;
  for (WasmCode* code : code_table_) {
    if (code && code->tier() == ExecutionTier::kTurbofan) {
      DCHECK(IsAligned(code->instructions().size(), kCodeAlignment));
      total_code_size += code->instructions().size();
    }
  }
  WriteHeader(writer, total_code_size);
451

452
  for (WasmCode* code : code_table_) {
453
    if (!WriteCode(code, writer)) return false;
454
  }
455 456 457 458

  // Make sure that the serialized total code size was correct.
  CHECK_EQ(total_written_code_, total_code_size);

459
  return true;
460 461
}

462 463
WasmSerializer::WasmSerializer(NativeModule* native_module)
    : native_module_(native_module),
464 465 466
      code_table_(native_module->SnapshotCodeTable()) {}

size_t WasmSerializer::GetSerializedNativeModuleSize() const {
467
  NativeModuleSerializer serializer(native_module_, VectorOf(code_table_));
468
  return kHeaderSize + serializer.Measure();
469
}
470

471
bool WasmSerializer::SerializeNativeModule(Vector<byte> buffer) const {
472
  NativeModuleSerializer serializer(native_module_, VectorOf(code_table_));
473
  size_t measured_size = kHeaderSize + serializer.Measure();
474
  if (buffer.size() < measured_size) return false;
475

476
  Writer writer(buffer);
477
  WriteHeader(&writer);
478

479 480 481
  if (!serializer.Write(&writer)) return false;
  DCHECK_EQ(measured_size, writer.bytes_written());
  return true;
482 483
}

484 485 486
struct DeserializationUnit {
  Vector<const byte> src_code_buffer;
  std::unique_ptr<WasmCode> code;
487
  NativeModule::JumpTablesRef jump_tables;
488 489
};

490 491
class DeserializationQueue {
 public:
492 493
  void Add(std::vector<DeserializationUnit> batch) {
    DCHECK(!batch.empty());
494
    base::MutexGuard guard(&mutex_);
495
    queue_.emplace(std::move(batch));
496 497
  }

498
  std::vector<DeserializationUnit> Pop() {
499
    base::MutexGuard guard(&mutex_);
500
    if (queue_.empty()) return {};
501
    auto batch = std::move(queue_.front());
502
    queue_.pop();
503 504 505
    return batch;
  }

506 507 508 509 510 511 512 513 514 515 516 517 518
  std::vector<DeserializationUnit> PopAll() {
    base::MutexGuard guard(&mutex_);
    if (queue_.empty()) return {};
    auto units = std::move(queue_.front());
    queue_.pop();
    while (!queue_.empty()) {
      units.insert(units.end(), std::make_move_iterator(queue_.front().begin()),
                   std::make_move_iterator(queue_.front().end()));
      queue_.pop();
    }
    return units;
  }

519 520 521
  size_t NumBatches() {
    base::MutexGuard guard(&mutex_);
    return queue_.size();
522 523
  }

524 525
 private:
  base::Mutex mutex_;
526
  std::queue<std::vector<DeserializationUnit>> queue_;
527 528
};

529 530
class V8_EXPORT_PRIVATE NativeModuleDeserializer {
 public:
531
  explicit NativeModuleDeserializer(NativeModule*);
532 533
  NativeModuleDeserializer(const NativeModuleDeserializer&) = delete;
  NativeModuleDeserializer& operator=(const NativeModuleDeserializer&) = delete;
534

535 536 537
  bool Read(Reader* reader);

 private:
538 539
  friend class CopyAndRelocTask;
  friend class PublishTask;
540

541
  void ReadHeader(Reader* reader);
542
  DeserializationUnit ReadCode(int fn_index, Reader* reader);
543
  void CopyAndRelocate(const DeserializationUnit& unit);
544
  void Publish(std::vector<DeserializationUnit> batch);
545 546

  NativeModule* const native_module_;
547 548 549 550 551 552 553
#ifdef DEBUG
  bool read_called_ = false;
#endif

  // Updated in {ReadCode}.
  size_t remaining_code_size_ = 0;
  Vector<byte> current_code_space_;
554
  NativeModule::JumpTablesRef current_jump_tables_;
555 556
};

557
class CopyAndRelocTask : public JobTask {
558
 public:
559
  CopyAndRelocTask(NativeModuleDeserializer* deserializer,
560 561 562 563
                   DeserializationQueue* from_queue,
                   DeserializationQueue* to_queue,
                   std::shared_ptr<JobHandle> publish_handle)
      : deserializer_(deserializer),
564
        from_queue_(from_queue),
565 566
        to_queue_(to_queue),
        publish_handle_(std::move(publish_handle)) {}
567

568
  void Run(JobDelegate* delegate) override {
569
    CODE_SPACE_WRITE_SCOPE
570 571
    NativeModuleModificationScope native_module_modification_scope(
        deserializer_->native_module_);
572 573 574 575
    do {
      auto batch = from_queue_->Pop();
      if (batch.empty()) break;
      for (const auto& unit : batch) {
576 577
        deserializer_->CopyAndRelocate(unit);
      }
578 579
      to_queue_->Add(std::move(batch));
      publish_handle_->NotifyConcurrencyIncrease();
580
    } while (!delegate->ShouldYield());
581 582
  }

583 584
  size_t GetMaxConcurrency(size_t /* worker_count */) const override {
    return from_queue_->NumBatches();
585 586 587
  }

 private:
588 589 590 591
  NativeModuleDeserializer* const deserializer_;
  DeserializationQueue* const from_queue_;
  DeserializationQueue* const to_queue_;
  std::shared_ptr<JobHandle> const publish_handle_;
592 593
};

594
class PublishTask : public JobTask {
595 596
 public:
  PublishTask(NativeModuleDeserializer* deserializer,
597 598
              DeserializationQueue* from_queue)
      : deserializer_(deserializer), from_queue_(from_queue) {}
599

600
  void Run(JobDelegate* delegate) override {
601
    WasmCodeRefScope code_scope;
602
    do {
603 604 605
      auto to_publish = from_queue_->PopAll();
      if (to_publish.empty()) break;
      deserializer_->Publish(std::move(to_publish));
606
    } while (!delegate->ShouldYield());
607 608
  }

609 610 611 612 613 614 615
  size_t GetMaxConcurrency(size_t worker_count) const override {
    // Publishing is sequential anyway, so never return more than 1. If a
    // worker is already running, don't spawn a second one.
    if (worker_count > 0) return 0;
    return std::min(size_t{1}, from_queue_->NumBatches());
  }

616
 private:
617 618
  NativeModuleDeserializer* const deserializer_;
  DeserializationQueue* const from_queue_;
619 620
};

621
NativeModuleDeserializer::NativeModuleDeserializer(NativeModule* native_module)
622
    : native_module_(native_module) {}
623 624 625

bool NativeModuleDeserializer::Read(Reader* reader) {
  DCHECK(!read_called_);
626
#ifdef DEBUG
627
  read_called_ = true;
628
#endif
629

630
  ReadHeader(reader);
631
  uint32_t total_fns = native_module_->num_functions();
632
  uint32_t first_wasm_fn = native_module_->num_imported_functions();
633

634
  WasmCodeRefScope wasm_code_ref_scope;
635 636 637 638

  DeserializationQueue reloc_queue;
  DeserializationQueue publish_queue;

639 640 641
  std::shared_ptr<JobHandle> publish_handle = V8::GetCurrentPlatform()->PostJob(
      TaskPriority::kUserVisible,
      std::make_unique<PublishTask>(this, &publish_queue));
642

643 644 645 646 647
  std::unique_ptr<JobHandle> copy_and_reloc_handle =
      V8::GetCurrentPlatform()->PostJob(
          TaskPriority::kUserVisible,
          std::make_unique<CopyAndRelocTask>(this, &reloc_queue, &publish_queue,
                                             publish_handle));
648

649
  std::vector<DeserializationUnit> batch;
650
  const byte* batch_start = reader->current_location();
651
  for (uint32_t i = first_wasm_fn; i < total_fns; ++i) {
652
    DeserializationUnit unit = ReadCode(i, reader);
653 654
    if (!unit.code) continue;
    batch.emplace_back(std::move(unit));
655 656 657
    uint64_t batch_size_in_bytes = reader->current_location() - batch_start;
    constexpr int kMinBatchSizeInBytes = 100000;
    if (batch_size_in_bytes >= kMinBatchSizeInBytes) {
658
      reloc_queue.Add(std::move(batch));
659
      DCHECK(batch.empty());
660
      batch_start = reader->current_location();
661
      copy_and_reloc_handle->NotifyConcurrencyIncrease();
662 663
    }
  }
664

665 666 667 668 669
  // We should have read the expected amount of code now, and should have fully
  // utilized the allocated code space.
  DCHECK_EQ(0, remaining_code_size_);
  DCHECK_EQ(0, current_code_space_.size());

670
  if (!batch.empty()) {
671
    reloc_queue.Add(std::move(batch));
672
    copy_and_reloc_handle->NotifyConcurrencyIncrease();
673 674
  }

675 676 677
  // Wait for all tasks to finish, while participating in their work.
  copy_and_reloc_handle->Join();
  publish_handle->Join();
678

679
  return reader->current_size() == 0;
680 681
}

682
void NativeModuleDeserializer::ReadHeader(Reader* reader) {
683
  remaining_code_size_ = reader->Read<size_t>();
684 685
}

686 687
DeserializationUnit NativeModuleDeserializer::ReadCode(int fn_index,
                                                       Reader* reader) {
688 689
  bool has_code = reader->Read<bool>();
  if (!has_code) {
690
    DCHECK(FLAG_wasm_lazy_compilation ||
691
           native_module_->enabled_features().has_compilation_hints());
692
    native_module_->UseLazyStub(fn_index);
693
    return {};
694
  }
695 696 697 698 699 700
  int constant_pool_offset = reader->Read<int>();
  int safepoint_table_offset = reader->Read<int>();
  int handler_table_offset = reader->Read<int>();
  int code_comment_offset = reader->Read<int>();
  int unpadded_binary_size = reader->Read<int>();
  int stack_slot_count = reader->Read<int>();
701
  uint32_t tagged_parameter_slots = reader->Read<uint32_t>();
702 703 704 705
  int code_size = reader->Read<int>();
  int reloc_size = reader->Read<int>();
  int source_position_size = reader->Read<int>();
  int protected_instructions_size = reader->Read<int>();
706
  WasmCode::Kind kind = reader->Read<WasmCode::Kind>();
707
  ExecutionTier tier = reader->Read<ExecutionTier>();
708

709 710 711 712 713 714 715 716
  DCHECK(IsAligned(code_size, kCodeAlignment));
  DCHECK_GE(remaining_code_size_, code_size);
  if (current_code_space_.size() < static_cast<size_t>(code_size)) {
    // Allocate the next code space. Don't allocate more than 90% of
    // {kMaxCodeSpaceSize}, to leave some space for jump tables.
    constexpr size_t kMaxReservation =
        RoundUp<kCodeAlignment>(WasmCodeAllocator::kMaxCodeSpaceSize * 9 / 10);
    size_t code_space_size = std::min(kMaxReservation, remaining_code_size_);
717
    std::tie(current_code_space_, current_jump_tables_) =
718 719
        native_module_->AllocateForDeserializedCode(code_space_size);
    DCHECK_EQ(current_code_space_.size(), code_space_size);
720
    DCHECK(current_jump_tables_.is_valid());
721 722
  }

723 724
  DeserializationUnit unit;
  unit.src_code_buffer = reader->ReadVector<byte>(code_size);
725 726
  auto reloc_info = reader->ReadVector<byte>(reloc_size);
  auto source_pos = reader->ReadVector<byte>(source_position_size);
727
  auto protected_instructions =
728
      reader->ReadVector<byte>(protected_instructions_size);
729 730 731 732 733 734 735

  Vector<uint8_t> instructions = current_code_space_.SubVector(0, code_size);
  current_code_space_ += code_size;
  remaining_code_size_ -= code_size;

  unit.code = native_module_->AddDeserializedCode(
      fn_index, instructions, stack_slot_count, tagged_parameter_slots,
736
      safepoint_table_offset, handler_table_offset, constant_pool_offset,
737
      code_comment_offset, unpadded_binary_size, protected_instructions,
738
      reloc_info, source_pos, kind, tier);
739
  unit.jump_tables = current_jump_tables_;
740 741 742 743 744
  return unit;
}

void NativeModuleDeserializer::CopyAndRelocate(
    const DeserializationUnit& unit) {
745 746
  memcpy(unit.code->instructions().begin(), unit.src_code_buffer.begin(),
         unit.src_code_buffer.size());
747

748
  // Relocate the code.
749 750
  int mask = RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
             RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL) |
751 752 753
             RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
754 755
  for (RelocIterator iter(unit.code->instructions(), unit.code->reloc_info(),
                          unit.code->constant_pool(), mask);
756 757 758
       !iter.done(); iter.next()) {
    RelocInfo::Mode mode = iter.rinfo()->rmode();
    switch (mode) {
759 760
      case RelocInfo::WASM_CALL: {
        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
761
        Address target =
762
            native_module_->GetNearCallTargetForFunction(tag, unit.jump_tables);
763 764 765
        iter.rinfo()->set_wasm_call_address(target, SKIP_ICACHE_FLUSH);
        break;
      }
766 767 768
      case RelocInfo::WASM_STUB_CALL: {
        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
        DCHECK_LT(tag, WasmCode::kRuntimeStubCount);
769
        Address target = native_module_->GetNearRuntimeStubEntry(
770
            static_cast<WasmCode::RuntimeStubId>(tag), unit.jump_tables);
771 772 773
        iter.rinfo()->set_wasm_stub_call_address(target, SKIP_ICACHE_FLUSH);
        break;
      }
774 775
      case RelocInfo::EXTERNAL_REFERENCE: {
        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
776
        Address address = ExternalReferenceList::Get().address_from_tag(tag);
777
        iter.rinfo()->set_target_external_reference(address, SKIP_ICACHE_FLUSH);
778
        break;
779
      }
780 781 782
      case RelocInfo::INTERNAL_REFERENCE:
      case RelocInfo::INTERNAL_REFERENCE_ENCODED: {
        Address offset = iter.rinfo()->target_internal_reference();
783
        Address target = unit.code->instruction_start() + offset;
784 785 786 787
        Assembler::deserialization_set_target_internal_reference_at(
            iter.rinfo()->pc(), target, mode);
        break;
      }
788 789
      default:
        UNREACHABLE();
790 791
    }
  }
792

793
  // Finally, flush the icache for that code.
794 795 796 797
  FlushInstructionCache(unit.code->instructions().begin(),
                        unit.code->instructions().size());
}

798 799
void NativeModuleDeserializer::Publish(std::vector<DeserializationUnit> batch) {
  DCHECK(!batch.empty());
800
  std::vector<std::unique_ptr<WasmCode>> codes;
801 802 803
  codes.reserve(batch.size());
  for (auto& unit : batch) {
    codes.emplace_back(std::move(unit).code);
804 805 806 807 808
  }
  auto published_codes = native_module_->PublishCode(VectorOf(codes));
  for (auto* wasm_code : published_codes) {
    wasm_code->MaybePrint();
    wasm_code->Validate();
809
  }
810 811
}

812 813 814 815 816 817 818
bool IsSupportedVersion(Vector<const byte> header) {
  if (header.size() < WasmSerializer::kHeaderSize) return false;
  byte current_version[WasmSerializer::kHeaderSize];
  Writer writer({current_version, WasmSerializer::kHeaderSize});
  WriteHeader(&writer);
  return memcmp(header.begin(), current_version, WasmSerializer::kHeaderSize) ==
         0;
819 820
}

821
MaybeHandle<WasmModuleObject> DeserializeNativeModule(
822
    Isolate* isolate, Vector<const byte> data,
823
    Vector<const byte> wire_bytes_vec, Vector<const char> source_url) {
824 825 826
  if (!IsWasmCodegenAllowed(isolate, isolate->native_context())) return {};
  if (!IsSupportedVersion(data)) return {};

827 828 829 830
  // Make the copy of the wire bytes early, so we use the same memory for
  // decoding, lookup in the native module cache, and insertion into the cache.
  auto owned_wire_bytes = OwnedVector<uint8_t>::Of(wire_bytes_vec);

831
  // TODO(titzer): module features should be part of the serialization format.
832
  WasmEngine* wasm_engine = isolate->wasm_engine();
833
  WasmFeatures enabled_features = WasmFeatures::FromIsolate(isolate);
834
  ModuleResult decode_result = DecodeWasmModule(
835
      enabled_features, owned_wire_bytes.start(), owned_wire_bytes.end(), false,
836 837 838
      i::wasm::kWasmOrigin, isolate->counters(), isolate->metrics_recorder(),
      isolate->GetOrRegisterRecorderContextId(isolate->native_context()),
      DecodingMethod::kDeserialize, wasm_engine->allocator());
839
  if (decode_result.failed()) return {};
840
  std::shared_ptr<WasmModule> module = std::move(decode_result).value();
841
  CHECK_NOT_NULL(module);
842

843
  auto shared_native_module = wasm_engine->MaybeGetNativeModule(
844
      module->origin, owned_wire_bytes.as_vector(), isolate);
845 846 847 848 849 850 851
  if (shared_native_module == nullptr) {
    const bool kIncludeLiftoff = false;
    size_t code_size_estimate =
        wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get(),
                                                            kIncludeLiftoff);
    shared_native_module = wasm_engine->NewNativeModule(
        isolate, enabled_features, std::move(module), code_size_estimate);
852 853 854 855 856 857
    // We have to assign a compilation ID here, as it is required for a
    // potential re-compilation, e.g. triggered by
    // {TierDownAllModulesPerIsolate}. The value is -2 so that it is different
    // than the compilation ID of actual compilations, and also different than
    // the sentinel value of the CompilationState.
    shared_native_module->compilation_state()->set_compilation_id(-2);
858
    shared_native_module->SetWireBytes(std::move(owned_wire_bytes));
859 860

    NativeModuleDeserializer deserializer(shared_native_module.get());
861
    Reader reader(data + WasmSerializer::kHeaderSize);
862
    bool error = !deserializer.Read(&reader);
863
    shared_native_module->compilation_state()->InitializeAfterDeserialization();
864
    wasm_engine->UpdateNativeModuleCache(error, &shared_native_module, isolate);
865 866 867
    if (error) return {};
  }

868 869 870
  Handle<FixedArray> export_wrappers;
  CompileJsToWasmWrappers(isolate, shared_native_module->module(),
                          &export_wrappers);
871

872
  Handle<Script> script =
873
      wasm_engine->GetOrCreateScript(isolate, shared_native_module, source_url);
874
  Handle<WasmModuleObject> module_object = WasmModuleObject::New(
875
      isolate, shared_native_module, script, export_wrappers);
876

877 878 879
  // Finish the Wasm script now and make it public to the debugger.
  isolate->debug()->OnAfterCompile(script);

880
  // Log the code within the generated module for profiling.
881
  shared_native_module->LogWasmCodes(isolate, *script);
882

883
  return module_object;
884 885 886 887 888
}

}  // namespace wasm
}  // namespace internal
}  // namespace v8