wasm-serialization.cc 32.7 KB
Newer Older
1 2 3 4 5 6
// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/wasm/wasm-serialization.h"

7
#include "src/base/platform/wrappers.h"
8
#include "src/codegen/assembler-inl.h"
9
#include "src/codegen/external-reference-table.h"
10 11
#include "src/objects/objects-inl.h"
#include "src/objects/objects.h"
12
#include "src/runtime/runtime.h"
13
#include "src/snapshot/code-serializer.h"
14 15 16
#include "src/utils/ostreams.h"
#include "src/utils/utils.h"
#include "src/utils/version.h"
17
#include "src/wasm/code-space-access.h"
18
#include "src/wasm/function-compiler.h"
19 20
#include "src/wasm/module-compiler.h"
#include "src/wasm/module-decoder.h"
21
#include "src/wasm/wasm-code-manager.h"
22
#include "src/wasm/wasm-engine.h"
23 24 25 26 27 28 29 30
#include "src/wasm/wasm-module.h"
#include "src/wasm/wasm-objects-inl.h"
#include "src/wasm/wasm-objects.h"
#include "src/wasm/wasm-result.h"

namespace v8 {
namespace internal {
namespace wasm {
31

32 33
namespace {

34
// TODO(bbudge) Try to unify the various implementations of readers and writers
35
// in Wasm, e.g. StreamProcessor and ZoneBuffer, with these.
36 37
class Writer {
 public:
38
  explicit Writer(Vector<byte> buffer)
39
      : start_(buffer.begin()), end_(buffer.end()), pos_(buffer.begin()) {}
40 41 42 43 44 45 46 47

  size_t bytes_written() const { return pos_ - start_; }
  byte* current_location() const { return pos_; }
  size_t current_size() const { return end_ - pos_; }
  Vector<byte> current_buffer() const {
    return {current_location(), current_size()};
  }

48 49
  template <typename T>
  void Write(const T& value) {
50 51 52
    DCHECK_GE(current_size(), sizeof(T));
    WriteUnalignedValue(reinterpret_cast<Address>(current_location()), value);
    pos_ += sizeof(T);
53
    if (FLAG_trace_wasm_serialization) {
54 55
      StdoutStream{} << "wrote: " << static_cast<size_t>(value)
                     << " sized: " << sizeof(T) << std::endl;
56 57 58
    }
  }

59 60 61
  void WriteVector(const Vector<const byte> v) {
    DCHECK_GE(current_size(), v.size());
    if (v.size() > 0) {
62
      base::Memcpy(current_location(), v.begin(), v.size());
63
      pos_ += v.size();
64
    }
65
    if (FLAG_trace_wasm_serialization) {
66 67
      StdoutStream{} << "wrote vector of " << v.size() << " elements"
                     << std::endl;
68 69 70
    }
  }

71
  void Skip(size_t size) { pos_ += size; }
72

73
 private:
74
  byte* const start_;
75 76
  byte* const end_;
  byte* pos_;
77 78 79 80
};

class Reader {
 public:
81
  explicit Reader(Vector<const byte> buffer)
82
      : start_(buffer.begin()), end_(buffer.end()), pos_(buffer.begin()) {}
83 84 85 86 87 88 89 90

  size_t bytes_read() const { return pos_ - start_; }
  const byte* current_location() const { return pos_; }
  size_t current_size() const { return end_ - pos_; }
  Vector<const byte> current_buffer() const {
    return {current_location(), current_size()};
  }

91 92
  template <typename T>
  T Read() {
93 94 95 96
    DCHECK_GE(current_size(), sizeof(T));
    T value =
        ReadUnalignedValue<T>(reinterpret_cast<Address>(current_location()));
    pos_ += sizeof(T);
97
    if (FLAG_trace_wasm_serialization) {
98 99
      StdoutStream{} << "read: " << static_cast<size_t>(value)
                     << " sized: " << sizeof(T) << std::endl;
100
    }
101
    return value;
102 103
  }

104 105 106 107 108
  template <typename T>
  Vector<const T> ReadVector(size_t size) {
    DCHECK_GE(current_size(), size);
    Vector<const byte> bytes{pos_, size * sizeof(T)};
    pos_ += size * sizeof(T);
109
    if (FLAG_trace_wasm_serialization) {
110 111
      StdoutStream{} << "read vector of " << size << " elements of size "
                     << sizeof(T) << " (total size " << size * sizeof(T) << ")"
112
                     << std::endl;
113
    }
114
    return Vector<const T>::cast(bytes);
115 116
  }

117
  void Skip(size_t size) { pos_ += size; }
118

119
 private:
120
  const byte* const start_;
121 122
  const byte* const end_;
  const byte* pos_;
123 124
};

125
void WriteHeader(Writer* writer) {
126
  writer->Write(SerializedData::kMagicNumber);
127 128 129
  writer->Write(Version::Hash());
  writer->Write(static_cast<uint32_t>(CpuFeatures::SupportedFeatures()));
  writer->Write(FlagList::Hash());
130
  DCHECK_EQ(WasmSerializer::kHeaderSize, writer->bytes_written());
131 132
}

133 134
// On Intel, call sites are encoded as a displacement. For linking and for
// serialization/deserialization, we want to store/retrieve a tag (the function
135 136 137
// index). On Intel, that means accessing the raw displacement.
// On ARM64, call sites are encoded as either a literal load or a direct branch.
// Other platforms simply require accessing the target address.
138 139
void SetWasmCalleeTag(RelocInfo* rinfo, uint32_t tag) {
#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_IA32
140
  DCHECK(rinfo->HasTargetAddressAddress());
141
  DCHECK(!RelocInfo::IsCompressedEmbeddedObject(rinfo->rmode()));
142
  WriteUnalignedValue(rinfo->target_address_address(), tag);
143 144 145
#elif V8_TARGET_ARCH_ARM64
  Instruction* instr = reinterpret_cast<Instruction*>(rinfo->pc());
  if (instr->IsLdrLiteralX()) {
146 147
    WriteUnalignedValue(rinfo->constant_pool_entry_address(),
                        static_cast<Address>(tag));
148 149 150
  } else {
    DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
    instr->SetBranchImmTarget(
151
        reinterpret_cast<Instruction*>(rinfo->pc() + tag * kInstrSize));
152
  }
153
#else
154
  Address addr = static_cast<Address>(tag);
155 156
  if (rinfo->rmode() == RelocInfo::EXTERNAL_REFERENCE) {
    rinfo->set_target_external_reference(addr, SKIP_ICACHE_FLUSH);
157 158
  } else if (rinfo->rmode() == RelocInfo::WASM_STUB_CALL) {
    rinfo->set_wasm_stub_call_address(addr, SKIP_ICACHE_FLUSH);
159 160 161
  } else {
    rinfo->set_target_address(addr, SKIP_WRITE_BARRIER, SKIP_ICACHE_FLUSH);
  }
162 163 164 165 166
#endif
}

uint32_t GetWasmCalleeTag(RelocInfo* rinfo) {
#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_IA32
167
  DCHECK(!RelocInfo::IsCompressedEmbeddedObject(rinfo->rmode()));
168
  return ReadUnalignedValue<uint32_t>(rinfo->target_address_address());
169 170 171
#elif V8_TARGET_ARCH_ARM64
  Instruction* instr = reinterpret_cast<Instruction*>(rinfo->pc());
  if (instr->IsLdrLiteralX()) {
172
    return ReadUnalignedValue<uint32_t>(rinfo->constant_pool_entry_address());
173 174
  } else {
    DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
175
    return static_cast<uint32_t>(instr->ImmPCOffset() / kInstrSize);
176
  }
177
#else
178 179 180 181 182 183 184 185
  Address addr;
  if (rinfo->rmode() == RelocInfo::EXTERNAL_REFERENCE) {
    addr = rinfo->target_external_reference();
  } else if (rinfo->rmode() == RelocInfo::WASM_STUB_CALL) {
    addr = rinfo->wasm_stub_call_address();
  } else {
    addr = rinfo->target_address();
  }
186
  return static_cast<uint32_t>(addr);
187 188 189
#endif
}

190
constexpr size_t kHeaderSize = sizeof(size_t);  // total code size
191

192 193 194 195 196 197 198 199 200 201 202 203 204 205
constexpr size_t kCodeHeaderSize = sizeof(bool) +  // whether code is present
                                   sizeof(int) +   // offset of constant pool
                                   sizeof(int) +   // offset of safepoint table
                                   sizeof(int) +   // offset of handler table
                                   sizeof(int) +   // offset of code comments
                                   sizeof(int) +   // unpadded binary size
                                   sizeof(int) +   // stack slots
                                   sizeof(int) +   // tagged parameter slots
                                   sizeof(int) +   // code size
                                   sizeof(int) +   // reloc size
                                   sizeof(int) +   // source positions size
                                   sizeof(int) +  // protected instructions size
                                   sizeof(WasmCode::Kind) +  // code kind
                                   sizeof(ExecutionTier);    // tier
206

207 208 209 210
// A List of all isolate-independent external references. This is used to create
// a tag from the Address of an external reference and vice versa.
class ExternalReferenceList {
 public:
211 212 213
  ExternalReferenceList(const ExternalReferenceList&) = delete;
  ExternalReferenceList& operator=(const ExternalReferenceList&) = delete;

214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
  uint32_t tag_from_address(Address ext_ref_address) const {
    auto tag_addr_less_than = [this](uint32_t tag, Address searched_addr) {
      return external_reference_by_tag_[tag] < searched_addr;
    };
    auto it = std::lower_bound(std::begin(tags_ordered_by_address_),
                               std::end(tags_ordered_by_address_),
                               ext_ref_address, tag_addr_less_than);
    DCHECK_NE(std::end(tags_ordered_by_address_), it);
    uint32_t tag = *it;
    DCHECK_EQ(address_from_tag(tag), ext_ref_address);
    return tag;
  }

  Address address_from_tag(uint32_t tag) const {
    DCHECK_GT(kNumExternalReferences, tag);
    return external_reference_by_tag_[tag];
  }

  static const ExternalReferenceList& Get() {
    static ExternalReferenceList list;  // Lazily initialized.
    return list;
  }

 private:
  // Private constructor. There will only be a single instance of this object.
  ExternalReferenceList() {
    for (uint32_t i = 0; i < kNumExternalReferences; ++i) {
      tags_ordered_by_address_[i] = i;
    }
    auto addr_by_tag_less_than = [this](uint32_t a, uint32_t b) {
      return external_reference_by_tag_[a] < external_reference_by_tag_[b];
    };
    std::sort(std::begin(tags_ordered_by_address_),
              std::end(tags_ordered_by_address_), addr_by_tag_less_than);
  }

250 251
#define COUNT_EXTERNAL_REFERENCE(name, ...) +1
  static constexpr uint32_t kNumExternalReferencesList =
252
      EXTERNAL_REFERENCE_LIST(COUNT_EXTERNAL_REFERENCE);
253 254 255 256
  static constexpr uint32_t kNumExternalReferencesIntrinsics =
      FOR_EACH_INTRINSIC(COUNT_EXTERNAL_REFERENCE);
  static constexpr uint32_t kNumExternalReferences =
      kNumExternalReferencesList + kNumExternalReferencesIntrinsics;
257 258 259
#undef COUNT_EXTERNAL_REFERENCE

  Address external_reference_by_tag_[kNumExternalReferences] = {
260 261
#define EXT_REF_ADDR(name, desc) ExternalReference::name().address(),
      EXTERNAL_REFERENCE_LIST(EXT_REF_ADDR)
262
#undef EXT_REF_ADDR
263 264 265 266 267
#define RUNTIME_ADDR(name, ...) \
  ExternalReference::Create(Runtime::k##name).address(),
          FOR_EACH_INTRINSIC(RUNTIME_ADDR)
#undef RUNTIME_ADDR
  };
268 269 270 271 272 273
  uint32_t tags_ordered_by_address_[kNumExternalReferences];
};

static_assert(std::is_trivially_destructible<ExternalReferenceList>::value,
              "static destructors not allowed");

274
}  // namespace
275 276 277

class V8_EXPORT_PRIVATE NativeModuleSerializer {
 public:
278
  NativeModuleSerializer(const NativeModule*, Vector<WasmCode* const>);
279 280
  NativeModuleSerializer(const NativeModuleSerializer&) = delete;
  NativeModuleSerializer& operator=(const NativeModuleSerializer&) = delete;
281

282
  size_t Measure() const;
283
  bool Write(Writer* writer);
284 285

 private:
286
  size_t MeasureCode(const WasmCode*) const;
287
  void WriteHeader(Writer*, size_t total_code_size);
288
  bool WriteCode(const WasmCode*, Writer*);
289

290
  const NativeModule* const native_module_;
291 292 293
  const Vector<WasmCode* const> code_table_;
  bool write_called_ = false;
  size_t total_written_code_ = 0;
294 295
};

296
NativeModuleSerializer::NativeModuleSerializer(
297
    const NativeModule* module, Vector<WasmCode* const> code_table)
298
    : native_module_(module), code_table_(code_table) {
299 300 301 302 303 304
  DCHECK_NOT_NULL(native_module_);
  // TODO(mtrofin): persist the export wrappers. Ideally, we'd only persist
  // the unique ones, i.e. the cache.
}

size_t NativeModuleSerializer::MeasureCode(const WasmCode* code) const {
305
  if (code == nullptr) return sizeof(bool);
306
  DCHECK_EQ(WasmCode::kFunction, code->kind());
307 308 309
  if (FLAG_wasm_lazy_compilation && code->tier() != ExecutionTier::kTurbofan) {
    return sizeof(bool);
  }
310 311
  return kCodeHeaderSize + code->instructions().size() +
         code->reloc_info().size() + code->source_positions().size() +
312
         code->protected_instructions_data().size();
313 314 315
}

size_t NativeModuleSerializer::Measure() const {
316
  size_t size = kHeaderSize;
317
  for (WasmCode* code : code_table_) {
318
    size += MeasureCode(code);
319
  }
320
  return size;
321 322
}

323 324
void NativeModuleSerializer::WriteHeader(Writer* writer,
                                         size_t total_code_size) {
325 326 327
  // TODO(eholk): We need to properly preserve the flag whether the trap
  // handler was used or not when serializing.

328
  writer->Write(total_code_size);
329 330
}

331 332
bool NativeModuleSerializer::WriteCode(const WasmCode* code, Writer* writer) {
  DCHECK_IMPLIES(!FLAG_wasm_lazy_compilation, code != nullptr);
333
  if (code == nullptr) {
334
    writer->Write(false);
335
    return true;
336
  }
337
  DCHECK_EQ(WasmCode::kFunction, code->kind());
338 339 340 341 342 343 344 345 346 347
  // Only serialize TurboFan code, as Liftoff code can contain breakpoints or
  // non-relocatable constants.
  if (code->tier() != ExecutionTier::kTurbofan) {
    if (FLAG_wasm_lazy_compilation) {
      writer->Write(false);
      return true;
    }
    return false;
  }
  writer->Write(true);
348 349 350 351
  // Write the size of the entire code section, followed by the code header.
  writer->Write(code->constant_pool_offset());
  writer->Write(code->safepoint_table_offset());
  writer->Write(code->handler_table_offset());
352 353
  writer->Write(code->code_comments_offset());
  writer->Write(code->unpadded_binary_size());
354
  writer->Write(code->stack_slots());
355
  writer->Write(code->tagged_parameter_slots());
356 357 358
  writer->Write(code->instructions().length());
  writer->Write(code->reloc_info().length());
  writer->Write(code->source_positions().length());
359
  writer->Write(code->protected_instructions_data().length());
360
  writer->Write(code->kind());
361
  writer->Write(code->tier());
362 363

  // Get a pointer to the destination buffer, to hold relocated code.
364
  byte* serialized_code_start = writer->current_buffer().begin();
365 366 367 368
  byte* code_start = serialized_code_start;
  size_t code_size = code->instructions().size();
  writer->Skip(code_size);
  // Write the reloc info, source positions, and protected code.
369 370
  writer->WriteVector(code->reloc_info());
  writer->WriteVector(code->source_positions());
371
  writer->WriteVector(code->protected_instructions_data());
372
#if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64 || V8_TARGET_ARCH_ARM || \
Brice Dobry's avatar
Brice Dobry committed
373 374
    V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64 || V8_TARGET_ARCH_S390X || \
    V8_TARGET_ARCH_RISCV64
375 376 377 378
  // On platforms that don't support misaligned word stores, copy to an aligned
  // buffer if necessary so we can relocate the serialized code.
  std::unique_ptr<byte[]> aligned_buffer;
  if (!IsAligned(reinterpret_cast<Address>(serialized_code_start),
379 380 381
                 kSystemPointerSize)) {
    // 'byte' does not guarantee an alignment but seems to work well enough in
    // practice.
382 383 384 385
    aligned_buffer.reset(new byte[code_size]);
    code_start = aligned_buffer.get();
  }
#endif
386
  base::Memcpy(code_start, code->instructions().begin(), code_size);
387
  // Relocate the code.
388
  int mask = RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
389
             RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL) |
390 391 392
             RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
393 394
  RelocIterator orig_iter(code->instructions(), code->reloc_info(),
                          code->constant_pool(), mask);
395 396
  for (RelocIterator iter(
           {code_start, code->instructions().size()}, code->reloc_info(),
397
           reinterpret_cast<Address>(code_start) + code->constant_pool_offset(),
398
           mask);
399 400 401 402 403
       !iter.done(); iter.next(), orig_iter.next()) {
    RelocInfo::Mode mode = orig_iter.rinfo()->rmode();
    switch (mode) {
      case RelocInfo::WASM_CALL: {
        Address orig_target = orig_iter.rinfo()->wasm_call_address();
404 405
        uint32_t tag =
            native_module_->GetFunctionIndexFromJumpTableSlot(orig_target);
406
        SetWasmCalleeTag(iter.rinfo(), tag);
407
      } break;
408
      case RelocInfo::WASM_STUB_CALL: {
409 410 411
        Address target = orig_iter.rinfo()->wasm_stub_call_address();
        uint32_t tag = native_module_->GetRuntimeStubId(target);
        DCHECK_GT(WasmCode::kRuntimeStubCount, tag);
412 413
        SetWasmCalleeTag(iter.rinfo(), tag);
      } break;
414 415
      case RelocInfo::EXTERNAL_REFERENCE: {
        Address orig_target = orig_iter.rinfo()->target_external_reference();
416 417 418
        uint32_t ext_ref_tag =
            ExternalReferenceList::Get().tag_from_address(orig_target);
        SetWasmCalleeTag(iter.rinfo(), ext_ref_tag);
419
      } break;
420 421 422 423 424 425 426
      case RelocInfo::INTERNAL_REFERENCE:
      case RelocInfo::INTERNAL_REFERENCE_ENCODED: {
        Address orig_target = orig_iter.rinfo()->target_internal_reference();
        Address offset = orig_target - code->instruction_start();
        Assembler::deserialization_set_target_internal_reference_at(
            iter.rinfo()->pc(), offset, mode);
      } break;
427 428 429 430
      default:
        UNREACHABLE();
    }
  }
431 432
  // If we copied to an aligned buffer, copy code into serialized buffer.
  if (code_start != serialized_code_start) {
433
    base::Memcpy(serialized_code_start, code_start, code_size);
434
  }
435
  total_written_code_ += code_size;
436
  return true;
437 438
}

439 440 441 442
bool NativeModuleSerializer::Write(Writer* writer) {
  DCHECK(!write_called_);
  write_called_ = true;

443 444 445 446 447 448 449 450
  size_t total_code_size = 0;
  for (WasmCode* code : code_table_) {
    if (code && code->tier() == ExecutionTier::kTurbofan) {
      DCHECK(IsAligned(code->instructions().size(), kCodeAlignment));
      total_code_size += code->instructions().size();
    }
  }
  WriteHeader(writer, total_code_size);
451

452
  for (WasmCode* code : code_table_) {
453
    if (!WriteCode(code, writer)) return false;
454
  }
455 456 457 458

  // Make sure that the serialized total code size was correct.
  CHECK_EQ(total_written_code_, total_code_size);

459
  return true;
460 461
}

462 463
WasmSerializer::WasmSerializer(NativeModule* native_module)
    : native_module_(native_module),
464 465 466
      code_table_(native_module->SnapshotCodeTable()) {}

size_t WasmSerializer::GetSerializedNativeModuleSize() const {
467
  NativeModuleSerializer serializer(native_module_, VectorOf(code_table_));
468
  return kHeaderSize + serializer.Measure();
469
}
470

471
bool WasmSerializer::SerializeNativeModule(Vector<byte> buffer) const {
472
  NativeModuleSerializer serializer(native_module_, VectorOf(code_table_));
473
  size_t measured_size = kHeaderSize + serializer.Measure();
474
  if (buffer.size() < measured_size) return false;
475

476
  Writer writer(buffer);
477
  WriteHeader(&writer);
478

479 480 481
  if (!serializer.Write(&writer)) return false;
  DCHECK_EQ(measured_size, writer.bytes_written());
  return true;
482 483
}

484 485 486
struct DeserializationUnit {
  Vector<const byte> src_code_buffer;
  std::unique_ptr<WasmCode> code;
487
  NativeModule::JumpTablesRef jump_tables;
488 489
};

490 491
class DeserializationQueue {
 public:
492 493
  void Add(std::vector<DeserializationUnit> batch) {
    DCHECK(!batch.empty());
494
    base::MutexGuard guard(&mutex_);
495
    queue_.emplace(std::move(batch));
496 497
  }

498
  std::vector<DeserializationUnit> Pop() {
499
    base::MutexGuard guard(&mutex_);
500
    if (queue_.empty()) return {};
501
    auto batch = std::move(queue_.front());
502
    queue_.pop();
503 504 505
    return batch;
  }

506 507 508 509 510 511 512 513 514 515 516 517 518
  std::vector<DeserializationUnit> PopAll() {
    base::MutexGuard guard(&mutex_);
    if (queue_.empty()) return {};
    auto units = std::move(queue_.front());
    queue_.pop();
    while (!queue_.empty()) {
      units.insert(units.end(), std::make_move_iterator(queue_.front().begin()),
                   std::make_move_iterator(queue_.front().end()));
      queue_.pop();
    }
    return units;
  }

519 520 521
  size_t NumBatches() {
    base::MutexGuard guard(&mutex_);
    return queue_.size();
522 523
  }

524 525
 private:
  base::Mutex mutex_;
526
  std::queue<std::vector<DeserializationUnit>> queue_;
527 528
};

529 530
class V8_EXPORT_PRIVATE NativeModuleDeserializer {
 public:
531
  explicit NativeModuleDeserializer(NativeModule*);
532 533
  NativeModuleDeserializer(const NativeModuleDeserializer&) = delete;
  NativeModuleDeserializer& operator=(const NativeModuleDeserializer&) = delete;
534

535 536 537
  bool Read(Reader* reader);

 private:
538 539
  friend class CopyAndRelocTask;
  friend class PublishTask;
540

541
  void ReadHeader(Reader* reader);
542
  DeserializationUnit ReadCode(int fn_index, Reader* reader);
543
  void CopyAndRelocate(const DeserializationUnit& unit);
544
  void Publish(std::vector<DeserializationUnit> batch);
545 546

  NativeModule* const native_module_;
547 548 549 550 551 552 553
#ifdef DEBUG
  bool read_called_ = false;
#endif

  // Updated in {ReadCode}.
  size_t remaining_code_size_ = 0;
  Vector<byte> current_code_space_;
554
  NativeModule::JumpTablesRef current_jump_tables_;
555 556
};

557
class CopyAndRelocTask : public JobTask {
558
 public:
559
  CopyAndRelocTask(NativeModuleDeserializer* deserializer,
560 561 562 563
                   DeserializationQueue* from_queue,
                   DeserializationQueue* to_queue,
                   std::shared_ptr<JobHandle> publish_handle)
      : deserializer_(deserializer),
564
        from_queue_(from_queue),
565 566
        to_queue_(to_queue),
        publish_handle_(std::move(publish_handle)) {}
567

568
  void Run(JobDelegate* delegate) override {
569
    CODE_SPACE_WRITE_SCOPE
570 571 572 573
    do {
      auto batch = from_queue_->Pop();
      if (batch.empty()) break;
      for (const auto& unit : batch) {
574 575
        deserializer_->CopyAndRelocate(unit);
      }
576 577
      to_queue_->Add(std::move(batch));
      publish_handle_->NotifyConcurrencyIncrease();
578
    } while (!delegate->ShouldYield());
579 580
  }

581 582
  size_t GetMaxConcurrency(size_t /* worker_count */) const override {
    return from_queue_->NumBatches();
583 584 585
  }

 private:
586 587 588 589
  NativeModuleDeserializer* const deserializer_;
  DeserializationQueue* const from_queue_;
  DeserializationQueue* const to_queue_;
  std::shared_ptr<JobHandle> const publish_handle_;
590 591
};

592
class PublishTask : public JobTask {
593 594
 public:
  PublishTask(NativeModuleDeserializer* deserializer,
595 596
              DeserializationQueue* from_queue)
      : deserializer_(deserializer), from_queue_(from_queue) {}
597

598
  void Run(JobDelegate* delegate) override {
599
    WasmCodeRefScope code_scope;
600
    do {
601 602 603
      auto to_publish = from_queue_->PopAll();
      if (to_publish.empty()) break;
      deserializer_->Publish(std::move(to_publish));
604
    } while (!delegate->ShouldYield());
605 606
  }

607 608 609 610 611 612 613
  size_t GetMaxConcurrency(size_t worker_count) const override {
    // Publishing is sequential anyway, so never return more than 1. If a
    // worker is already running, don't spawn a second one.
    if (worker_count > 0) return 0;
    return std::min(size_t{1}, from_queue_->NumBatches());
  }

614
 private:
615 616
  NativeModuleDeserializer* const deserializer_;
  DeserializationQueue* const from_queue_;
617 618
};

619
NativeModuleDeserializer::NativeModuleDeserializer(NativeModule* native_module)
620
    : native_module_(native_module) {}
621 622 623

bool NativeModuleDeserializer::Read(Reader* reader) {
  DCHECK(!read_called_);
624
#ifdef DEBUG
625
  read_called_ = true;
626
#endif
627

628
  ReadHeader(reader);
629
  uint32_t total_fns = native_module_->num_functions();
630
  uint32_t first_wasm_fn = native_module_->num_imported_functions();
631

632
  WasmCodeRefScope wasm_code_ref_scope;
633 634 635 636

  DeserializationQueue reloc_queue;
  DeserializationQueue publish_queue;

637 638 639
  std::shared_ptr<JobHandle> publish_handle = V8::GetCurrentPlatform()->PostJob(
      TaskPriority::kUserVisible,
      std::make_unique<PublishTask>(this, &publish_queue));
640

641 642 643 644 645
  std::unique_ptr<JobHandle> copy_and_reloc_handle =
      V8::GetCurrentPlatform()->PostJob(
          TaskPriority::kUserVisible,
          std::make_unique<CopyAndRelocTask>(this, &reloc_queue, &publish_queue,
                                             publish_handle));
646

647
  std::vector<DeserializationUnit> batch;
648
  const byte* batch_start = reader->current_location();
649
  for (uint32_t i = first_wasm_fn; i < total_fns; ++i) {
650
    DeserializationUnit unit = ReadCode(i, reader);
651 652
    if (!unit.code) continue;
    batch.emplace_back(std::move(unit));
653 654 655
    uint64_t batch_size_in_bytes = reader->current_location() - batch_start;
    constexpr int kMinBatchSizeInBytes = 100000;
    if (batch_size_in_bytes >= kMinBatchSizeInBytes) {
656
      reloc_queue.Add(std::move(batch));
657
      DCHECK(batch.empty());
658
      batch_start = reader->current_location();
659
      copy_and_reloc_handle->NotifyConcurrencyIncrease();
660 661
    }
  }
662

663 664 665 666 667
  // We should have read the expected amount of code now, and should have fully
  // utilized the allocated code space.
  DCHECK_EQ(0, remaining_code_size_);
  DCHECK_EQ(0, current_code_space_.size());

668
  if (!batch.empty()) {
669
    reloc_queue.Add(std::move(batch));
670
    copy_and_reloc_handle->NotifyConcurrencyIncrease();
671 672
  }

673 674 675
  // Wait for all tasks to finish, while participating in their work.
  copy_and_reloc_handle->Join();
  publish_handle->Join();
676

677
  return reader->current_size() == 0;
678 679
}

680
void NativeModuleDeserializer::ReadHeader(Reader* reader) {
681
  remaining_code_size_ = reader->Read<size_t>();
682 683
}

684 685
DeserializationUnit NativeModuleDeserializer::ReadCode(int fn_index,
                                                       Reader* reader) {
686 687
  bool has_code = reader->Read<bool>();
  if (!has_code) {
688
    DCHECK(FLAG_wasm_lazy_compilation ||
689
           native_module_->enabled_features().has_compilation_hints());
690
    native_module_->UseLazyStub(fn_index);
691
    return {};
692
  }
693 694 695 696 697 698 699 700 701 702 703
  int constant_pool_offset = reader->Read<int>();
  int safepoint_table_offset = reader->Read<int>();
  int handler_table_offset = reader->Read<int>();
  int code_comment_offset = reader->Read<int>();
  int unpadded_binary_size = reader->Read<int>();
  int stack_slot_count = reader->Read<int>();
  int tagged_parameter_slots = reader->Read<int>();
  int code_size = reader->Read<int>();
  int reloc_size = reader->Read<int>();
  int source_position_size = reader->Read<int>();
  int protected_instructions_size = reader->Read<int>();
704
  WasmCode::Kind kind = reader->Read<WasmCode::Kind>();
705
  ExecutionTier tier = reader->Read<ExecutionTier>();
706

707 708 709 710 711 712 713 714
  DCHECK(IsAligned(code_size, kCodeAlignment));
  DCHECK_GE(remaining_code_size_, code_size);
  if (current_code_space_.size() < static_cast<size_t>(code_size)) {
    // Allocate the next code space. Don't allocate more than 90% of
    // {kMaxCodeSpaceSize}, to leave some space for jump tables.
    constexpr size_t kMaxReservation =
        RoundUp<kCodeAlignment>(WasmCodeAllocator::kMaxCodeSpaceSize * 9 / 10);
    size_t code_space_size = std::min(kMaxReservation, remaining_code_size_);
715
    std::tie(current_code_space_, current_jump_tables_) =
716 717
        native_module_->AllocateForDeserializedCode(code_space_size);
    DCHECK_EQ(current_code_space_.size(), code_space_size);
718
    DCHECK(current_jump_tables_.is_valid());
719 720
  }

721 722
  DeserializationUnit unit;
  unit.src_code_buffer = reader->ReadVector<byte>(code_size);
723 724
  auto reloc_info = reader->ReadVector<byte>(reloc_size);
  auto source_pos = reader->ReadVector<byte>(source_position_size);
725
  auto protected_instructions =
726
      reader->ReadVector<byte>(protected_instructions_size);
727 728 729 730 731 732 733

  Vector<uint8_t> instructions = current_code_space_.SubVector(0, code_size);
  current_code_space_ += code_size;
  remaining_code_size_ -= code_size;

  unit.code = native_module_->AddDeserializedCode(
      fn_index, instructions, stack_slot_count, tagged_parameter_slots,
734
      safepoint_table_offset, handler_table_offset, constant_pool_offset,
735
      code_comment_offset, unpadded_binary_size, protected_instructions,
736
      reloc_info, source_pos, kind, tier);
737
  unit.jump_tables = current_jump_tables_;
738 739 740 741 742 743 744
  return unit;
}

void NativeModuleDeserializer::CopyAndRelocate(
    const DeserializationUnit& unit) {
  base::Memcpy(unit.code->instructions().begin(), unit.src_code_buffer.begin(),
               unit.src_code_buffer.size());
745

746
  // Relocate the code.
747 748
  int mask = RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
             RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL) |
749 750 751
             RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
752 753
  for (RelocIterator iter(unit.code->instructions(), unit.code->reloc_info(),
                          unit.code->constant_pool(), mask);
754 755 756
       !iter.done(); iter.next()) {
    RelocInfo::Mode mode = iter.rinfo()->rmode();
    switch (mode) {
757 758
      case RelocInfo::WASM_CALL: {
        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
759
        Address target =
760
            native_module_->GetNearCallTargetForFunction(tag, unit.jump_tables);
761 762 763
        iter.rinfo()->set_wasm_call_address(target, SKIP_ICACHE_FLUSH);
        break;
      }
764 765 766
      case RelocInfo::WASM_STUB_CALL: {
        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
        DCHECK_LT(tag, WasmCode::kRuntimeStubCount);
767
        Address target = native_module_->GetNearRuntimeStubEntry(
768
            static_cast<WasmCode::RuntimeStubId>(tag), unit.jump_tables);
769 770 771
        iter.rinfo()->set_wasm_stub_call_address(target, SKIP_ICACHE_FLUSH);
        break;
      }
772 773
      case RelocInfo::EXTERNAL_REFERENCE: {
        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
774
        Address address = ExternalReferenceList::Get().address_from_tag(tag);
775
        iter.rinfo()->set_target_external_reference(address, SKIP_ICACHE_FLUSH);
776
        break;
777
      }
778 779 780
      case RelocInfo::INTERNAL_REFERENCE:
      case RelocInfo::INTERNAL_REFERENCE_ENCODED: {
        Address offset = iter.rinfo()->target_internal_reference();
781
        Address target = unit.code->instruction_start() + offset;
782 783 784 785
        Assembler::deserialization_set_target_internal_reference_at(
            iter.rinfo()->pc(), target, mode);
        break;
      }
786 787
      default:
        UNREACHABLE();
788 789
    }
  }
790

791
  // Finally, flush the icache for that code.
792 793 794 795
  FlushInstructionCache(unit.code->instructions().begin(),
                        unit.code->instructions().size());
}

796 797
void NativeModuleDeserializer::Publish(std::vector<DeserializationUnit> batch) {
  DCHECK(!batch.empty());
798
  std::vector<std::unique_ptr<WasmCode>> codes;
799 800 801
  codes.reserve(batch.size());
  for (auto& unit : batch) {
    codes.emplace_back(std::move(unit).code);
802 803 804 805 806
  }
  auto published_codes = native_module_->PublishCode(VectorOf(codes));
  for (auto* wasm_code : published_codes) {
    wasm_code->MaybePrint();
    wasm_code->Validate();
807
  }
808 809
}

810 811 812 813 814 815 816
bool IsSupportedVersion(Vector<const byte> header) {
  if (header.size() < WasmSerializer::kHeaderSize) return false;
  byte current_version[WasmSerializer::kHeaderSize];
  Writer writer({current_version, WasmSerializer::kHeaderSize});
  WriteHeader(&writer);
  return memcmp(header.begin(), current_version, WasmSerializer::kHeaderSize) ==
         0;
817 818
}

819
MaybeHandle<WasmModuleObject> DeserializeNativeModule(
820
    Isolate* isolate, Vector<const byte> data,
821
    Vector<const byte> wire_bytes_vec, Vector<const char> source_url) {
822 823 824
  if (!IsWasmCodegenAllowed(isolate, isolate->native_context())) return {};
  if (!IsSupportedVersion(data)) return {};

825 826 827 828
  // Make the copy of the wire bytes early, so we use the same memory for
  // decoding, lookup in the native module cache, and insertion into the cache.
  auto owned_wire_bytes = OwnedVector<uint8_t>::Of(wire_bytes_vec);

829
  // TODO(titzer): module features should be part of the serialization format.
830
  WasmEngine* wasm_engine = isolate->wasm_engine();
831
  WasmFeatures enabled_features = WasmFeatures::FromIsolate(isolate);
832
  ModuleResult decode_result = DecodeWasmModule(
833
      enabled_features, owned_wire_bytes.start(), owned_wire_bytes.end(), false,
834 835 836
      i::wasm::kWasmOrigin, isolate->counters(), isolate->metrics_recorder(),
      isolate->GetOrRegisterRecorderContextId(isolate->native_context()),
      DecodingMethod::kDeserialize, wasm_engine->allocator());
837
  if (decode_result.failed()) return {};
838
  std::shared_ptr<WasmModule> module = std::move(decode_result).value();
839
  CHECK_NOT_NULL(module);
840

841
  auto shared_native_module = wasm_engine->MaybeGetNativeModule(
842
      module->origin, owned_wire_bytes.as_vector(), isolate);
843 844 845 846 847 848 849
  if (shared_native_module == nullptr) {
    const bool kIncludeLiftoff = false;
    size_t code_size_estimate =
        wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get(),
                                                            kIncludeLiftoff);
    shared_native_module = wasm_engine->NewNativeModule(
        isolate, enabled_features, std::move(module), code_size_estimate);
850 851 852 853 854 855
    // We have to assign a compilation ID here, as it is required for a
    // potential re-compilation, e.g. triggered by
    // {TierDownAllModulesPerIsolate}. The value is -2 so that it is different
    // than the compilation ID of actual compilations, and also different than
    // the sentinel value of the CompilationState.
    shared_native_module->compilation_state()->set_compilation_id(-2);
856
    shared_native_module->SetWireBytes(std::move(owned_wire_bytes));
857 858

    NativeModuleDeserializer deserializer(shared_native_module.get());
859
    Reader reader(data + WasmSerializer::kHeaderSize);
860
    bool error = !deserializer.Read(&reader);
861
    shared_native_module->compilation_state()->InitializeAfterDeserialization();
862
    wasm_engine->UpdateNativeModuleCache(error, &shared_native_module, isolate);
863 864 865
    if (error) return {};
  }

866 867 868
  Handle<FixedArray> export_wrappers;
  CompileJsToWasmWrappers(isolate, shared_native_module->module(),
                          &export_wrappers);
869

870
  Handle<Script> script =
871
      wasm_engine->GetOrCreateScript(isolate, shared_native_module, source_url);
872
  Handle<WasmModuleObject> module_object = WasmModuleObject::New(
873
      isolate, shared_native_module, script, export_wrappers);
874

875 876 877
  // Finish the Wasm script now and make it public to the debugger.
  isolate->debug()->OnAfterCompile(script);

878
  // Log the code within the generated module for profiling.
879
  shared_native_module->LogWasmCodes(isolate, *script);
880

881
  return module_object;
882 883 884 885 886
}

}  // namespace wasm
}  // namespace internal
}  // namespace v8