wasm-serialization.cc 32.8 KB
Newer Older
1 2 3 4 5 6
// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/wasm/wasm-serialization.h"

7
#include "src/base/platform/wrappers.h"
8
#include "src/codegen/assembler-inl.h"
9
#include "src/codegen/external-reference-table.h"
10 11
#include "src/objects/objects-inl.h"
#include "src/objects/objects.h"
12
#include "src/runtime/runtime.h"
13
#include "src/snapshot/code-serializer.h"
14 15 16
#include "src/utils/ostreams.h"
#include "src/utils/utils.h"
#include "src/utils/version.h"
17
#include "src/wasm/code-space-access.h"
18
#include "src/wasm/function-compiler.h"
19 20
#include "src/wasm/module-compiler.h"
#include "src/wasm/module-decoder.h"
21
#include "src/wasm/wasm-code-manager.h"
22 23 24 25 26 27 28 29
#include "src/wasm/wasm-module.h"
#include "src/wasm/wasm-objects-inl.h"
#include "src/wasm/wasm-objects.h"
#include "src/wasm/wasm-result.h"

namespace v8 {
namespace internal {
namespace wasm {
30

31 32
namespace {

33
// TODO(bbudge) Try to unify the various implementations of readers and writers
34
// in Wasm, e.g. StreamProcessor and ZoneBuffer, with these.
35 36
class Writer {
 public:
37
  explicit Writer(Vector<byte> buffer)
38
      : start_(buffer.begin()), end_(buffer.end()), pos_(buffer.begin()) {}
39 40 41 42 43 44 45 46

  size_t bytes_written() const { return pos_ - start_; }
  byte* current_location() const { return pos_; }
  size_t current_size() const { return end_ - pos_; }
  Vector<byte> current_buffer() const {
    return {current_location(), current_size()};
  }

47 48
  template <typename T>
  void Write(const T& value) {
49 50 51
    DCHECK_GE(current_size(), sizeof(T));
    WriteUnalignedValue(reinterpret_cast<Address>(current_location()), value);
    pos_ += sizeof(T);
52
    if (FLAG_trace_wasm_serialization) {
53 54
      StdoutStream{} << "wrote: " << static_cast<size_t>(value)
                     << " sized: " << sizeof(T) << std::endl;
55 56 57
    }
  }

58 59 60
  void WriteVector(const Vector<const byte> v) {
    DCHECK_GE(current_size(), v.size());
    if (v.size() > 0) {
61
      base::Memcpy(current_location(), v.begin(), v.size());
62
      pos_ += v.size();
63
    }
64
    if (FLAG_trace_wasm_serialization) {
65 66
      StdoutStream{} << "wrote vector of " << v.size() << " elements"
                     << std::endl;
67 68 69
    }
  }

70
  void Skip(size_t size) { pos_ += size; }
71

72
 private:
73
  byte* const start_;
74 75
  byte* const end_;
  byte* pos_;
76 77 78 79
};

class Reader {
 public:
80
  explicit Reader(Vector<const byte> buffer)
81
      : start_(buffer.begin()), end_(buffer.end()), pos_(buffer.begin()) {}
82 83 84 85 86 87 88 89

  size_t bytes_read() const { return pos_ - start_; }
  const byte* current_location() const { return pos_; }
  size_t current_size() const { return end_ - pos_; }
  Vector<const byte> current_buffer() const {
    return {current_location(), current_size()};
  }

90 91
  template <typename T>
  T Read() {
92 93 94 95
    DCHECK_GE(current_size(), sizeof(T));
    T value =
        ReadUnalignedValue<T>(reinterpret_cast<Address>(current_location()));
    pos_ += sizeof(T);
96
    if (FLAG_trace_wasm_serialization) {
97 98
      StdoutStream{} << "read: " << static_cast<size_t>(value)
                     << " sized: " << sizeof(T) << std::endl;
99
    }
100
    return value;
101 102
  }

103 104 105 106 107
  template <typename T>
  Vector<const T> ReadVector(size_t size) {
    DCHECK_GE(current_size(), size);
    Vector<const byte> bytes{pos_, size * sizeof(T)};
    pos_ += size * sizeof(T);
108
    if (FLAG_trace_wasm_serialization) {
109 110
      StdoutStream{} << "read vector of " << size << " elements of size "
                     << sizeof(T) << " (total size " << size * sizeof(T) << ")"
111
                     << std::endl;
112
    }
113
    return Vector<const T>::cast(bytes);
114 115
  }

116
  void Skip(size_t size) { pos_ += size; }
117

118
 private:
119
  const byte* const start_;
120 121
  const byte* const end_;
  const byte* pos_;
122 123
};

124
void WriteHeader(Writer* writer) {
125
  writer->Write(SerializedData::kMagicNumber);
126 127 128
  writer->Write(Version::Hash());
  writer->Write(static_cast<uint32_t>(CpuFeatures::SupportedFeatures()));
  writer->Write(FlagList::Hash());
129
  DCHECK_EQ(WasmSerializer::kHeaderSize, writer->bytes_written());
130 131
}

132 133
// On Intel, call sites are encoded as a displacement. For linking and for
// serialization/deserialization, we want to store/retrieve a tag (the function
134 135 136
// index). On Intel, that means accessing the raw displacement.
// On ARM64, call sites are encoded as either a literal load or a direct branch.
// Other platforms simply require accessing the target address.
137 138
void SetWasmCalleeTag(RelocInfo* rinfo, uint32_t tag) {
#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_IA32
139
  DCHECK(rinfo->HasTargetAddressAddress());
140
  DCHECK(!RelocInfo::IsCompressedEmbeddedObject(rinfo->rmode()));
141
  WriteUnalignedValue(rinfo->target_address_address(), tag);
142 143 144
#elif V8_TARGET_ARCH_ARM64
  Instruction* instr = reinterpret_cast<Instruction*>(rinfo->pc());
  if (instr->IsLdrLiteralX()) {
145 146
    WriteUnalignedValue(rinfo->constant_pool_entry_address(),
                        static_cast<Address>(tag));
147 148 149
  } else {
    DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
    instr->SetBranchImmTarget(
150
        reinterpret_cast<Instruction*>(rinfo->pc() + tag * kInstrSize));
151
  }
152
#else
153
  Address addr = static_cast<Address>(tag);
154 155
  if (rinfo->rmode() == RelocInfo::EXTERNAL_REFERENCE) {
    rinfo->set_target_external_reference(addr, SKIP_ICACHE_FLUSH);
156 157
  } else if (rinfo->rmode() == RelocInfo::WASM_STUB_CALL) {
    rinfo->set_wasm_stub_call_address(addr, SKIP_ICACHE_FLUSH);
158 159 160
  } else {
    rinfo->set_target_address(addr, SKIP_WRITE_BARRIER, SKIP_ICACHE_FLUSH);
  }
161 162 163 164 165
#endif
}

uint32_t GetWasmCalleeTag(RelocInfo* rinfo) {
#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_IA32
166
  DCHECK(!RelocInfo::IsCompressedEmbeddedObject(rinfo->rmode()));
167
  return ReadUnalignedValue<uint32_t>(rinfo->target_address_address());
168 169 170
#elif V8_TARGET_ARCH_ARM64
  Instruction* instr = reinterpret_cast<Instruction*>(rinfo->pc());
  if (instr->IsLdrLiteralX()) {
171
    return ReadUnalignedValue<uint32_t>(rinfo->constant_pool_entry_address());
172 173
  } else {
    DCHECK(instr->IsBranchAndLink() || instr->IsUnconditionalBranch());
174
    return static_cast<uint32_t>(instr->ImmPCOffset() / kInstrSize);
175
  }
176
#else
177 178 179 180 181 182 183 184
  Address addr;
  if (rinfo->rmode() == RelocInfo::EXTERNAL_REFERENCE) {
    addr = rinfo->target_external_reference();
  } else if (rinfo->rmode() == RelocInfo::WASM_STUB_CALL) {
    addr = rinfo->wasm_stub_call_address();
  } else {
    addr = rinfo->target_address();
  }
185
  return static_cast<uint32_t>(addr);
186 187 188
#endif
}

189
constexpr size_t kHeaderSize = sizeof(size_t);  // total code size
190

191 192 193 194 195 196 197 198 199 200 201 202 203 204
constexpr size_t kCodeHeaderSize = sizeof(bool) +  // whether code is present
                                   sizeof(int) +   // offset of constant pool
                                   sizeof(int) +   // offset of safepoint table
                                   sizeof(int) +   // offset of handler table
                                   sizeof(int) +   // offset of code comments
                                   sizeof(int) +   // unpadded binary size
                                   sizeof(int) +   // stack slots
                                   sizeof(int) +   // tagged parameter slots
                                   sizeof(int) +   // code size
                                   sizeof(int) +   // reloc size
                                   sizeof(int) +   // source positions size
                                   sizeof(int) +  // protected instructions size
                                   sizeof(WasmCode::Kind) +  // code kind
                                   sizeof(ExecutionTier);    // tier
205

206 207 208 209
// A List of all isolate-independent external references. This is used to create
// a tag from the Address of an external reference and vice versa.
class ExternalReferenceList {
 public:
210 211 212
  ExternalReferenceList(const ExternalReferenceList&) = delete;
  ExternalReferenceList& operator=(const ExternalReferenceList&) = delete;

213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
  uint32_t tag_from_address(Address ext_ref_address) const {
    auto tag_addr_less_than = [this](uint32_t tag, Address searched_addr) {
      return external_reference_by_tag_[tag] < searched_addr;
    };
    auto it = std::lower_bound(std::begin(tags_ordered_by_address_),
                               std::end(tags_ordered_by_address_),
                               ext_ref_address, tag_addr_less_than);
    DCHECK_NE(std::end(tags_ordered_by_address_), it);
    uint32_t tag = *it;
    DCHECK_EQ(address_from_tag(tag), ext_ref_address);
    return tag;
  }

  Address address_from_tag(uint32_t tag) const {
    DCHECK_GT(kNumExternalReferences, tag);
    return external_reference_by_tag_[tag];
  }

  static const ExternalReferenceList& Get() {
    static ExternalReferenceList list;  // Lazily initialized.
    return list;
  }

 private:
  // Private constructor. There will only be a single instance of this object.
  ExternalReferenceList() {
    for (uint32_t i = 0; i < kNumExternalReferences; ++i) {
      tags_ordered_by_address_[i] = i;
    }
    auto addr_by_tag_less_than = [this](uint32_t a, uint32_t b) {
      return external_reference_by_tag_[a] < external_reference_by_tag_[b];
    };
    std::sort(std::begin(tags_ordered_by_address_),
              std::end(tags_ordered_by_address_), addr_by_tag_less_than);
  }

249 250
#define COUNT_EXTERNAL_REFERENCE(name, ...) +1
  static constexpr uint32_t kNumExternalReferencesList =
251
      EXTERNAL_REFERENCE_LIST(COUNT_EXTERNAL_REFERENCE);
252 253 254 255
  static constexpr uint32_t kNumExternalReferencesIntrinsics =
      FOR_EACH_INTRINSIC(COUNT_EXTERNAL_REFERENCE);
  static constexpr uint32_t kNumExternalReferences =
      kNumExternalReferencesList + kNumExternalReferencesIntrinsics;
256 257 258
#undef COUNT_EXTERNAL_REFERENCE

  Address external_reference_by_tag_[kNumExternalReferences] = {
259 260
#define EXT_REF_ADDR(name, desc) ExternalReference::name().address(),
      EXTERNAL_REFERENCE_LIST(EXT_REF_ADDR)
261
#undef EXT_REF_ADDR
262 263 264 265 266
#define RUNTIME_ADDR(name, ...) \
  ExternalReference::Create(Runtime::k##name).address(),
          FOR_EACH_INTRINSIC(RUNTIME_ADDR)
#undef RUNTIME_ADDR
  };
267 268 269 270 271 272
  uint32_t tags_ordered_by_address_[kNumExternalReferences];
};

static_assert(std::is_trivially_destructible<ExternalReferenceList>::value,
              "static destructors not allowed");

273
}  // namespace
274 275 276

class V8_EXPORT_PRIVATE NativeModuleSerializer {
 public:
277
  NativeModuleSerializer(const NativeModule*, Vector<WasmCode* const>);
278 279
  NativeModuleSerializer(const NativeModuleSerializer&) = delete;
  NativeModuleSerializer& operator=(const NativeModuleSerializer&) = delete;
280

281
  size_t Measure() const;
282
  bool Write(Writer* writer);
283 284

 private:
285
  size_t MeasureCode(const WasmCode*) const;
286
  void WriteHeader(Writer*, size_t total_code_size);
287
  bool WriteCode(const WasmCode*, Writer*);
288

289
  const NativeModule* const native_module_;
290 291 292
  const Vector<WasmCode* const> code_table_;
  bool write_called_ = false;
  size_t total_written_code_ = 0;
293 294
};

295
NativeModuleSerializer::NativeModuleSerializer(
296
    const NativeModule* module, Vector<WasmCode* const> code_table)
297
    : native_module_(module), code_table_(code_table) {
298 299 300 301 302 303
  DCHECK_NOT_NULL(native_module_);
  // TODO(mtrofin): persist the export wrappers. Ideally, we'd only persist
  // the unique ones, i.e. the cache.
}

size_t NativeModuleSerializer::MeasureCode(const WasmCode* code) const {
304
  if (code == nullptr) return sizeof(bool);
305
  DCHECK_EQ(WasmCode::kFunction, code->kind());
306 307 308
  if (FLAG_wasm_lazy_compilation && code->tier() != ExecutionTier::kTurbofan) {
    return sizeof(bool);
  }
309 310
  return kCodeHeaderSize + code->instructions().size() +
         code->reloc_info().size() + code->source_positions().size() +
311
         code->protected_instructions_data().size();
312 313 314
}

size_t NativeModuleSerializer::Measure() const {
315
  size_t size = kHeaderSize;
316
  for (WasmCode* code : code_table_) {
317
    size += MeasureCode(code);
318
  }
319
  return size;
320 321
}

322 323
void NativeModuleSerializer::WriteHeader(Writer* writer,
                                         size_t total_code_size) {
324 325 326
  // TODO(eholk): We need to properly preserve the flag whether the trap
  // handler was used or not when serializing.

327
  writer->Write(total_code_size);
328 329
}

330 331
bool NativeModuleSerializer::WriteCode(const WasmCode* code, Writer* writer) {
  DCHECK_IMPLIES(!FLAG_wasm_lazy_compilation, code != nullptr);
332
  if (code == nullptr) {
333
    writer->Write(false);
334
    return true;
335
  }
336
  DCHECK_EQ(WasmCode::kFunction, code->kind());
337 338 339 340 341 342 343 344 345 346
  // Only serialize TurboFan code, as Liftoff code can contain breakpoints or
  // non-relocatable constants.
  if (code->tier() != ExecutionTier::kTurbofan) {
    if (FLAG_wasm_lazy_compilation) {
      writer->Write(false);
      return true;
    }
    return false;
  }
  writer->Write(true);
347 348 349 350
  // Write the size of the entire code section, followed by the code header.
  writer->Write(code->constant_pool_offset());
  writer->Write(code->safepoint_table_offset());
  writer->Write(code->handler_table_offset());
351 352
  writer->Write(code->code_comments_offset());
  writer->Write(code->unpadded_binary_size());
353
  writer->Write(code->stack_slots());
354
  writer->Write(code->tagged_parameter_slots());
355 356 357
  writer->Write(code->instructions().length());
  writer->Write(code->reloc_info().length());
  writer->Write(code->source_positions().length());
358
  writer->Write(code->protected_instructions_data().length());
359
  writer->Write(code->kind());
360
  writer->Write(code->tier());
361 362

  // Get a pointer to the destination buffer, to hold relocated code.
363
  byte* serialized_code_start = writer->current_buffer().begin();
364 365 366 367
  byte* code_start = serialized_code_start;
  size_t code_size = code->instructions().size();
  writer->Skip(code_size);
  // Write the reloc info, source positions, and protected code.
368 369
  writer->WriteVector(code->reloc_info());
  writer->WriteVector(code->source_positions());
370
  writer->WriteVector(code->protected_instructions_data());
371
#if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64 || V8_TARGET_ARCH_ARM || \
Brice Dobry's avatar
Brice Dobry committed
372 373
    V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64 || V8_TARGET_ARCH_S390X || \
    V8_TARGET_ARCH_RISCV64
374 375 376 377
  // On platforms that don't support misaligned word stores, copy to an aligned
  // buffer if necessary so we can relocate the serialized code.
  std::unique_ptr<byte[]> aligned_buffer;
  if (!IsAligned(reinterpret_cast<Address>(serialized_code_start),
378 379 380
                 kSystemPointerSize)) {
    // 'byte' does not guarantee an alignment but seems to work well enough in
    // practice.
381 382 383 384
    aligned_buffer.reset(new byte[code_size]);
    code_start = aligned_buffer.get();
  }
#endif
385
  base::Memcpy(code_start, code->instructions().begin(), code_size);
386
  // Relocate the code.
387
  int mask = RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
388
             RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL) |
389 390 391
             RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
392 393
  RelocIterator orig_iter(code->instructions(), code->reloc_info(),
                          code->constant_pool(), mask);
394 395
  for (RelocIterator iter(
           {code_start, code->instructions().size()}, code->reloc_info(),
396
           reinterpret_cast<Address>(code_start) + code->constant_pool_offset(),
397
           mask);
398 399 400 401 402
       !iter.done(); iter.next(), orig_iter.next()) {
    RelocInfo::Mode mode = orig_iter.rinfo()->rmode();
    switch (mode) {
      case RelocInfo::WASM_CALL: {
        Address orig_target = orig_iter.rinfo()->wasm_call_address();
403 404
        uint32_t tag =
            native_module_->GetFunctionIndexFromJumpTableSlot(orig_target);
405
        SetWasmCalleeTag(iter.rinfo(), tag);
406
      } break;
407
      case RelocInfo::WASM_STUB_CALL: {
408 409 410
        Address target = orig_iter.rinfo()->wasm_stub_call_address();
        uint32_t tag = native_module_->GetRuntimeStubId(target);
        DCHECK_GT(WasmCode::kRuntimeStubCount, tag);
411 412
        SetWasmCalleeTag(iter.rinfo(), tag);
      } break;
413 414
      case RelocInfo::EXTERNAL_REFERENCE: {
        Address orig_target = orig_iter.rinfo()->target_external_reference();
415 416 417
        uint32_t ext_ref_tag =
            ExternalReferenceList::Get().tag_from_address(orig_target);
        SetWasmCalleeTag(iter.rinfo(), ext_ref_tag);
418
      } break;
419 420 421 422 423 424 425
      case RelocInfo::INTERNAL_REFERENCE:
      case RelocInfo::INTERNAL_REFERENCE_ENCODED: {
        Address orig_target = orig_iter.rinfo()->target_internal_reference();
        Address offset = orig_target - code->instruction_start();
        Assembler::deserialization_set_target_internal_reference_at(
            iter.rinfo()->pc(), offset, mode);
      } break;
426 427 428 429
      default:
        UNREACHABLE();
    }
  }
430 431
  // If we copied to an aligned buffer, copy code into serialized buffer.
  if (code_start != serialized_code_start) {
432
    base::Memcpy(serialized_code_start, code_start, code_size);
433
  }
434
  total_written_code_ += code_size;
435
  return true;
436 437
}

438 439 440 441
bool NativeModuleSerializer::Write(Writer* writer) {
  DCHECK(!write_called_);
  write_called_ = true;

442 443 444 445 446 447 448 449
  size_t total_code_size = 0;
  for (WasmCode* code : code_table_) {
    if (code && code->tier() == ExecutionTier::kTurbofan) {
      DCHECK(IsAligned(code->instructions().size(), kCodeAlignment));
      total_code_size += code->instructions().size();
    }
  }
  WriteHeader(writer, total_code_size);
450

451
  for (WasmCode* code : code_table_) {
452
    if (!WriteCode(code, writer)) return false;
453
  }
454 455 456 457

  // Make sure that the serialized total code size was correct.
  CHECK_EQ(total_written_code_, total_code_size);

458
  return true;
459 460
}

461 462
WasmSerializer::WasmSerializer(NativeModule* native_module)
    : native_module_(native_module),
463 464 465
      code_table_(native_module->SnapshotCodeTable()) {}

size_t WasmSerializer::GetSerializedNativeModuleSize() const {
466
  NativeModuleSerializer serializer(native_module_, VectorOf(code_table_));
467
  return kHeaderSize + serializer.Measure();
468
}
469

470
bool WasmSerializer::SerializeNativeModule(Vector<byte> buffer) const {
471
  NativeModuleSerializer serializer(native_module_, VectorOf(code_table_));
472
  size_t measured_size = kHeaderSize + serializer.Measure();
473
  if (buffer.size() < measured_size) return false;
474

475
  Writer writer(buffer);
476
  WriteHeader(&writer);
477

478 479 480
  if (!serializer.Write(&writer)) return false;
  DCHECK_EQ(measured_size, writer.bytes_written());
  return true;
481 482
}

483 484 485
struct DeserializationUnit {
  Vector<const byte> src_code_buffer;
  std::unique_ptr<WasmCode> code;
486
  NativeModule::JumpTablesRef jump_tables;
487 488
};

489 490
class DeserializationQueue {
 public:
491 492
  void Add(std::vector<DeserializationUnit> batch) {
    DCHECK(!batch.empty());
493
    base::MutexGuard guard(&mutex_);
494
    queue_.emplace(std::move(batch));
495 496
  }

497
  std::vector<DeserializationUnit> Pop() {
498
    base::MutexGuard guard(&mutex_);
499
    if (queue_.empty()) return {};
500
    auto batch = std::move(queue_.front());
501
    queue_.pop();
502 503 504
    return batch;
  }

505 506 507 508 509 510 511 512 513 514 515 516 517
  std::vector<DeserializationUnit> PopAll() {
    base::MutexGuard guard(&mutex_);
    if (queue_.empty()) return {};
    auto units = std::move(queue_.front());
    queue_.pop();
    while (!queue_.empty()) {
      units.insert(units.end(), std::make_move_iterator(queue_.front().begin()),
                   std::make_move_iterator(queue_.front().end()));
      queue_.pop();
    }
    return units;
  }

518 519 520
  size_t NumBatches() {
    base::MutexGuard guard(&mutex_);
    return queue_.size();
521 522
  }

523 524
 private:
  base::Mutex mutex_;
525
  std::queue<std::vector<DeserializationUnit>> queue_;
526 527
};

528 529
class V8_EXPORT_PRIVATE NativeModuleDeserializer {
 public:
530
  explicit NativeModuleDeserializer(NativeModule*);
531 532
  NativeModuleDeserializer(const NativeModuleDeserializer&) = delete;
  NativeModuleDeserializer& operator=(const NativeModuleDeserializer&) = delete;
533

534 535 536
  bool Read(Reader* reader);

 private:
537 538
  friend class CopyAndRelocTask;
  friend class PublishTask;
539

540
  void ReadHeader(Reader* reader);
541
  DeserializationUnit ReadCode(int fn_index, Reader* reader);
542
  void CopyAndRelocate(const DeserializationUnit& unit);
543
  void Publish(std::vector<DeserializationUnit> batch);
544 545

  NativeModule* const native_module_;
546 547 548 549 550 551 552
#ifdef DEBUG
  bool read_called_ = false;
#endif

  // Updated in {ReadCode}.
  size_t remaining_code_size_ = 0;
  Vector<byte> current_code_space_;
553
  NativeModule::JumpTablesRef current_jump_tables_;
554 555
};

556
class CopyAndRelocTask : public JobTask {
557
 public:
558
  CopyAndRelocTask(NativeModuleDeserializer* deserializer,
559 560 561 562
                   DeserializationQueue* from_queue,
                   DeserializationQueue* to_queue,
                   std::shared_ptr<JobHandle> publish_handle)
      : deserializer_(deserializer),
563
        from_queue_(from_queue),
564 565
        to_queue_(to_queue),
        publish_handle_(std::move(publish_handle)) {}
566

567
  void Run(JobDelegate* delegate) override {
568
    CODE_SPACE_WRITE_SCOPE
569 570 571 572
    do {
      auto batch = from_queue_->Pop();
      if (batch.empty()) break;
      for (const auto& unit : batch) {
573 574
        deserializer_->CopyAndRelocate(unit);
      }
575 576
      to_queue_->Add(std::move(batch));
      publish_handle_->NotifyConcurrencyIncrease();
577
    } while (!delegate->ShouldYield());
578 579
  }

580 581
  size_t GetMaxConcurrency(size_t /* worker_count */) const override {
    return from_queue_->NumBatches();
582 583 584
  }

 private:
585 586 587 588
  NativeModuleDeserializer* const deserializer_;
  DeserializationQueue* const from_queue_;
  DeserializationQueue* const to_queue_;
  std::shared_ptr<JobHandle> const publish_handle_;
589 590
};

591
class PublishTask : public JobTask {
592 593
 public:
  PublishTask(NativeModuleDeserializer* deserializer,
594 595
              DeserializationQueue* from_queue)
      : deserializer_(deserializer), from_queue_(from_queue) {}
596

597
  void Run(JobDelegate* delegate) override {
598
    WasmCodeRefScope code_scope;
599
    do {
600 601 602
      auto to_publish = from_queue_->PopAll();
      if (to_publish.empty()) break;
      deserializer_->Publish(std::move(to_publish));
603
    } while (!delegate->ShouldYield());
604 605
  }

606 607 608 609 610 611 612
  size_t GetMaxConcurrency(size_t worker_count) const override {
    // Publishing is sequential anyway, so never return more than 1. If a
    // worker is already running, don't spawn a second one.
    if (worker_count > 0) return 0;
    return std::min(size_t{1}, from_queue_->NumBatches());
  }

613
 private:
614 615
  NativeModuleDeserializer* const deserializer_;
  DeserializationQueue* const from_queue_;
616 617
};

618
NativeModuleDeserializer::NativeModuleDeserializer(NativeModule* native_module)
619
    : native_module_(native_module) {}
620 621 622

bool NativeModuleDeserializer::Read(Reader* reader) {
  DCHECK(!read_called_);
623
#ifdef DEBUG
624
  read_called_ = true;
625
#endif
626

627
  ReadHeader(reader);
628
  uint32_t total_fns = native_module_->num_functions();
629
  uint32_t first_wasm_fn = native_module_->num_imported_functions();
630

631
  WasmCodeRefScope wasm_code_ref_scope;
632 633 634 635

  DeserializationQueue reloc_queue;
  DeserializationQueue publish_queue;

636 637 638
  std::shared_ptr<JobHandle> publish_handle = V8::GetCurrentPlatform()->PostJob(
      TaskPriority::kUserVisible,
      std::make_unique<PublishTask>(this, &publish_queue));
639

640 641 642 643 644
  std::unique_ptr<JobHandle> copy_and_reloc_handle =
      V8::GetCurrentPlatform()->PostJob(
          TaskPriority::kUserVisible,
          std::make_unique<CopyAndRelocTask>(this, &reloc_queue, &publish_queue,
                                             publish_handle));
645

646
  std::vector<DeserializationUnit> batch;
647
  const byte* batch_start = reader->current_location();
648
  for (uint32_t i = first_wasm_fn; i < total_fns; ++i) {
649
    DeserializationUnit unit = ReadCode(i, reader);
650 651
    if (!unit.code) continue;
    batch.emplace_back(std::move(unit));
652 653 654
    uint64_t batch_size_in_bytes = reader->current_location() - batch_start;
    constexpr int kMinBatchSizeInBytes = 100000;
    if (batch_size_in_bytes >= kMinBatchSizeInBytes) {
655
      reloc_queue.Add(std::move(batch));
656
      DCHECK(batch.empty());
657
      batch_start = reader->current_location();
658
      copy_and_reloc_handle->NotifyConcurrencyIncrease();
659 660
    }
  }
661

662 663 664 665 666
  // We should have read the expected amount of code now, and should have fully
  // utilized the allocated code space.
  DCHECK_EQ(0, remaining_code_size_);
  DCHECK_EQ(0, current_code_space_.size());

667
  if (!batch.empty()) {
668
    reloc_queue.Add(std::move(batch));
669
    copy_and_reloc_handle->NotifyConcurrencyIncrease();
670 671
  }

672 673 674
  // Wait for all tasks to finish, while participating in their work.
  copy_and_reloc_handle->Join();
  publish_handle->Join();
675

676
  return reader->current_size() == 0;
677 678
}

679
void NativeModuleDeserializer::ReadHeader(Reader* reader) {
680
  remaining_code_size_ = reader->Read<size_t>();
681 682
}

683 684
DeserializationUnit NativeModuleDeserializer::ReadCode(int fn_index,
                                                       Reader* reader) {
685 686
  bool has_code = reader->Read<bool>();
  if (!has_code) {
687
    DCHECK(FLAG_wasm_lazy_compilation ||
688
           native_module_->enabled_features().has_compilation_hints());
689
    native_module_->UseLazyStub(fn_index);
690
    return {};
691
  }
692 693 694 695 696 697 698 699 700 701 702
  int constant_pool_offset = reader->Read<int>();
  int safepoint_table_offset = reader->Read<int>();
  int handler_table_offset = reader->Read<int>();
  int code_comment_offset = reader->Read<int>();
  int unpadded_binary_size = reader->Read<int>();
  int stack_slot_count = reader->Read<int>();
  int tagged_parameter_slots = reader->Read<int>();
  int code_size = reader->Read<int>();
  int reloc_size = reader->Read<int>();
  int source_position_size = reader->Read<int>();
  int protected_instructions_size = reader->Read<int>();
703
  WasmCode::Kind kind = reader->Read<WasmCode::Kind>();
704
  ExecutionTier tier = reader->Read<ExecutionTier>();
705

706 707 708 709 710 711 712 713 714 715 716
  DCHECK(IsAligned(code_size, kCodeAlignment));
  DCHECK_GE(remaining_code_size_, code_size);
  if (current_code_space_.size() < static_cast<size_t>(code_size)) {
    // Allocate the next code space. Don't allocate more than 90% of
    // {kMaxCodeSpaceSize}, to leave some space for jump tables.
    constexpr size_t kMaxReservation =
        RoundUp<kCodeAlignment>(WasmCodeAllocator::kMaxCodeSpaceSize * 9 / 10);
    size_t code_space_size = std::min(kMaxReservation, remaining_code_size_);
    current_code_space_ =
        native_module_->AllocateForDeserializedCode(code_space_size);
    DCHECK_EQ(current_code_space_.size(), code_space_size);
717 718 719
    current_jump_tables_ = native_module_->FindJumpTablesForRegion(
        base::AddressRegionOf(current_code_space_));
    DCHECK(current_jump_tables_.is_valid());
720 721
  }

722 723
  DeserializationUnit unit;
  unit.src_code_buffer = reader->ReadVector<byte>(code_size);
724 725
  auto reloc_info = reader->ReadVector<byte>(reloc_size);
  auto source_pos = reader->ReadVector<byte>(source_position_size);
726
  auto protected_instructions =
727
      reader->ReadVector<byte>(protected_instructions_size);
728 729 730 731 732 733 734

  Vector<uint8_t> instructions = current_code_space_.SubVector(0, code_size);
  current_code_space_ += code_size;
  remaining_code_size_ -= code_size;

  unit.code = native_module_->AddDeserializedCode(
      fn_index, instructions, stack_slot_count, tagged_parameter_slots,
735
      safepoint_table_offset, handler_table_offset, constant_pool_offset,
736
      code_comment_offset, unpadded_binary_size, protected_instructions,
737
      reloc_info, source_pos, kind, tier);
738
  unit.jump_tables = current_jump_tables_;
739 740 741 742 743 744 745
  return unit;
}

void NativeModuleDeserializer::CopyAndRelocate(
    const DeserializationUnit& unit) {
  base::Memcpy(unit.code->instructions().begin(), unit.src_code_buffer.begin(),
               unit.src_code_buffer.size());
746

747
  // Relocate the code.
748 749
  int mask = RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
             RelocInfo::ModeMask(RelocInfo::WASM_STUB_CALL) |
750 751 752
             RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
             RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED);
753 754
  for (RelocIterator iter(unit.code->instructions(), unit.code->reloc_info(),
                          unit.code->constant_pool(), mask);
755 756 757
       !iter.done(); iter.next()) {
    RelocInfo::Mode mode = iter.rinfo()->rmode();
    switch (mode) {
758 759
      case RelocInfo::WASM_CALL: {
        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
760
        Address target =
761
            native_module_->GetNearCallTargetForFunction(tag, unit.jump_tables);
762 763 764
        iter.rinfo()->set_wasm_call_address(target, SKIP_ICACHE_FLUSH);
        break;
      }
765 766 767
      case RelocInfo::WASM_STUB_CALL: {
        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
        DCHECK_LT(tag, WasmCode::kRuntimeStubCount);
768
        Address target = native_module_->GetNearRuntimeStubEntry(
769
            static_cast<WasmCode::RuntimeStubId>(tag), unit.jump_tables);
770 771 772
        iter.rinfo()->set_wasm_stub_call_address(target, SKIP_ICACHE_FLUSH);
        break;
      }
773 774
      case RelocInfo::EXTERNAL_REFERENCE: {
        uint32_t tag = GetWasmCalleeTag(iter.rinfo());
775
        Address address = ExternalReferenceList::Get().address_from_tag(tag);
776
        iter.rinfo()->set_target_external_reference(address, SKIP_ICACHE_FLUSH);
777
        break;
778
      }
779 780 781
      case RelocInfo::INTERNAL_REFERENCE:
      case RelocInfo::INTERNAL_REFERENCE_ENCODED: {
        Address offset = iter.rinfo()->target_internal_reference();
782
        Address target = unit.code->instruction_start() + offset;
783 784 785 786
        Assembler::deserialization_set_target_internal_reference_at(
            iter.rinfo()->pc(), target, mode);
        break;
      }
787 788
      default:
        UNREACHABLE();
789 790
    }
  }
791

792
  // Finally, flush the icache for that code.
793 794 795 796
  FlushInstructionCache(unit.code->instructions().begin(),
                        unit.code->instructions().size());
}

797 798
void NativeModuleDeserializer::Publish(std::vector<DeserializationUnit> batch) {
  DCHECK(!batch.empty());
799
  std::vector<std::unique_ptr<WasmCode>> codes;
800 801 802
  codes.reserve(batch.size());
  for (auto& unit : batch) {
    codes.emplace_back(std::move(unit).code);
803 804 805 806 807
  }
  auto published_codes = native_module_->PublishCode(VectorOf(codes));
  for (auto* wasm_code : published_codes) {
    wasm_code->MaybePrint();
    wasm_code->Validate();
808
  }
809 810
}

811 812 813 814 815 816 817
bool IsSupportedVersion(Vector<const byte> header) {
  if (header.size() < WasmSerializer::kHeaderSize) return false;
  byte current_version[WasmSerializer::kHeaderSize];
  Writer writer({current_version, WasmSerializer::kHeaderSize});
  WriteHeader(&writer);
  return memcmp(header.begin(), current_version, WasmSerializer::kHeaderSize) ==
         0;
818 819
}

820
MaybeHandle<WasmModuleObject> DeserializeNativeModule(
821
    Isolate* isolate, Vector<const byte> data,
822
    Vector<const byte> wire_bytes_vec, Vector<const char> source_url) {
823 824 825
  if (!IsWasmCodegenAllowed(isolate, isolate->native_context())) return {};
  if (!IsSupportedVersion(data)) return {};

826 827 828 829
  // Make the copy of the wire bytes early, so we use the same memory for
  // decoding, lookup in the native module cache, and insertion into the cache.
  auto owned_wire_bytes = OwnedVector<uint8_t>::Of(wire_bytes_vec);

830
  // TODO(titzer): module features should be part of the serialization format.
831
  WasmEngine* wasm_engine = isolate->wasm_engine();
832
  WasmFeatures enabled_features = WasmFeatures::FromIsolate(isolate);
833
  ModuleResult decode_result = DecodeWasmModule(
834
      enabled_features, owned_wire_bytes.start(), owned_wire_bytes.end(), false,
835 836 837
      i::wasm::kWasmOrigin, isolate->counters(), isolate->metrics_recorder(),
      isolate->GetOrRegisterRecorderContextId(isolate->native_context()),
      DecodingMethod::kDeserialize, wasm_engine->allocator());
838
  if (decode_result.failed()) return {};
839
  std::shared_ptr<WasmModule> module = std::move(decode_result).value();
840
  CHECK_NOT_NULL(module);
841

842
  auto shared_native_module = wasm_engine->MaybeGetNativeModule(
843
      module->origin, owned_wire_bytes.as_vector(), isolate);
844 845 846 847 848 849 850
  if (shared_native_module == nullptr) {
    const bool kIncludeLiftoff = false;
    size_t code_size_estimate =
        wasm::WasmCodeManager::EstimateNativeModuleCodeSize(module.get(),
                                                            kIncludeLiftoff);
    shared_native_module = wasm_engine->NewNativeModule(
        isolate, enabled_features, std::move(module), code_size_estimate);
851 852 853 854 855 856
    // We have to assign a compilation ID here, as it is required for a
    // potential re-compilation, e.g. triggered by
    // {TierDownAllModulesPerIsolate}. The value is -2 so that it is different
    // than the compilation ID of actual compilations, and also different than
    // the sentinel value of the CompilationState.
    shared_native_module->compilation_state()->set_compilation_id(-2);
857
    shared_native_module->SetWireBytes(std::move(owned_wire_bytes));
858 859

    NativeModuleDeserializer deserializer(shared_native_module.get());
860
    Reader reader(data + WasmSerializer::kHeaderSize);
861
    bool error = !deserializer.Read(&reader);
862
    shared_native_module->compilation_state()->InitializeAfterDeserialization();
863
    wasm_engine->UpdateNativeModuleCache(error, &shared_native_module, isolate);
864 865 866
    if (error) return {};
  }

867 868 869
  Handle<FixedArray> export_wrappers;
  CompileJsToWasmWrappers(isolate, shared_native_module->module(),
                          &export_wrappers);
870

871
  Handle<Script> script =
872
      wasm_engine->GetOrCreateScript(isolate, shared_native_module, source_url);
873
  Handle<WasmModuleObject> module_object = WasmModuleObject::New(
874
      isolate, shared_native_module, script, export_wrappers);
875

876 877 878
  // Finish the Wasm script now and make it public to the debugger.
  isolate->debug()->OnAfterCompile(script);

879
  // Log the code within the generated module for profiling.
880
  shared_native_module->LogWasmCodes(isolate, *script);
881

882
  return module_object;
883 884 885 886 887
}

}  // namespace wasm
}  // namespace internal
}  // namespace v8