jump-table-assembler.h 12.3 KB
Newer Older
1 2 3 4
// Copyright 2018 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6 7 8
#if !V8_ENABLE_WEBASSEMBLY
#error This header should only be included if WebAssembly is enabled.
#endif  // !V8_ENABLE_WEBASSEMBLY

9 10 11
#ifndef V8_WASM_JUMP_TABLE_ASSEMBLER_H_
#define V8_WASM_JUMP_TABLE_ASSEMBLER_H_

12
#include "src/codegen/macro-assembler.h"
13 14 15 16 17

namespace v8 {
namespace internal {
namespace wasm {

18 19 20 21 22
// The jump table is the central dispatch point for all (direct and indirect)
// invocations in WebAssembly. It holds one slot per function in a module, with
// each slot containing a dispatch to the currently published {WasmCode} that
// corresponds to the function.
//
23 24
// Additionally to this main jump table, there exist special jump tables for
// other purposes:
25
// - the far stub table contains one entry per wasm runtime stub (see
26
//   {WasmCode::RuntimeStubId}, which jumps to the corresponding embedded
27 28
//   builtin, plus (if not the full address space can be reached via the jump
//   table) one entry per wasm function.
29 30 31 32 33
// - the lazy compile table contains one entry per wasm function which jumps to
//   the common {WasmCompileLazy} builtin and passes the function index that was
//   invoked.
//
// The main jump table is split into lines of fixed size, with lines laid out
34 35 36 37 38 39 40 41 42
// consecutively within the executable memory of the {NativeModule}. The slots
// in turn are consecutive within a line, but do not cross line boundaries.
//
//   +- L1 -------------------+ +- L2 -------------------+ +- L3 ...
//   | S1 | S2 | ... | Sn | x | | S1 | S2 | ... | Sn | x | | S1  ...
//   +------------------------+ +------------------------+ +---- ...
//
// The above illustrates jump table lines {Li} containing slots {Si} with each
// line containing {n} slots and some padding {x} for alignment purposes.
43
// Other jump tables are just consecutive.
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
//
// The main jump table will be patched concurrently while other threads execute
// it. The code at the new target might also have been emitted concurrently, so
// we need to ensure that there is proper synchronization between code emission,
// jump table patching and code execution.
// On Intel platforms, this all works out of the box because there is cache
// coherency between i-cache and d-cache.
// On ARM, it is safe because the i-cache flush after code emission executes an
// "ic ivau" (Instruction Cache line Invalidate by Virtual Address to Point of
// Unification), which broadcasts to all cores. A core which sees the jump table
// update thus also sees the new code. Since the other core does not explicitly
// execute an "isb" (Instruction Synchronization Barrier), it might still
// execute the old code afterwards, which is no problem, since that code remains
// available until it is garbage collected. Garbage collection itself is a
// synchronization barrier though.
59
class V8_EXPORT_PRIVATE JumpTableAssembler : public MacroAssembler {
60
 public:
61 62 63 64 65 66 67 68 69 70
  // Translate an offset into the continuous jump table to a jump table index.
  static uint32_t SlotOffsetToIndex(uint32_t slot_offset) {
    uint32_t line_index = slot_offset / kJumpTableLineSize;
    uint32_t line_offset = slot_offset % kJumpTableLineSize;
    DCHECK_EQ(0, line_offset % kJumpTableSlotSize);
    return line_index * kJumpTableSlotsPerLine +
           line_offset / kJumpTableSlotSize;
  }

  // Translate a jump table index to an offset into the continuous jump table.
71
  static uint32_t JumpSlotIndexToOffset(uint32_t slot_index) {
72 73 74 75 76 77 78 79 80 81 82 83 84
    uint32_t line_index = slot_index / kJumpTableSlotsPerLine;
    uint32_t line_offset =
        (slot_index % kJumpTableSlotsPerLine) * kJumpTableSlotSize;
    return line_index * kJumpTableLineSize + line_offset;
  }

  // Determine the size of a jump table containing the given number of slots.
  static constexpr uint32_t SizeForNumberOfSlots(uint32_t slot_count) {
    return ((slot_count + kJumpTableSlotsPerLine - 1) /
            kJumpTableSlotsPerLine) *
           kJumpTableLineSize;
  }

85 86 87 88 89
  // Translate a far jump table index to an offset into the table.
  static uint32_t FarJumpSlotIndexToOffset(uint32_t slot_index) {
    return slot_index * kFarJumpTableSlotSize;
  }

90 91 92 93 94 95
  // Translate a far jump table offset to the index into the table.
  static uint32_t FarJumpSlotOffsetToIndex(uint32_t offset) {
    DCHECK_EQ(0, offset % kFarJumpTableSlotSize);
    return offset / kFarJumpTableSlotSize;
  }

96 97
  // Determine the size of a far jump table containing the given number of
  // slots.
98
  static constexpr uint32_t SizeForNumberOfFarJumpSlots(
99 100
      int num_runtime_slots, int num_function_slots) {
    int num_entries = num_runtime_slots + num_function_slots;
101
    return num_entries * kFarJumpTableSlotSize;
102 103
  }

104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
  // Translate a slot index to an offset into the lazy compile table.
  static uint32_t LazyCompileSlotIndexToOffset(uint32_t slot_index) {
    return slot_index * kLazyCompileTableSlotSize;
  }

  // Determine the size of a lazy compile table.
  static constexpr uint32_t SizeForNumberOfLazyFunctions(uint32_t slot_count) {
    return slot_count * kLazyCompileTableSlotSize;
  }

  static void GenerateLazyCompileTable(Address base, uint32_t num_slots,
                                       uint32_t num_imported_functions,
                                       Address wasm_compile_lazy_target) {
    uint32_t lazy_compile_table_size = num_slots * kLazyCompileTableSlotSize;
    // Assume enough space, so the Assembler does not try to grow the buffer.
    JumpTableAssembler jtasm(base, lazy_compile_table_size + 256);
    for (uint32_t slot_index = 0; slot_index < num_slots; ++slot_index) {
      DCHECK_EQ(slot_index * kLazyCompileTableSlotSize, jtasm.pc_offset());
      jtasm.EmitLazyCompileJumpSlot(slot_index + num_imported_functions,
                                    wasm_compile_lazy_target);
124
    }
125 126
    DCHECK_EQ(lazy_compile_table_size, jtasm.pc_offset());
    FlushInstructionCache(base, lazy_compile_table_size);
127 128
  }

129
  static void GenerateFarJumpTable(Address base, Address* stub_targets,
130 131
                                   int num_runtime_slots,
                                   int num_function_slots) {
132
    uint32_t table_size =
133
        SizeForNumberOfFarJumpSlots(num_runtime_slots, num_function_slots);
134 135 136
    // Assume enough space, so the Assembler does not try to grow the buffer.
    JumpTableAssembler jtasm(base, table_size + 256);
    int offset = 0;
137 138
    for (int index = 0; index < num_runtime_slots + num_function_slots;
         ++index) {
139
      DCHECK_EQ(offset, FarJumpSlotIndexToOffset(index));
140 141
      // Functions slots initially jump to themselves. They are patched before
      // being used.
142 143
      Address target =
          index < num_runtime_slots ? stub_targets[index] : base + offset;
144
      jtasm.EmitFarJumpSlot(target);
145
      offset += kFarJumpTableSlotSize;
146
      DCHECK_EQ(offset, jtasm.pc_offset());
147
    }
148
    FlushInstructionCache(base, table_size);
149 150
  }

151 152 153 154 155 156 157 158 159 160 161
  static void PatchJumpTableSlot(Address jump_table_slot,
                                 Address far_jump_table_slot, Address target) {
    // First, try to patch the jump table slot.
    JumpTableAssembler jtasm(jump_table_slot);
    if (!jtasm.EmitJumpSlot(target)) {
      // If that fails, we need to patch the far jump table slot, and then
      // update the jump table slot to jump to this far jump table slot.
      DCHECK_NE(kNullAddress, far_jump_table_slot);
      JumpTableAssembler::PatchFarJumpSlot(far_jump_table_slot, target);
      CHECK(jtasm.EmitJumpSlot(far_jump_table_slot));
    }
162
    jtasm.NopBytes(kJumpTableSlotSize - jtasm.pc_offset());
163
    FlushInstructionCache(jump_table_slot, kJumpTableSlotSize);
164 165 166
  }

 private:
167 168
  // Instantiate a {JumpTableAssembler} for patching.
  explicit JumpTableAssembler(Address slot_addr, int size = 256)
169
      : MacroAssembler(nullptr, JumpTableAssemblerOptions(),
170 171 172
                       CodeObjectRequired::kNo,
                       ExternalAssemblerBuffer(
                           reinterpret_cast<uint8_t*>(slot_addr), size)) {}
173

174 175 176
// To allow concurrent patching of the jump table entries, we need to ensure
// that the instruction containing the call target does not cross cache-line
// boundaries. The jump table line size has been chosen to satisfy this.
177
#if V8_TARGET_ARCH_X64
178
  static constexpr int kJumpTableLineSize = 64;
179
  static constexpr int kJumpTableSlotSize = 5;
180
  static constexpr int kFarJumpTableSlotSize = 16;
181
  static constexpr int kLazyCompileTableSlotSize = 10;
182
#elif V8_TARGET_ARCH_IA32
183
  static constexpr int kJumpTableLineSize = 64;
184
  static constexpr int kJumpTableSlotSize = 5;
185
  static constexpr int kFarJumpTableSlotSize = 5;
186
  static constexpr int kLazyCompileTableSlotSize = 10;
187
#elif V8_TARGET_ARCH_ARM
188 189
  static constexpr int kJumpTableLineSize = 3 * kInstrSize;
  static constexpr int kJumpTableSlotSize = 3 * kInstrSize;
190
  static constexpr int kFarJumpTableSlotSize = 2 * kInstrSize;
191
  static constexpr int kLazyCompileTableSlotSize = 5 * kInstrSize;
192 193 194 195 196 197
#elif V8_TARGET_ARCH_ARM64 && V8_ENABLE_CONTROL_FLOW_INTEGRITY
  static constexpr int kJumpTableLineSize = 2 * kInstrSize;
  static constexpr int kJumpTableSlotSize = 2 * kInstrSize;
  static constexpr int kFarJumpTableSlotSize = 6 * kInstrSize;
  static constexpr int kLazyCompileTableSlotSize = 4 * kInstrSize;
#elif V8_TARGET_ARCH_ARM64 && !V8_ENABLE_CONTROL_FLOW_INTEGRITY
198 199
  static constexpr int kJumpTableLineSize = 1 * kInstrSize;
  static constexpr int kJumpTableSlotSize = 1 * kInstrSize;
200
  static constexpr int kFarJumpTableSlotSize = 4 * kInstrSize;
201
  static constexpr int kLazyCompileTableSlotSize = 3 * kInstrSize;
202
#elif V8_TARGET_ARCH_S390X
203
  static constexpr int kJumpTableLineSize = 128;
204
  static constexpr int kJumpTableSlotSize = 14;
205
  static constexpr int kFarJumpTableSlotSize = 14;
206
  static constexpr int kLazyCompileTableSlotSize = 20;
207
#elif V8_TARGET_ARCH_PPC64
208
  static constexpr int kJumpTableLineSize = 64;
209
  static constexpr int kJumpTableSlotSize = 7 * kInstrSize;
210
  static constexpr int kFarJumpTableSlotSize = 7 * kInstrSize;
211
  static constexpr int kLazyCompileTableSlotSize = 12 * kInstrSize;
212
#elif V8_TARGET_ARCH_MIPS
213 214
  static constexpr int kJumpTableLineSize = 8 * kInstrSize;
  static constexpr int kJumpTableSlotSize = 8 * kInstrSize;
215
  static constexpr int kFarJumpTableSlotSize = 4 * kInstrSize;
216
  static constexpr int kLazyCompileTableSlotSize = 6 * kInstrSize;
217
#elif V8_TARGET_ARCH_MIPS64
218
  static constexpr int kJumpTableLineSize = 8 * kInstrSize;
219
  static constexpr int kJumpTableSlotSize = 8 * kInstrSize;
220
  static constexpr int kFarJumpTableSlotSize = 6 * kInstrSize;
221
  static constexpr int kLazyCompileTableSlotSize = 8 * kInstrSize;
Brice Dobry's avatar
Brice Dobry committed
222 223 224 225 226
#elif V8_TARGET_ARCH_RISCV64
  static constexpr int kJumpTableLineSize = 6 * kInstrSize;
  static constexpr int kJumpTableSlotSize = 6 * kInstrSize;
  static constexpr int kFarJumpTableSlotSize = 6 * kInstrSize;
  static constexpr int kLazyCompileTableSlotSize = 10 * kInstrSize;
227 228 229 230 231
#elif V8_TARGET_ARCH_LOONG64
  static constexpr int kJumpTableLineSize = 8 * kInstrSize;
  static constexpr int kJumpTableSlotSize = 8 * kInstrSize;
  static constexpr int kFarJumpTableSlotSize = 4 * kInstrSize;
  static constexpr int kLazyCompileTableSlotSize = 8 * kInstrSize;
232
#else
233
#error Unknown architecture.
234 235
#endif

236 237
  static constexpr int kJumpTableSlotsPerLine =
      kJumpTableLineSize / kJumpTableSlotSize;
238
  STATIC_ASSERT(kJumpTableSlotsPerLine >= 1);
239

240 241 242 243 244 245 246 247 248 249
  // {JumpTableAssembler} is never used during snapshot generation, and its code
  // must be independent of the code range of any isolate anyway. Just ensure
  // that no relocation information is recorded, there is no buffer to store it
  // since it is instantiated in patching mode in existing code directly.
  static AssemblerOptions JumpTableAssemblerOptions() {
    AssemblerOptions options;
    options.disable_reloc_info_for_patching = true;
    return options;
  }

250 251 252
  void EmitLazyCompileJumpSlot(uint32_t func_index,
                               Address lazy_compile_target);

253 254
  // Returns {true} if the jump fits in the jump table slot, {false} otherwise.
  bool EmitJumpSlot(Address target);
255

256
  // Initially emit a far jump slot.
257 258
  void EmitFarJumpSlot(Address target);

259 260 261 262
  // Patch an existing far jump slot, and make sure that this updated eventually
  // becomes available to all execution units that might execute this code.
  static void PatchFarJumpSlot(Address slot, Address target);

263
  void NopBytes(int bytes);
264 265 266 267 268 269 270
};

}  // namespace wasm
}  // namespace internal
}  // namespace v8

#endif  // V8_WASM_JUMP_TABLE_ASSEMBLER_H_