[wasm][memory64] Start implementing the memory64 proposal

This is a first small step for implementing the memory64 proposal:
1. Add a feature flag.
2. Add the 0x04 and 0x05 limits flag for memory64.
3. Read memory limits as LEB-encoded u64 (instead of u32) if a memory64
   limit flag was read.
4. Unify {MaximumFlag} and {MemoryFlag}, which was used inconsistently
   before.
5. Add test for memory limits encoded with >5 bytes.
6. Move some macros from module-decoder-unittest.cc to wasm-macro-gen.h.

Note that still the same limits for the maximum number of pages applies
as before, i.e. you cannot specify a memory >4GB yet. But you can encode
that small number in >5 bytes.

R=manoskouk@chromium.org

Bug: v8:10949
Change-Id: I90a4f08426ae714a67440281785eb00cfc24a349
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2423712
Commit-Queue: Clemens Backes <clemensb@chromium.org>
Reviewed-by: Manos Koukoutos <manoskouk@chromium.org>
Cr-Commit-Position: refs/heads/master@{#70110}

src/wasm/decoder.h

@@ -196,6 +196,13 @@ class Decoder {
     return read_leb<int32_t, kValidate, kAdvancePc, kTrace>(pc_, &length, name);
   }
 
+  // Reads a LEB128 variable-length unsigned 64-bit integer and advances {pc_}.
+  uint64_t consume_u64v(const char* name = nullptr) {
+    uint32_t length = 0;
+    return read_leb<uint64_t, kValidate, kAdvancePc, kTrace>(pc_, &length,
+                                                             name);
+  }
+
   // Consume {size} bytes and send them to the bit bucket, advancing {pc_}.
   void consume_bytes(uint32_t size, const char* name = "skip") {
     // Only trace if the name is not null.

src/wasm/module-decoder.cc

@@ -1607,33 +1607,44 @@ class ModuleDecoderImpl : public Decoder {
   }
 
   uint8_t validate_table_flags(const char* name) {
-    uint8_t flags = consume_u8("resizable limits flags");
-    const byte* pos = pc();
-    if (flags & 0xFE) {
-      errorf(pos - 1, "invalid %s limits flags", name);
+    uint8_t flags = consume_u8("table limits flags");
+    STATIC_ASSERT(kNoMaximum < kWithMaximum);
+    if (V8_UNLIKELY(flags > kWithMaximum)) {
+      errorf(pc() - 1, "invalid %s limits flags", name);
     }
     return flags;
   }
 
   uint8_t validate_memory_flags(bool* has_shared_memory) {
-    uint8_t flags = consume_u8("resizable limits flags");
-    const byte* pos = pc();
+    uint8_t flags = consume_u8("memory limits flags");
     *has_shared_memory = false;
-    if (enabled_features_.has_threads()) {
-      if (flags & 0xFC) {
-        errorf(pos - 1, "invalid memory limits flags");
-      } else if (flags == 3) {
-        DCHECK_NOT_NULL(has_shared_memory);
+    switch (flags) {
+      case kNoMaximum:
+      case kWithMaximum:
+        break;
+      case kSharedNoMaximum:
+      case kSharedWithMaximum:
+        if (!enabled_features_.has_threads()) {
+          errorf(pc() - 1,
+                 "invalid memory limits flags (enable via "
+                 "--experimental-wasm-threads)");
+        }
         *has_shared_memory = true;
-      } else if (flags == 2) {
-        errorf(pos - 1,
-               "memory limits flags should have maximum defined if shared is "
-               "true");
-      }
-    } else {
-      if (flags & 0xFE) {
-        errorf(pos - 1, "invalid memory limits flags");
-      }
+        // V8 does not support shared memory without a maximum.
+        if (flags == kSharedNoMaximum) {
+          errorf(pc() - 1,
+                 "memory limits flags must have maximum defined if shared is "
+                 "true");
+        }
+        break;
+      case kMemory64NoMaximum:
+      case kMemory64WithMaximum:
+        if (!enabled_features_.has_memory64()) {
+          errorf(pc() - 1,
+                 "invalid memory limits flags (enable via "
+                 "--experimental-wasm-memory64)");
+        }
+        break;
     }
     return flags;
   }
@@ -1643,27 +1654,36 @@ class ModuleDecoderImpl : public Decoder {
                                 bool* has_max, uint32_t max_maximum,
                                 uint32_t* maximum, uint8_t flags) {
     const byte* pos = pc();
-    *initial = consume_u32v("initial size");
-    *has_max = false;
-    if (*initial > max_initial) {
+    // For memory64 we need to read the numbers as LEB-encoded 64-bit unsigned
+    // integer. All V8 limits are still within uint32_t range though.
+    const bool is_memory64 =
+        flags == kMemory64NoMaximum || flags == kMemory64WithMaximum;
+    uint64_t initial_64 = is_memory64 ? consume_u64v("initial size")
+                                      : consume_u32v("initial size");
+    if (initial_64 > max_initial) {
       errorf(pos,
-             "initial %s size (%u %s) is larger than implementation limit (%u)",
-             name, *initial, units, max_initial);
+             "initial %s size (%" PRIu64
+             " %s) is larger than implementation limit (%u)",
+             name, initial_64, units, max_initial);
     }
+    *initial = static_cast<uint32_t>(initial_64);
     if (flags & 1) {
       *has_max = true;
       pos = pc();
-      *maximum = consume_u32v("maximum size");
-      if (*maximum > max_maximum) {
-        errorf(
-            pos,
-            "maximum %s size (%u %s) is larger than implementation limit (%u)",
-            name, *maximum, units, max_maximum);
+      uint64_t maximum_64 = is_memory64 ? consume_u64v("maximum size")
+                                        : consume_u32v("maximum size");
+      if (maximum_64 > max_maximum) {
+        errorf(pos,
+               "maximum %s size (%" PRIu64
+               " %s) is larger than implementation limit (%u)",
+               name, maximum_64, units, max_maximum);
       }
-      if (*maximum < *initial) {
-        errorf(pos, "maximum %s size (%u %s) is less than initial (%u %s)",
-               name, *maximum, units, *initial, units);
+      if (maximum_64 < *initial) {
+        errorf(pos,
+               "maximum %s size (%" PRIu64 " %s) is less than initial (%u %s)",
+               name, maximum_64, units, *initial, units);
       }
+      *maximum = static_cast<uint32_t>(maximum_64);
     } else {
       *has_max = false;
       *maximum = max_initial;

src/wasm/wasm-constants.h

@@ -56,14 +56,13 @@ enum ImportExportKindCode : uint8_t {
   kExternalException = 4
 };
 
-// Binary encoding of maximum and shared flags for memories.
-enum MaximumFlag : uint8_t { kNoMaximumFlag = 0, kHasMaximumFlag = 1 };
-
-enum MemoryFlags : uint8_t {
-  kNoMaximum = 0,
-  kMaximum = 1,
-  kSharedNoMaximum = 2,
-  kSharedAndMaximum = 3
+enum LimitsFlags : uint8_t {
+  kNoMaximum = 0x00,           // Also valid for table limits.
+  kWithMaximum = 0x01,         // Also valid for table limits.
+  kSharedNoMaximum = 0x02,     // Only valid for memory limits.
+  kSharedWithMaximum = 0x03,   // Only valid for memory limits.
+  kMemory64NoMaximum = 0x04,   // Only valid for memory limits.
+  kMemory64WithMaximum = 0x05  // Only valid for memory limits.
 };
 
 // Flags for data and element segments.

src/wasm/wasm-feature-flags.h

@@ -28,7 +28,12 @@
   /* Typed function references proposal. */                                    \
   /* Official proposal: https://github.com/WebAssembly/function-references */  \
   /* V8 side owner: ahaas */                                                   \
-  V(typed_funcref, "typed function references", false)
+  V(typed_funcref, "typed function references", false)                         \
+                                                                               \
+  /* Memory64 proposal. */                                                     \
+  /* https://github.com/WebAssembly/memory64 */                                \
+  /* V8 side owner: clemensb */                                                \
+  V(memory64, "memory64", false)
 
 // #############################################################################
 // Staged features (disabled by default, but enabled via --wasm-staging (also

src/wasm/wasm-module-builder.cc

@@ -598,7 +598,7 @@ void WasmModuleBuilder::WriteTo(ZoneBuffer* buffer) const {
     buffer->write_size(tables_.size());
     for (const WasmTable& table : tables_) {
       buffer->write_u8(table.type.value_type_code());
-      buffer->write_u8(table.has_maximum ? kHasMaximumFlag : kNoMaximumFlag);
+      buffer->write_u8(table.has_maximum ? kWithMaximum : kNoMaximum);
       buffer->write_size(table.min_size);
       if (table.has_maximum) buffer->write_size(table.max_size);
     }
@@ -610,11 +610,10 @@ void WasmModuleBuilder::WriteTo(ZoneBuffer* buffer) const {
     size_t start = EmitSection(kMemorySectionCode, buffer);
     buffer->write_u8(1);  // memory count
     if (has_shared_memory_) {
-      buffer->write_u8(has_max_memory_size_ ? MemoryFlags::kSharedAndMaximum
-                                            : MemoryFlags::kSharedNoMaximum);
+      buffer->write_u8(has_max_memory_size_ ? kSharedWithMaximum
+                                            : kSharedNoMaximum);
     } else {
-      buffer->write_u8(has_max_memory_size_ ? MemoryFlags::kMaximum
-                                            : MemoryFlags::kNoMaximum);
+      buffer->write_u8(has_max_memory_size_ ? kWithMaximum : kNoMaximum);
     }
     buffer->write_u32v(min_memory_size_);
     if (has_max_memory_size_) {

test/cctest/wasm/test-run-wasm-module.cc

@@ -814,7 +814,7 @@ TEST(InitDataAtTheUpperLimit) {
         kMemorySectionCode,   // --
         U32V_1(4),            // section size
         ENTRY_COUNT(1),       // --
-        kHasMaximumFlag,      // --
+        kWithMaximum,         // --
         1,                    // initial size
         2,                    // maximum size
         kDataSectionCode,     // --
@@ -850,7 +850,7 @@ TEST(EmptyMemoryNonEmptyDataSegment) {
         kMemorySectionCode,  // --
         U32V_1(4),           // section size
         ENTRY_COUNT(1),      // --
-        kHasMaximumFlag,     // --
+        kWithMaximum,        // --
         0,                   // initial size
         0,                   // maximum size
         kDataSectionCode,    // --
@@ -884,7 +884,7 @@ TEST(EmptyMemoryEmptyDataSegment) {
         kMemorySectionCode,  // --
         U32V_1(4),           // section size
         ENTRY_COUNT(1),      // --
-        kHasMaximumFlag,     // --
+        kWithMaximum,        // --
         0,                   // initial size
         0,                   // maximum size
         kDataSectionCode,    // --
@@ -919,7 +919,7 @@ TEST(MemoryWithOOBEmptyDataSegment) {
         kMemorySectionCode,      // --
         U32V_1(4),               // section size
         ENTRY_COUNT(1),          // --
-        kHasMaximumFlag,         // --
+        kWithMaximum,            // --
         1,                       // initial size
         1,                       // maximum size
         kDataSectionCode,        // --

test/common/wasm/wasm-macro-gen.h

@@ -67,6 +67,18 @@
       static_cast<byte>((((x) >> 21) & MASK_7) | 0x80), \
       static_cast<byte>((((x) >> 28) & MASK_7))
 
+#define U64V_10(x)                                                 \
+  static_cast<uint8_t>((uint64_t{x} & MASK_7) | 0x80),             \
+      static_cast<uint8_t>(((uint64_t{x} >> 7) & MASK_7) | 0x80),  \
+      static_cast<uint8_t>(((uint64_t{x} >> 14) & MASK_7) | 0x80), \
+      static_cast<uint8_t>(((uint64_t{x} >> 21) & MASK_7) | 0x80), \
+      static_cast<uint8_t>(((uint64_t{x} >> 28) & MASK_7) | 0x80), \
+      static_cast<uint8_t>(((uint64_t{x} >> 35) & MASK_7) | 0x80), \
+      static_cast<uint8_t>(((uint64_t{x} >> 42) & MASK_7) | 0x80), \
+      static_cast<uint8_t>(((uint64_t{x} >> 49) & MASK_7) | 0x80), \
+      static_cast<uint8_t>(((uint64_t{x} >> 56) & MASK_7) | 0x80), \
+      static_cast<uint8_t>((uint64_t{x} >> 63) & MASK_7)
+
 // Convenience macros for building Wasm bytecode directly into a byte array.
 
 //------------------------------------------------------------------------------
@@ -209,6 +221,24 @@
 
 #define WASM_NO_LOCALS 0
 
+//------------------------------------------------------------------------------
+// Helpers for encoding sections and other fields with length prefix.
+//------------------------------------------------------------------------------
+
+template <typename... Args>
+std::integral_constant<size_t, sizeof...(Args)> CountArgsHelper(Args...);
+#define COUNT_ARGS(...) (decltype(CountArgsHelper(__VA_ARGS__))::value)
+
+template <size_t num>
+struct CheckLEB1 : std::integral_constant<size_t, num> {
+  static_assert(num <= I32V_MAX(1), "LEB range check");
+};
+#define CHECK_LEB1(num) CheckLEB1<num>::value
+
+#define ADD_COUNT(...) CHECK_LEB1(COUNT_ARGS(__VA_ARGS__)), __VA_ARGS__
+
+#define SECTION(name, ...) k##name##SectionCode, ADD_COUNT(__VA_ARGS__)
+
 namespace v8 {
 namespace internal {
 namespace wasm {

test/unittests/BUILD.gn

@@ -355,6 +355,7 @@ v8_source_set("unittests_sources") {
     "wasm/function-body-decoder-unittest.cc",
     "wasm/leb-helper-unittest.cc",
     "wasm/loop-assignment-analysis-unittest.cc",
+    "wasm/module-decoder-memory64-unittest.cc",
     "wasm/module-decoder-unittest.cc",
     "wasm/simd-shuffle-unittest.cc",
     "wasm/streaming-decoder-unittest.cc",

test/unittests/wasm/module-decoder-memory64-unittest.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/objects/objects-inl.h"
+#include "src/wasm/module-decoder.h"
+#include "src/wasm/wasm-features.h"
+#include "src/wasm/wasm-limits.h"
+#include "test/common/wasm/wasm-macro-gen.h"
+#include "test/unittests/test-utils.h"
+
+namespace v8 {
+namespace internal {
+namespace wasm {
+namespace module_decoder_unittest {
+
+#define EXPECT_OK(result)                                        \
+  do {                                                           \
+    if (!result.ok()) {                                          \
+      GTEST_NONFATAL_FAILURE_(result.error().message().c_str()); \
+      return;                                                    \
+    }                                                            \
+  } while (false)
+
+class Memory64DecodingTest : public TestWithIsolateAndZone {
+ public:
+  ModuleResult DecodeModule(std::initializer_list<uint8_t> module_body_bytes) {
+    // Add the wasm magic and version number automatically.
+    std::vector<uint8_t> module_bytes{WASM_MODULE_HEADER};
+    module_bytes.insert(module_bytes.end(), module_body_bytes);
+    static constexpr WasmFeatures kEnabledFeatures{
+        WasmFeature::kFeature_memory64};
+    return DecodeWasmModule(
+        kEnabledFeatures, module_bytes.data(),
+        module_bytes.data() + module_bytes.size(), false, kWasmOrigin,
+        isolate()->counters(), isolate()->metrics_recorder(),
+        v8::metrics::Recorder::ContextId::Empty(), DecodingMethod::kSync,
+        isolate()->wasm_engine()->allocator());
+  }
+};
+
+TEST_F(Memory64DecodingTest, MemoryLimitLEB64) {
+  // 2 bytes LEB (32-bit range), no maximum.
+  ModuleResult result = DecodeModule(
+      {SECTION(Memory, ENTRY_COUNT(1), kMemory64NoMaximum, U32V_2(5))});
+  EXPECT_OK(result);
+  EXPECT_EQ(5u, result.value()->initial_pages);
+  EXPECT_EQ(false, result.value()->has_maximum_pages);
+
+  // 2 bytes LEB (32-bit range), with maximum.
+  result = DecodeModule({SECTION(Memory, ENTRY_COUNT(1), kMemory64WithMaximum,
+                                 U32V_2(7), U32V_2(47))});
+  EXPECT_OK(result);
+  EXPECT_EQ(7u, result.value()->initial_pages);
+  EXPECT_EQ(true, result.value()->has_maximum_pages);
+  EXPECT_EQ(47u, result.value()->maximum_pages);
+
+  // 10 bytes LEB, 32-bit range, no maximum.
+  result = DecodeModule(
+      {SECTION(Memory, ENTRY_COUNT(1), kMemory64NoMaximum, U64V_10(2))});
+  EXPECT_OK(result);
+  EXPECT_EQ(2u, result.value()->initial_pages);
+  EXPECT_EQ(false, result.value()->has_maximum_pages);
+
+  // 10 bytes LEB, 32-bit range, with maximum.
+  result = DecodeModule({SECTION(Memory, ENTRY_COUNT(1), kMemory64WithMaximum,
+                                 U64V_10(2), U64V_10(6))});
+  EXPECT_OK(result);
+  EXPECT_EQ(2u, result.value()->initial_pages);
+  EXPECT_EQ(true, result.value()->has_maximum_pages);
+  EXPECT_EQ(6u, result.value()->maximum_pages);
+
+  // TODO(clemensb): Test numbers outside the 32-bit range once that's
+  // supported.
+}
+
+}  // namespace module_decoder_unittest
+}  // namespace wasm
+}  // namespace internal
+}  // namespace v8

test/unittests/wasm/module-decoder-unittest.cc

@@ -44,20 +44,6 @@ namespace module_decoder_unittest {
 
 #define UNKNOWN_SECTION(size) 0, U32V_1(size + 5), ADD_COUNT('l', 'u', 'l', 'z')
 
-template <typename... Args>
-std::integral_constant<size_t, sizeof...(Args)> CountArgsHelper(Args...);
-#define COUNT_ARGS(...) (decltype(CountArgsHelper(__VA_ARGS__))::value)
-
-template <size_t num>
-struct CheckLEB1 : std::integral_constant<size_t, num> {
-  static_assert(num <= I32V_MAX(1), "LEB range check");
-};
-#define CHECK_LEB1(num) CheckLEB1<num>::value
-
-#define ADD_COUNT(...) CHECK_LEB1(COUNT_ARGS(__VA_ARGS__)), __VA_ARGS__
-
-#define SECTION(name, ...) k##name##SectionCode, ADD_COUNT(__VA_ARGS__)
-
 #define SIGNATURES_SECTION(count, ...) SECTION(Type, U32V_1(count), __VA_ARGS__)
 #define FUNCTION_SIGNATURES_SECTION(count, ...) \
   SECTION(Function, U32V_1(count), __VA_ARGS__)
@@ -1200,7 +1186,7 @@ TEST_F(WasmModuleVerifyTest, DataSegmentWithImmutableImportedGlobal) {
               kExternalGlobal,  // import kind
               kLocalI32,        // type
               0),               // mutability
-      SECTION(Memory, ENTRY_COUNT(1), kHasMaximumFlag, 28, 28),
+      SECTION(Memory, ENTRY_COUNT(1), kWithMaximum, 28, 28),
       SECTION(Data, ENTRY_COUNT(1), LINEAR_MEMORY_INDEX_0,
               WASM_INIT_EXPR_GLOBAL(1),  // dest addr
               U32V_1(3),                 // source size
@@ -1223,7 +1209,7 @@ TEST_F(WasmModuleVerifyTest, DataSegmentWithMutableImportedGlobal) {
               kExternalGlobal,  // import kind
               kLocalI32,        // type
               1),               // mutability
-      SECTION(Memory, ENTRY_COUNT(1), kHasMaximumFlag, 28, 28),
+      SECTION(Memory, ENTRY_COUNT(1), kWithMaximum, 28, 28),
       SECTION(Data, ENTRY_COUNT(1), LINEAR_MEMORY_INDEX_0,
               WASM_INIT_EXPR_GLOBAL(0),  // dest addr
               U32V_1(3),                 // source size
@@ -1234,7 +1220,7 @@ TEST_F(WasmModuleVerifyTest, DataSegmentWithMutableImportedGlobal) {
 TEST_F(WasmModuleVerifyTest, DataSegmentWithImmutableGlobal) {
   // Only an immutable imported global can be used as an init_expr.
   const byte data[] = {
-      SECTION(Memory, ENTRY_COUNT(1), kHasMaximumFlag, 28, 28),
+      SECTION(Memory, ENTRY_COUNT(1), kWithMaximum, 28, 28),
       SECTION(Global, ENTRY_COUNT(1),
               kLocalI32,                        // local type
               0,                                // immutable
@@ -1250,7 +1236,7 @@ TEST_F(WasmModuleVerifyTest, DataSegmentWithImmutableGlobal) {
 TEST_F(WasmModuleVerifyTest, OneDataSegment) {
   const byte kDataSegmentSourceOffset = 24;
   const byte data[] = {
-      SECTION(Memory, ENTRY_COUNT(1), kHasMaximumFlag, 28, 28),
+      SECTION(Memory, ENTRY_COUNT(1), kWithMaximum, 28, 28),
       SECTION(Data, ENTRY_COUNT(1), LINEAR_MEMORY_INDEX_0,
               WASM_INIT_EXPR_I32V_3(0x9BBAA),  // dest addr
               U32V_1(3),                       // source size
@@ -1281,7 +1267,7 @@ TEST_F(WasmModuleVerifyTest, TwoDataSegments) {
   const byte kDataSegment1SourceOffset = kDataSegment0SourceOffset + 11;
 
   const byte data[] = {
-      SECTION(Memory, ENTRY_COUNT(1), kHasMaximumFlag, 28, 28),
+      SECTION(Memory, ENTRY_COUNT(1), kWithMaximum, 28, 28),
       SECTION(Data,
               ENTRY_COUNT(2),  // segment count
               LINEAR_MEMORY_INDEX_0,
@@ -1331,19 +1317,19 @@ TEST_F(WasmModuleVerifyTest, DataWithoutMemory) {
 TEST_F(WasmModuleVerifyTest, MaxMaximumMemorySize) {
   {
     const byte data[] = {
-        SECTION(Memory, ENTRY_COUNT(1), kHasMaximumFlag, 0, U32V_3(65536))};
+        SECTION(Memory, ENTRY_COUNT(1), kWithMaximum, 0, U32V_3(65536))};
     EXPECT_VERIFIES(data);
   }
   {
     const byte data[] = {
-        SECTION(Memory, ENTRY_COUNT(1), kHasMaximumFlag, 0, U32V_3(65537))};
+        SECTION(Memory, ENTRY_COUNT(1), kWithMaximum, 0, U32V_3(65537))};
     EXPECT_FAILURE(data);
   }
 }
 
 TEST_F(WasmModuleVerifyTest, DataSegment_wrong_init_type) {
   const byte data[] = {
-      SECTION(Memory, ENTRY_COUNT(1), kHasMaximumFlag, 28, 28),
+      SECTION(Memory, ENTRY_COUNT(1), kWithMaximum, 28, 28),
       SECTION(Data, ENTRY_COUNT(1), LINEAR_MEMORY_INDEX_0,
               WASM_INIT_EXPR_F64(9.9),  // dest addr
               U32V_1(3),                // source size
@@ -1356,7 +1342,7 @@ TEST_F(WasmModuleVerifyTest, DataSegment_wrong_init_type) {
 TEST_F(WasmModuleVerifyTest, DataSegmentEndOverflow) {
   const byte data[] = {
       SECTION(Memory,  // memory section
-              ENTRY_COUNT(1), kHasMaximumFlag, 28, 28),
+              ENTRY_COUNT(1), kWithMaximum, 28, 28),
       SECTION(Data,                      // data section
               ENTRY_COUNT(1),            // one entry
               LINEAR_MEMORY_INDEX_0,     // mem index