[rab/gsab] Fix memory size computations close to size_t limit

Bug: v8:11111,v8:1321980
Change-Id: I4dead5d50a2e1a9c1011c16d13aad2722598e456
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3642297Reviewed-by: Shu-yu Guo <syg@chromium.org>
Commit-Queue: Marja Hölttä <marja@chromium.org>
Reviewed-by: Clemens Backes <clemensb@chromium.org>
Cr-Commit-Position: refs/heads/main@{#80541}

src/base/macros.h

@@ -338,11 +338,16 @@ constexpr inline T RoundDown(T x) {
 template <typename T>
 inline T RoundUp(T x, intptr_t m) {
   static_assert(std::is_integral<T>::value);
-  return RoundDown<T>(static_cast<T>(x + m - 1), m);
+  DCHECK_GE(x, 0);
+  DCHECK_GE(std::numeric_limits<T>::max() - x, m - 1);  // Overflow check.
+  return RoundDown<T>(static_cast<T>(x + (m - 1)), m);
 }
+
 template <intptr_t m, typename T>
 constexpr inline T RoundUp(T x) {
   static_assert(std::is_integral<T>::value);
+  DCHECK_GE(x, 0);
+  DCHECK_GE(std::numeric_limits<T>::max() - x, m - 1);  // Overflow check.
   return RoundDown<m, T>(static_cast<T>(x + (m - 1)));
 }
 

src/utils/utils.h

@@ -661,6 +661,11 @@ V8_INLINE void ZapCode(Address addr, size_t size_in_bytes) {
 
 inline bool RoundUpToPageSize(size_t byte_length, size_t page_size,
                               size_t max_allowed_byte_length, size_t* pages) {
+  // This check is needed, since the arithmetic in RoundUp only works when
+  // byte_length is not too close to the size_t limit.
+  if (byte_length > max_allowed_byte_length) {
+    return false;
+  }
   size_t bytes_wanted = RoundUp(byte_length, page_size);
   if (bytes_wanted > max_allowed_byte_length) {
     return false;

test/mjsunit/regress-crbug-1321980.js

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --harmony-rab-gsab
+
+try {
+  // We're only testing these don't crash. It's platform-dependent which of them throw.
+  new ArrayBuffer(1, {maxByteLength: 2147483647});
+  new ArrayBuffer(1, {maxByteLength: 9007199254740000});
+} catch (e) {
+
+}