[asm.js] Fail gracefully on overly large buffers.

This makes sure instantiate of asm.js modules fails gracefully on heap
buffers exceeding the uint32_t range supported by WebAssembly.

R=clemensh@chromium.org
TEST=mjsunit/regress/regress-crbug-754175
BUG=chromium:754175

Change-Id: I4a9c6791beaab6da826b5b6b5a495f97e9d3b4e9
Reviewed-on: https://chromium-review.googlesource.com/632618Reviewed-by: Clemens Hammacher <clemensh@chromium.org>
Commit-Queue: Michael Starzinger <mstarzinger@chromium.org>
Cr-Commit-Position: refs/heads/master@{#47598}

src/asmjs/asm-js.cc

@@ -364,6 +364,13 @@ MaybeHandle<Object> AsmJs::InstantiateAsmWasm(Isolate* isolate,
       ReportInstantiationFailure(script, position, "Unexpected heap size");
       return MaybeHandle<Object>();
     }
+    // Currently WebAssembly only supports heap sizes within the uint32_t range.
+    if (size > std::numeric_limits<uint32_t>::max()) {
+      ReportInstantiationFailure(script, position, "Unexpected heap size");
+      return MaybeHandle<Object>();
+    }
+  } else {
+    memory = Handle<JSArrayBuffer>::null();
   }
 
   wasm::ErrorThrower thrower(isolate, "AsmJs::Instantiate");

test/mjsunit/regress/regress-crbug-754175.js

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --mock-arraybuffer-allocator
+
+function Module(stdlib, foreign, buffer) {
+  "use asm";
+  var heap = new stdlib.Int8Array(buffer);
+  function foo() { return heap[23] | 0 }
+  return { foo:foo };
+}
+function instantiate() {
+  // On 32-bit architectures buffer allocation will throw.
+  var buffer = new ArrayBuffer(0x100000000);
+  // On 64-bit architectures instantiation will throw.
+  var module = Module(this, {}, buffer);
+}
+assertThrows(instantiate, RangeError);