Make regress-crbug-514081 less flaky by having max serialization size

BUG=v8:5906
R=machenbach@chromium.org

Review-Url: https://codereview.chromium.org/2697723004
Cr-Commit-Position: refs/heads/master@{#43292}

src/d8.cc

@@ -66,6 +66,7 @@ namespace {
 
 const int MB = 1024 * 1024;
 const int kMaxWorkers = 50;
+const int kMaxSerializerMemoryUsage = 1 * MB;  // Arbitrary maximum for testing.
 
 #define USE_VM 1
 #define VM_THRESHOLD 65536
@@ -2567,7 +2568,9 @@ void Shell::EmptyMessageQueues(Isolate* isolate) {
 class Serializer : public ValueSerializer::Delegate {
  public:
   explicit Serializer(Isolate* isolate)
-      : isolate_(isolate), serializer_(isolate, this) {}
+      : isolate_(isolate),
+        serializer_(isolate, this),
+        current_memory_usage_(0) {}
 
   Maybe<bool> WriteValue(Local<Context> context, Local<Value> value,
                          Local<Value> transfer) {
@@ -2618,6 +2621,11 @@ class Serializer : public ValueSerializer::Delegate {
 
   void* ReallocateBufferMemory(void* old_buffer, size_t size,
                                size_t* actual_size) override {
+    // Not accurate, because we don't take into account reallocated buffers,
+    // but this is fine for testing.
+    current_memory_usage_ += size;
+    if (current_memory_usage_ > kMaxSerializerMemoryUsage) return nullptr;
+
     void* result = realloc(old_buffer, size);
     *actual_size = result ? size : 0;
     return result;
@@ -2695,6 +2703,7 @@ class Serializer : public ValueSerializer::Delegate {
   std::unique_ptr<SerializationData> data_;
   std::vector<Global<ArrayBuffer>> array_buffers_;
   std::vector<Global<SharedArrayBuffer>> shared_array_buffers_;
+  size_t current_memory_usage_;
 
   DISALLOW_COPY_AND_ASSIGN(Serializer);
 };

test/mjsunit/mjsunit.status

@@ -172,9 +172,6 @@
 
   # BUG(v8:5807): Flaky data race.
   'wasm/embenchen/fannkuch': [PASS, ['tsan', SKIP]],
-
-  # BUG(v8:5906).
-  'regress/regress-crbug-514081': [PASS, ['system == linux', SKIP]],
 }],  # ALWAYS
 
 ['novfp3 == True', {

test/mjsunit/regress/regress-crbug-514081.js

@@ -5,16 +5,14 @@
 if (this.Worker) {
   var __v_7 = new Worker('onmessage = function() {};');
   var e;
+  var ab = new ArrayBuffer(2 * 1000 * 1000);
   try {
-    var ab = new ArrayBuffer(2147483648);
-    try {
-      __v_7.postMessage(ab);
-    } catch (e) {
-      // postMessage failed, should be a DataCloneError message.
-      assertContains('cloned', e.message);
-    }
+    __v_7.postMessage(ab);
+    threw = false;
   } catch (e) {
-    // Creating the ArrayBuffer failed.
-    assertInstanceof(e, RangeError);
+    // postMessage failed, should be a DataCloneError message.
+    assertContains('cloned', e.message);
+    threw = true;
   }
+  assertTrue(threw, 'Should throw when trying to serialize large message.');
 }