[regexp] Periodically check for stack overflow during node generation

Recursive ToNode node generation may overflow the stack for large
graphs. As a quick fix, insert periodic stack overflow checks in
selected ToNode methods.

As a more permanent fix, in the future we could abort gracefully
(instead of crashing on a CHECK), and/or refactor into iterative node
generation.

Bug: v8:12472
Change-Id: Ie5fbe838c5f6a5192d7d9b44bfe6f6c76a8d26e7
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3398112Reviewed-by: Leszek Swirski <leszeks@chromium.org>
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/main@{#78667}

src/regexp/regexp-compiler-tonode.cc

@@ -778,6 +778,8 @@ void RegExpDisjunction::FixSingleCharacterDisjunctions(
 
 RegExpNode* RegExpDisjunction::ToNode(RegExpCompiler* compiler,
                                       RegExpNode* on_success) {
+  compiler->ToNodeMaybeCheckForStackOverflow();
+
   ZoneList<RegExpTree*>* alternatives = this->alternatives();
 
   if (alternatives->length() > 2) {
@@ -1089,6 +1091,8 @@ class AssertionSequenceRewriter final {
 
 RegExpNode* RegExpAlternative::ToNode(RegExpCompiler* compiler,
                                       RegExpNode* on_success) {
+  compiler->ToNodeMaybeCheckForStackOverflow();
+
   ZoneList<RegExpTree*>* children = nodes();
 
   AssertionSequenceRewriter::MaybeRewrite(children, compiler->zone());

src/regexp/regexp-compiler.cc

@@ -3950,5 +3950,9 @@ RegExpNode* RegExpCompiler::PreprocessRegExp(RegExpCompileData* data,
   return node;
 }
 
+void RegExpCompiler::ToNodeCheckForStackOverflow() {
+  CHECK(!StackLimitCheck{isolate()}.HasOverflowed());
+}
+
 }  // namespace internal
 }  // namespace v8

src/regexp/regexp-compiler.h

@@ -550,6 +550,18 @@ class RegExpCompiler {
     current_expansion_factor_ = value;
   }
 
+  // The recursive nature of ToNode node generation means we may run into stack
+  // overflow issues. We introduce periodic checks to detect these, and the
+  // tick counter helps limit overhead of these checks.
+  // TODO(jgruber): This is super hacky and should be replaced by an abort
+  // mechanism or iterative node generation.
+  void ToNodeMaybeCheckForStackOverflow() {
+    if ((to_node_overflow_check_ticks_++ % 16 == 0)) {
+      ToNodeCheckForStackOverflow();
+    }
+  }
+  void ToNodeCheckForStackOverflow();
+
   Isolate* isolate() const { return isolate_; }
   Zone* zone() const { return zone_; }
 
@@ -567,6 +579,7 @@ class RegExpCompiler {
   bool one_byte_;
   bool reg_exp_too_big_;
   bool limiting_recursion_;
+  int to_node_overflow_check_ticks_ = 0;
   bool optimize_;
   bool read_backward_;
   int current_expansion_factor_;

test/mjsunit/mjsunit.status

@@ -180,6 +180,9 @@
   # https://crbug.com/v8/10948
   'wasm/atomics': [PASS, ['arch == arm and not simulator_run', SKIP]],
 
+  # crbug.com/v8/12472 Stack overflow during regexp node generation.
+  'regress/regress-crbug-595657': [SKIP],
+
   ##############################################################################
   # Tests where variants make no sense.
   'd8/enable-tracing': [PASS, NO_VARIANTS],