[ubsan][regexp] Avoid out-of-range casts from int to enum Result

NativeRegExpMacroAssembler::Match() can return either a Result sentinel
or an int indicating the number of matches, so it should return a plain
int which we can only safely cast to Result or IrregexpResult when it's
guaranteed to be the former case.

Bug: v8:3770
Change-Id: I4c3447e0cdebd5f825964e086574ab504a1799cd
Reviewed-on: https://chromium-review.googlesource.com/c/1435735Reviewed-by: Yang Guo <yangguo@chromium.org>
Commit-Queue: Jakob Kummerow <jkummerow@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59142}

src/regexp/jsregexp.cc

@@ -444,9 +444,8 @@ int RegExpImpl::IrregexpExecRaw(Isolate* isolate, Handle<JSRegExp> regexp,
       // This means that in case of failure, the output registers array is left
       // untouched and contains the capture results from the previous successful
       // match.  We can use that to set the last match info lazily.
-      NativeRegExpMacroAssembler::Result res =
-          NativeRegExpMacroAssembler::Match(code, subject, output, output_size,
-                                            index, isolate);
+      int res = NativeRegExpMacroAssembler::Match(code, subject, output,
+                                                  output_size, index, isolate);
       if (res != NativeRegExpMacroAssembler::RETRY) {
         DCHECK(res != NativeRegExpMacroAssembler::EXCEPTION ||
                isolate->has_pending_exception());
@@ -456,7 +455,7 @@ int RegExpImpl::IrregexpExecRaw(Isolate* isolate, Handle<JSRegExp> regexp,
                       RE_FAILURE);
         STATIC_ASSERT(static_cast<int>(NativeRegExpMacroAssembler::EXCEPTION) ==
                       RE_EXCEPTION);
-        return static_cast<IrregexpResult>(res);
+        return res;
       }
       // If result is RETRY, the string has changed representation, and we
       // must restart from scratch.

src/regexp/regexp-macro-assembler.cc

@@ -207,14 +207,12 @@ int NativeRegExpMacroAssembler::CheckStackGuardState(
   return return_value;
 }
 
-NativeRegExpMacroAssembler::Result NativeRegExpMacroAssembler::Match(
-    Handle<Code> regexp_code,
-    Handle<String> subject,
-    int* offsets_vector,
-    int offsets_vector_length,
-    int previous_index,
-    Isolate* isolate) {
-
+// Returns a {Result} sentinel, or the number of successful matches.
+int NativeRegExpMacroAssembler::Match(Handle<Code> regexp_code,
+                                      Handle<String> subject,
+                                      int* offsets_vector,
+                                      int offsets_vector_length,
+                                      int previous_index, Isolate* isolate) {
   DCHECK(subject->IsFlat());
   DCHECK_LE(0, previous_index);
   DCHECK_LE(previous_index, subject->length());
@@ -253,18 +251,12 @@ NativeRegExpMacroAssembler::Result NativeRegExpMacroAssembler::Match(
       StringCharacterPosition(subject_ptr, start_offset + slice_offset, no_gc);
   int byte_length = char_length << char_size_shift;
   const byte* input_end = input_start + byte_length;
-  Result res = Execute(*regexp_code,
-                       *subject,
-                       start_offset,
-                       input_start,
-                       input_end,
-                       offsets_vector,
-                       offsets_vector_length,
-                       isolate);
-  return res;
+  return Execute(*regexp_code, *subject, start_offset, input_start, input_end,
+                 offsets_vector, offsets_vector_length, isolate);
 }
 
-NativeRegExpMacroAssembler::Result NativeRegExpMacroAssembler::Execute(
+// Returns a {Result} sentinel, or the number of successful matches.
+int NativeRegExpMacroAssembler::Execute(
     Code code,
     String input,  // This needs to be the unpacked (sliced, cons) string.
     int start_offset, const byte* input_start, const byte* input_end,
@@ -293,7 +285,7 @@ NativeRegExpMacroAssembler::Result NativeRegExpMacroAssembler::Execute(
     AllowHeapAllocation allow_allocation;
     isolate->StackOverflow();
   }
-  return static_cast<Result>(result);
+  return result;
 }
 
 // clang-format off

src/regexp/regexp-macro-assembler.h

@@ -212,12 +212,10 @@ class NativeRegExpMacroAssembler: public RegExpMacroAssembler {
   ~NativeRegExpMacroAssembler() override;
   bool CanReadUnaligned() override;
 
-  static Result Match(Handle<Code> regexp,
-                      Handle<String> subject,
-                      int* offsets_vector,
-                      int offsets_vector_length,
-                      int previous_index,
-                      Isolate* isolate);
+  // Returns a {Result} sentinel, or the number of successful matches.
+  static int Match(Handle<Code> regexp, Handle<String> subject,
+                   int* offsets_vector, int offsets_vector_length,
+                   int previous_index, Isolate* isolate);
 
   // Called from RegExp if the backtrack stack limit is hit.
   // Tries to expand the stack. Returns the new stack-pointer if
@@ -245,9 +243,10 @@ class NativeRegExpMacroAssembler: public RegExpMacroAssembler {
     return reinterpret_cast<Address>(&word_character_map[0]);
   }
 
-  static Result Execute(Code code, String input, int start_offset,
-                        const byte* input_start, const byte* input_end,
-                        int* output, int output_size, Isolate* isolate);
+  // Returns a {Result} sentinel, or the number of successful matches.
+  static int Execute(Code code, String input, int start_offset,
+                     const byte* input_start, const byte* input_end,
+                     int* output, int output_size, Isolate* isolate);
 };
 
 }  // namespace internal

test/cctest/test-regexp.cc

@@ -771,9 +771,11 @@ static ArchRegExpMacroAssembler::Result Execute(Code code, String input,
                                                 Address input_start,
                                                 Address input_end,
                                                 int* captures) {
-  return NativeRegExpMacroAssembler::Execute(
-      code, input, start_offset, reinterpret_cast<byte*>(input_start),
-      reinterpret_cast<byte*>(input_end), captures, 0, CcTest::i_isolate());
+  return static_cast<NativeRegExpMacroAssembler::Result>(
+      NativeRegExpMacroAssembler::Execute(code, input, start_offset,
+                                          reinterpret_cast<byte*>(input_start),
+                                          reinterpret_cast<byte*>(input_end),
+                                          captures, 0, CcTest::i_isolate()));
 }
 
 TEST(MacroAssemblerNativeSuccess) {