[test] Share resource-fetching logic with all d8 test cases

This shares logic for finding additional resources in JS source code.
Previously the logic was implemented for mjsunit, now it will be used
across all d8-based test cases.

This'll enable adding those test suites for Android testing.

Bug: chromium:866862
Cq-Include-Trybots: luci.v8.try:v8_linux_noi18n_rel_ng
Change-Id: I7c89ba141483aaf692a03c0e168edb61bbb7b010
Reviewed-on: https://chromium-review.googlesource.com/1193873
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Georg Neis <neis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#55503}

test/benchmarks/testcfg.py

@@ -105,7 +105,7 @@ class TestSuite(testsuite.TestSuite):
     return TestCase
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def _get_files_params(self):
     path = self.path
     testroot = self.suite.testroot

test/debugger/testcfg.py

@@ -32,7 +32,7 @@ class TestSuite(testsuite.TestSuite):
     return TestCase
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 

test/intl/testcfg.py

@@ -56,7 +56,7 @@ class TestSuite(testsuite.TestSuite):
     return TestCase
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 

test/message/testcfg.py

@@ -59,7 +59,7 @@ class TestSuite(testsuite.TestSuite):
     return TestCase
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 

test/mjsunit/testcfg.py

@@ -41,21 +41,6 @@ SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
 MODULE_PATTERN = re.compile(r"^// MODULE$", flags=re.MULTILINE)
 NO_HARNESS_PATTERN = re.compile(r"^// NO HARNESS$", flags=re.MULTILINE)
 
-# Patterns for additional resource files on Android. Files that are not covered
-# by one of the other patterns below will be specified in the resources section.
-RESOURCES_PATTERN = re.compile(r"//\s+Resources:(.*)")
-# Pattern to auto-detect files to push on Android for statements like:
-# load("path/to/file.js")
-LOAD_PATTERN = re.compile(
-    r"(?:load|readbuffer|read)\((?:'|\")([^'\"]*)(?:'|\")\)")
-# Pattern to auto-detect files to push on Android for statements like:
-# import "path/to/file.js"
-MODULE_RESOURCES_PATTERN_1 = re.compile(
-    r"(?:import|export)(?:\(| )(?:'|\")([^'\"]*)(?:'|\")")
-# Pattern to auto-detect files to push on Android for statements like:
-# import foobar from "path/to/file.js"
-MODULE_RESOURCES_PATTERN_2 = re.compile(
-    r"(?:import|export).*from (?:'|\")([^'\"]*)(?:'|\")")
 
 # Flags known to misbehave when combining arbitrary mjsunit tests. Can also
 # be compiled regular expressions.
@@ -100,7 +85,7 @@ class TestSuite(testsuite.TestSuite):
     return SuppressedTestCase
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 
@@ -140,47 +125,6 @@ class TestCase(testcase.TestCase):
     self._files_suffix = files_suffix
     self._env = self._parse_source_env(source)
 
-  def _get_resources_for_file(self, file):
-    """Returns for a given file a list of absolute paths of files needed by the
-    given file.
-    """
-    with open(file) as f:
-      source = f.read()
-    result = []
-    def add_path(path):
-      result.append(os.path.abspath(path.replace('/', os.path.sep)))
-    for match in RESOURCES_PATTERN.finditer(source):
-      # There are several resources per line. Relative to base dir.
-      for path in match.group(1).strip().split():
-        add_path(path)
-    for match in LOAD_PATTERN.finditer(source):
-      # Files in load statements are relative to base dir.
-      add_path(match.group(1))
-    for match in MODULE_RESOURCES_PATTERN_1.finditer(source):
-      # Imported files are side by side with the test case.
-      add_path(os.path.join(
-          self.suite.root, os.path.dirname(self.path), match.group(1)))
-    for match in MODULE_RESOURCES_PATTERN_2.finditer(source):
-      # Imported files are side by side with the test case.
-      add_path(os.path.join(
-          self.suite.root, os.path.dirname(self.path), match.group(1)))
-    return result
-
-  def _get_resources(self):
-    """Returns the list of files needed by a test case."""
-    result = set()
-    to_check = [self._get_source_path()]
-    # Recurse over all files until reaching a fixpoint.
-    while to_check:
-      next_resource = to_check.pop()
-      result.add(next_resource)
-      for resource in self._get_resources_for_file(next_resource):
-        # Only add files that exist on disc. The pattens we check for give some
-        # false positives otherwise.
-        if resource not in result and os.path.exists(resource):
-          to_check.append(resource)
-    return sorted(list(result))
-
   def _parse_source_env(self, source):
     env_match = ENV_PATTERN.search(source)
     env = {}
@@ -244,7 +188,7 @@ class TestCombiner(testsuite.TestCombiner):
     return CombinedTest
 
 
-class CombinedTest(testcase.TestCase):
+class CombinedTest(testcase.D8TestCase):
   """Behaves like normal mjsunit tests except:
     1. Expected outcome is always PASS
     2. Instead of one file there is a try-catch wrapper with all combined tests
@@ -259,7 +203,7 @@ class CombinedTest(testcase.TestCase):
     self._statusfile_outcomes = outproc.OUTCOMES_PASS_OR_TIMEOUT
     self.expected_outcomes = outproc.OUTCOMES_PASS_OR_TIMEOUT
 
-  def _get_shell_with_flags(self):
+  def _get_shell_flags(self):
     """In addition to standard set of shell flags it appends:
       --disable-abortjs: %AbortJS can abort the test even inside
         trycatch-wrapper, so we disable it.
@@ -269,15 +213,13 @@ class CombinedTest(testcase.TestCase):
       --quiet-load: suppress any stdout from load() function used by
         trycatch-wrapper.
     """
-    shell = 'd8'
-    shell_flags = [
+    return [
       '--test',
       '--disable-abortjs',
       '--es-staging',
       '--omit-quit',
       '--quiet-load',
     ]
-    return shell, shell_flags
 
   def _get_cmd_params(self):
     return (

test/mozilla/testcfg.py

@@ -85,7 +85,7 @@ class TestSuite(testsuite.TestSuite):
     return TestCase
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def _get_files_params(self):
     files = [os.path.join(self.suite.root, "mozilla-shell-emulation.js")]
     testfilename = self.path + ".js"

test/preparser/testcfg.py

@@ -81,7 +81,7 @@ class TestSuite(testsuite.TestSuite):
     return VariantsGenerator
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def __init__(self, suite, path, name, test_config, source, template_flags):
     super(TestCase, self).__init__(suite, path, name, test_config)
 

test/test262/testcfg.py

@@ -154,7 +154,7 @@ class TestSuite(testsuite.TestSuite):
     return VariantsGenerator
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 

test/wasm-spec-tests/testcfg.py

@@ -26,7 +26,7 @@ class TestSuite(testsuite.TestSuite):
     return TestCase
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def _get_files_params(self):
     return [os.path.join(self.suite.root, self.path + self._get_suffix())]
 

test/webkit/testcfg.py

@@ -61,7 +61,7 @@ class TestSuite(testsuite.TestSuite):
     return TestCase
 
 
-class TestCase(testcase.TestCase):
+class TestCase(testcase.D8TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 

tools/testrunner/objects/testcase.py

@@ -37,6 +37,21 @@ from ..local import utils
 
 FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
 
+# Patterns for additional resource files on Android. Files that are not covered
+# by one of the other patterns below will be specified in the resources section.
+RESOURCES_PATTERN = re.compile(r"//\s+Resources:(.*)")
+# Pattern to auto-detect files to push on Android for statements like:
+# load("path/to/file.js")
+LOAD_PATTERN = re.compile(
+    r"(?:load|readbuffer|read)\((?:'|\")([^'\"]*)(?:'|\")\)")
+# Pattern to auto-detect files to push on Android for statements like:
+# import "path/to/file.js"
+MODULE_RESOURCES_PATTERN_1 = re.compile(
+    r"(?:import|export)(?:\(| )(?:'|\")([^'\"]*)(?:'|\")")
+# Pattern to auto-detect files to push on Android for statements like:
+# import foobar from "path/to/file.js"
+MODULE_RESOURCES_PATTERN_2 = re.compile(
+    r"(?:import|export).*from (?:'|\")([^'\"]*)(?:'|\")")
 
 
 class TestCase(object):
@@ -143,7 +158,10 @@ class TestCase(object):
   def get_command(self):
     params = self._get_cmd_params()
     env = self._get_cmd_env()
-    shell, shell_flags = self._get_shell_with_flags()
+    shell = self.get_shell()
+    if utils.IsWindows():
+      shell += '.exe'
+    shell_flags = self._get_shell_flags()
     timeout = self._get_timeout(params)
     return self._create_cmd(shell, shell_flags + params, env, timeout)
 
@@ -207,14 +225,8 @@ class TestCase(object):
   def _get_suite_flags(self):
     return []
 
-  def _get_shell_with_flags(self):
-    shell = self.get_shell()
-    shell_flags = []
-    if shell == 'd8':
-      shell_flags.append('--test')
-    if utils.IsWindows():
-      shell += '.exe'
-    return shell, shell_flags
+  def _get_shell_flags(self):
+    return []
 
   def _get_timeout(self, params):
     timeout = self._test_config.timeout
@@ -228,7 +240,7 @@ class TestCase(object):
     return timeout
 
   def get_shell(self):
-    return 'd8'
+    raise NotImplementedError()
 
   def _get_suffix(self):
     return '.js'
@@ -285,3 +297,52 @@ class TestCase(object):
 
   def __str__(self):
     return self.suite.name + '/' + self.name
+
+
+class D8TestCase(TestCase):
+  def get_shell(self):
+    return "d8"
+
+  def _get_shell_flags(self):
+    return ['--test']
+
+  def _get_resources_for_file(self, file):
+    """Returns for a given file a list of absolute paths of files needed by the
+    given file.
+    """
+    with open(file) as f:
+      source = f.read()
+    result = []
+    def add_path(path):
+      result.append(os.path.abspath(path.replace('/', os.path.sep)))
+    for match in RESOURCES_PATTERN.finditer(source):
+      # There are several resources per line. Relative to base dir.
+      for path in match.group(1).strip().split():
+        add_path(path)
+    for match in LOAD_PATTERN.finditer(source):
+      # Files in load statements are relative to base dir.
+      add_path(match.group(1))
+    for match in MODULE_RESOURCES_PATTERN_1.finditer(source):
+      # Imported files are side by side with the test case.
+      add_path(os.path.join(
+          self.suite.root, os.path.dirname(self.path), match.group(1)))
+    for match in MODULE_RESOURCES_PATTERN_2.finditer(source):
+      # Imported files are side by side with the test case.
+      add_path(os.path.join(
+          self.suite.root, os.path.dirname(self.path), match.group(1)))
+    return result
+
+  def _get_resources(self):
+    """Returns the list of files needed by a test case."""
+    result = set()
+    to_check = [self._get_source_path()]
+    # Recurse over all files until reaching a fixpoint.
+    while to_check:
+      next_resource = to_check.pop()
+      result.add(next_resource)
+      for resource in self._get_resources_for_file(next_resource):
+        # Only add files that exist on disc. The pattens we check for give some
+        # false positives otherwise.
+        if resource not in result and os.path.exists(resource):
+          to_check.append(resource)
+    return sorted(list(result))