[test] Let flags from test case have precedence

The status-file flags and the flags from the test case's source code
must always overwrite extra flags set by bots.

Bug: v8:6924
Cq-Include-Trybots: master.tryserver.v8:v8_linux_noi18n_rel_ng
Change-Id: I0e2aabb69da7cfb8ba6c1c79bd3851462071a6ac
Reviewed-on: https://chromium-review.googlesource.com/732656
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#49001}

test/benchmarks/testcfg.py

@@ -117,27 +117,27 @@ class BenchmarksTestSuite(testsuite.TestSuite):
       tests.append(testcase.TestCase(self, test))
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
-    result = []
-    result += context.mode_flags
+  def GetParametersForTestCase(self, testcase, context):
+    files = []
     if testcase.path.startswith("kraken"):
-      result.append(os.path.join(self.testroot, "%s-data.js" % testcase.path))
-      result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
+      files.append(os.path.join(self.testroot, "%s-data.js" % testcase.path))
+      files.append(os.path.join(self.testroot, "%s.js" % testcase.path))
     elif testcase.path.startswith("octane"):
-      result.append(os.path.join(self.testroot, "octane/base.js"))
-      result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
+      files.append(os.path.join(self.testroot, "octane/base.js"))
+      files.append(os.path.join(self.testroot, "%s.js" % testcase.path))
       if testcase.path.startswith("octane/gbemu"):
-        result.append(os.path.join(self.testroot, "octane/gbemu-part2.js"))
+        files.append(os.path.join(self.testroot, "octane/gbemu-part2.js"))
       elif testcase.path.startswith("octane/typescript"):
-        result.append(os.path.join(self.testroot,
-                                   "octane/typescript-compiler.js"))
-        result.append(os.path.join(self.testroot, "octane/typescript-input.js"))
+        files.append(os.path.join(self.testroot,
+                                  "octane/typescript-compiler.js"))
+        files.append(os.path.join(self.testroot, "octane/typescript-input.js"))
       elif testcase.path.startswith("octane/zlib"):
-        result.append(os.path.join(self.testroot, "octane/zlib-data.js"))
-      result += ["-e", "BenchmarkSuite.RunSuites({});"]
+        files.append(os.path.join(self.testroot, "octane/zlib-data.js"))
+      files += ["-e", "BenchmarkSuite.RunSuites({});"]
     elif testcase.path.startswith("sunspider"):
-      result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
-    return testcase.flags + result
+      files.append(os.path.join(self.testroot, "%s.js" % testcase.path))
+
+    return files, testcase.flags + context.mode_flags
 
   def GetSourceForTest(self, testcase):
     filename = os.path.join(self.testroot, testcase.path + ".js")

test/cctest/testcfg.py

@@ -61,9 +61,8 @@ class CcTestSuite(testsuite.TestSuite):
     tests.sort(key=lambda t: t.path)
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
-    testname = testcase.path.split(os.path.sep)[-1]
-    return (testcase.flags + [testcase.path] + context.mode_flags)
+  def GetParametersForTestCase(self, testcase, context):
+    return [testcase.path], testcase.flags + context.mode_flags
 
   def shell(self):
     return "cctest"

test/debugger/testcfg.py

@@ -33,9 +33,13 @@ class DebuggerTestSuite(testsuite.TestSuite):
           tests.append(test)
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
+  def GetParametersForTestCase(self, testcase, context):
+    flags = (
+        testcase.flags +
+        ["--enable-inspector", "--allow-natives-syntax"] +
+        context.mode_flags
+    )
     source = self.GetSourceForTest(testcase)
-    flags = ["--enable-inspector", "--allow-natives-syntax"] + context.mode_flags
     flags_match = re.findall(FLAGS_PATTERN, source)
     for match in flags_match:
       flags += match.strip().split()
@@ -59,12 +63,13 @@ class DebuggerTestSuite(testsuite.TestSuite):
       files.append("--module")
     files.append(os.path.join(self.root, testcase.path + self.suffix()))
 
-    flags += files
+    all_files = []
+    all_files += files
     if context.isolates:
-      flags.append("--isolate")
-      flags += files
+      all_files.append("--isolate")
+      all_files += files
 
-    return testcase.flags + flags
+    return all_files, flags
 
   def GetSourceForTest(self, testcase):
     filename = os.path.join(self.root, testcase.path + self.suffix())

test/fuzzer/testcfg.py

@@ -40,9 +40,9 @@ class FuzzerTestSuite(testsuite.TestSuite):
     tests.sort()
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
+  def GetParametersForTestCase(self, testcase, context):
     suite, name = testcase.path.split('/')
-    return [os.path.join(self.root, suite, name)]
+    return [os.path.join(self.root, suite, name)], []
 
   def _VariantGeneratorFactory(self):
     return FuzzerVariantGenerator

test/inspector/testcfg.py

@@ -39,16 +39,17 @@ class InspectorProtocolTestSuite(testsuite.TestSuite):
           tests.append(test)
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
+  def GetParametersForTestCase(self, testcase, context):
     source = self.GetSourceForTest(testcase)
-    flags = [] + context.mode_flags
+    flags = testcase.flags + context.mode_flags
     flags_match = re.findall(FLAGS_PATTERN, source)
     for match in flags_match:
       flags += shlex.split(match.strip())
-    testname = testcase.path.split(os.path.sep)[-1]
-    testfilename = os.path.join(self.root, testcase.path + self.suffix())
-    protocoltestfilename = os.path.join(self.root, PROTOCOL_TEST_JS)
-    return testcase.flags + flags + [ protocoltestfilename, testfilename ]
+    files = [
+      os.path.join(self.root, PROTOCOL_TEST_JS),
+      os.path.join(self.root, testcase.path + self.suffix()),
+    ]
+    return files, flags
 
   def GetSourceForTest(self, testcase):
     filename = os.path.join(self.root, testcase.path + self.suffix())

test/intl/testcfg.py

@@ -56,9 +56,9 @@ class IntlTestSuite(testsuite.TestSuite):
           tests.append(test)
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
+  def GetParametersForTestCase(self, testcase, context):
     source = self.GetSourceForTest(testcase)
-    flags = ["--allow-natives-syntax"] + context.mode_flags
+    flags = testcase.flags + ["--allow-natives-syntax"] + context.mode_flags
     flags_match = re.findall(FLAGS_PATTERN, source)
     for match in flags_match:
       flags += match.strip().split()
@@ -70,12 +70,13 @@ class IntlTestSuite(testsuite.TestSuite):
     files.append(os.path.join(self.root, testcase.path + self.suffix()))
     files.append(os.path.join(self.root, "regexp-assert.js"))
 
-    flags += files
+    all_files = []
+    all_files += files
     if context.isolates:
-      flags.append("--isolate")
-      flags += files
+      all_files.append("--isolate")
+      all_files += files
 
-    return testcase.flags + flags
+    return all_files, flags
 
   def GetSourceForTest(self, testcase):
     filename = os.path.join(self.root, testcase.path + self.suffix())

test/message/testcfg.py

@@ -63,18 +63,18 @@ class MessageTestSuite(testsuite.TestSuite):
     return super(MessageTestSuite, self).CreateVariantGenerator(
         variants + ["preparser"])
 
-  def GetFlagsForTestCase(self, testcase, context):
+  def GetParametersForTestCase(self, testcase, context):
     source = self.GetSourceForTest(testcase)
-    result = []
+    files = []
+    if MODULE_PATTERN.search(source):
+      files.append("--module")
+    files.append(os.path.join(self.root, testcase.path + ".js"))
+    flags = testcase.flags + context.mode_flags
     flags_match = re.findall(FLAGS_PATTERN, source)
     for match in flags_match:
-      result += match.strip().split()
-    result += context.mode_flags
-    if MODULE_PATTERN.search(source):
-      result.append("--module")
-    result = [x for x in result if x not in INVALID_FLAGS]
-    result.append(os.path.join(self.root, testcase.path + ".js"))
-    return testcase.flags + result
+      flags += match.strip().split()
+    flags = [x for x in flags if x not in INVALID_FLAGS]
+    return files, flags
 
   def GetSourceForTest(self, testcase):
     filename = os.path.join(self.root, testcase.path + self.suffix())

test/mjsunit/testcfg.py

@@ -60,9 +60,9 @@ class MjsunitTestSuite(testsuite.TestSuite):
           tests.append(test)
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
+  def GetParametersForTestCase(self, testcase, context):
+    flags = testcase.flags + context.mode_flags
     source = self.GetSourceForTest(testcase)
-    flags = [] + context.mode_flags
     flags_match = re.findall(FLAGS_PATTERN, source)
     for match in flags_match:
       flags += match.strip().split()
@@ -90,10 +90,11 @@ class MjsunitTestSuite(testsuite.TestSuite):
       files.append("--module")
     files.append(testfilename)
 
-    flags += files
+    all_files = []
+    all_files += files
     if context.isolates:
-      flags.append("--isolate")
-      flags += files
+      all_files.append("--isolate")
+      all_files += files
 
     env_match = ENV_PATTERN.search(source)
     if env_match:
@@ -101,7 +102,7 @@ class MjsunitTestSuite(testsuite.TestSuite):
         var, value = env_pair.split('=')
         testcase.env[var] = value
 
-    return testcase.flags + flags
+    return all_files, flags
 
   def GetSourceForTest(self, testcase):
     filename = os.path.join(self.root, testcase.path + self.suffix())

test/mkgrokdump/testcfg.py

@@ -18,8 +18,8 @@ class MkGrokdump(testsuite.TestSuite):
     test = testcase.TestCase(self, self.shell())
     return [test]
 
-  def GetFlagsForTestCase(self, testcase, context):
-    return []
+  def GetParametersForTestCase(self, testcase, context):
+    return [], []
 
   def IsFailureOutput(self, testcase):
     output = testcase.output

test/mozilla/testcfg.py

@@ -81,11 +81,8 @@ class MozillaTestSuite(testsuite.TestSuite):
             tests.append(case)
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
-    result = []
-    result += context.mode_flags
-    result += ["--expose-gc"]
-    result += [os.path.join(self.root, "mozilla-shell-emulation.js")]
+  def GetParametersForTestCase(self, testcase, context):
+    files = [os.path.join(self.root, "mozilla-shell-emulation.js")]
     testfilename = testcase.path + ".js"
     testfilepath = testfilename.split("/")
     for i in xrange(len(testfilepath)):
@@ -93,9 +90,10 @@ class MozillaTestSuite(testsuite.TestSuite):
                             reduce(os.path.join, testfilepath[:i], ""),
                             "shell.js")
       if os.path.exists(script):
-        result.append(script)
-    result.append(os.path.join(self.testroot, testfilename))
-    return testcase.flags + result
+        files.append(script)
+    files.append(os.path.join(self.testroot, testfilename))
+    flags = testcase.flags + context.mode_flags + ["--expose-gc"]
+    return files, flags
 
   def GetSourceForTest(self, testcase):
     filename = os.path.join(self.testroot, testcase.path + ".js")

test/preparser/testcfg.py

@@ -71,8 +71,8 @@ class PreparserTestSuite(testsuite.TestSuite):
       self._ParsePythonTestTemplates(result, f)
     return result
 
-  def GetFlagsForTestCase(self, testcase, context):
-    return testcase.flags
+  def GetParametersForTestCase(self, testcase, context):
+    return [], testcase.flags
 
   def GetSourceForTest(self, testcase):
     assert testcase.flags[0] == "-e"

test/promises-aplus/testcfg.py

@@ -71,11 +71,14 @@ class PromiseAplusTestSuite(testsuite.TestSuite):
             os.listdir(os.path.join(self.root, TEST_NAME, 'lib', 'tests'))
             if fname.endswith('.js')]
 
-  def GetFlagsForTestCase(self, testcase, context):
-    return (testcase.flags + context.mode_flags + ['--allow-natives-syntax'] +
-            self.helper_files_pre +
-            [os.path.join(self.test_files_root, testcase.path + '.js')] +
-            self.helper_files_post)
+  def GetParametersForTestCase(self, testcase, context):
+    files = (
+        self.helper_files_pre +
+        [os.path.join(self.test_files_root, testcase.path + '.js')] +
+        self.helper_files_post
+    )
+    flags = testcase.flags + context.mode_flags + ['--allow-natives-syntax']
+    return files, flags
 
   def GetSourceForTest(self, testcase):
     filename = os.path.join(self.root, TEST_NAME,

test/test262/testcfg.py

@@ -153,22 +153,28 @@ class Test262TestSuite(testsuite.TestSuite):
                 SKIPPED_FEATURES.intersection(
                     self.GetTestRecord(case).get("features", []))) == 0]
 
-  def GetFlagsForTestCase(self, testcase, context):
-    return (testcase.flags + context.mode_flags + self.harness +
-            ([os.path.join(self.root, "harness-agent.js")]
-             if testcase.path.startswith('built-ins/Atomics') else []) +
-            self.GetIncludesForTest(testcase) +
-            (["--module"] if "module" in self.GetTestRecord(testcase) else []) +
-            [self.GetPathForTest(testcase)] +
-            (["--throws"] if "negative" in self.GetTestRecord(testcase)
-                          else []) +
-            (["--allow-natives-syntax"]
-             if "detachArrayBuffer.js" in
-                self.GetTestRecord(testcase).get("includes", [])
-             else []) +
-            ([flag for flag in testcase.outcomes if flag.startswith("--")]) +
-            ([flag for (feature, flag) in FEATURE_FLAGS.items()
-              if feature in self.GetTestRecord(testcase).get("features", [])]))
+  def GetParametersForTestCase(self, testcase, context):
+    files = (
+        list(self.harness) +
+        ([os.path.join(self.root, "harness-agent.js")]
+         if testcase.path.startswith('built-ins/Atomics') else []) +
+        self.GetIncludesForTest(testcase) +
+        (["--module"] if "module" in self.GetTestRecord(testcase) else []) +
+        [self.GetPathForTest(testcase)]
+    )
+    flags = (
+        testcase.flags + context.mode_flags +
+        (["--throws"] if "negative" in self.GetTestRecord(testcase)
+                      else []) +
+        (["--allow-natives-syntax"]
+         if "detachArrayBuffer.js" in
+            self.GetTestRecord(testcase).get("includes", [])
+         else []) +
+        ([flag for flag in testcase.outcomes if flag.startswith("--")]) +
+        ([flag for (feature, flag) in FEATURE_FLAGS.items()
+          if feature in self.GetTestRecord(testcase).get("features", [])])
+    )
+    return files, flags
 
   def _VariantGeneratorFactory(self):
     return Test262VariantGenerator
@@ -201,12 +207,8 @@ class Test262TestSuite(testsuite.TestSuite):
 
   def GetIncludesForTest(self, testcase):
     test_record = self.GetTestRecord(testcase)
-    if "includes" in test_record:
-      return [os.path.join(self.BasePath(filename), filename)
-              for filename in test_record.get("includes", [])]
-    else:
-      includes = []
-    return includes
+    return [os.path.join(self.BasePath(filename), filename)
+            for filename in test_record.get("includes", [])]
 
   def GetPathForTest(self, testcase):
     filename = os.path.join(self.localtestroot, testcase.path + ".js")

test/wasm-spec-tests/testcfg.py

@@ -25,10 +25,10 @@ class WasmSpecTestsTestSuite(testsuite.TestSuite):
           tests.append(test)
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
-    flags = [] + context.mode_flags
-    flags.append(os.path.join(self.root, testcase.path + self.suffix()))
-    return testcase.flags + flags
+  def GetParametersForTestCase(self, testcase, context):
+    flags = testcase.flags + context.mode_flags
+    files = [os.path.join(self.root, testcase.path + self.suffix())]
+    return files, flags
 
 
 def GetSuite(name, root):

test/webkit/testcfg.py

@@ -62,9 +62,9 @@ class WebkitTestSuite(testsuite.TestSuite):
           tests.append(test)
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
+  def GetParametersForTestCase(self, testcase, context):
     source = self.GetSourceForTest(testcase)
-    flags = [] + context.mode_flags
+    flags = testcase.flags + context.mode_flags
     flags_match = re.findall(FLAGS_PATTERN, source)
     for match in flags_match:
       flags += match.strip().split()
@@ -88,12 +88,13 @@ class WebkitTestSuite(testsuite.TestSuite):
     files.append(testfilename)
     files.append(os.path.join(self.root, "resources/standalone-post.js"))
 
-    flags += files
+    all_files = []
+    all_files += files
     if context.isolates:
-      flags.append("--isolate")
-      flags += files
+      all_files.append("--isolate")
+      all_files += files
 
-    return testcase.flags + flags
+    return all_files, flags
 
   def GetSourceForTest(self, testcase):
     filename = os.path.join(self.root, testcase.path + self.suffix())

tools/testrunner/local/execution.py

@@ -92,11 +92,16 @@ def GetCommand(test, context):
     shell += ".exe"
   if context.random_seed:
     d8testflag += ["--random-seed=%s" % context.random_seed]
-  cmd = (context.command_prefix +
-         [os.path.abspath(os.path.join(context.shell_dir, shell))] +
-         d8testflag +
-         test.suite.GetFlagsForTestCase(test, context) +
-         context.extra_flags)
+  files, flags = test.suite.GetParametersForTestCase(test, context)
+  cmd = (
+      context.command_prefix +
+      [os.path.abspath(os.path.join(context.shell_dir, shell))] +
+      d8testflag +
+      files +
+      context.extra_flags +
+      # Flags from test cases can overwrite extra cmd-line flags.
+      flags
+  )
   return cmd
 
 

tools/testrunner/local/testsuite.py

@@ -257,7 +257,8 @@ class TestSuite(object):
           break
     self.tests = filtered
 
-  def GetFlagsForTestCase(self, testcase, context):
+  def GetParametersForTestCase(self, testcase, context):
+    """Returns a tuple of (files, flags) for this test case."""
     raise NotImplementedError
 
   def GetSourceForTest(self, testcase):
@@ -348,11 +349,11 @@ class GoogleTestSuite(TestSuite):
     tests.sort(key=lambda t: t.path)
     return tests
 
-  def GetFlagsForTestCase(self, testcase, context):
-    return (testcase.flags + ["--gtest_filter=" + testcase.path] +
-            ["--gtest_random_seed=%s" % context.random_seed] +
-            ["--gtest_print_time=0"] +
-            context.mode_flags)
+  def GetParametersForTestCase(self, testcase, context):
+    return [], (testcase.flags + ["--gtest_filter=" + testcase.path] +
+                ["--gtest_random_seed=%s" % context.random_seed] +
+                ["--gtest_print_time=0"] +
+                context.mode_flags)
 
   def _VariantGeneratorFactory(self):
     return StandardVariantGenerator