Refactor the tools/test.py script and related testcfg.py files.

This makes it possible to run several variants of the tests (with different flags that is) by adding extra lists to the VARIANT_FLAGS list. In addition, there is a number of smaller refactorings. 


Review URL: http://codereview.chromium.org/3164023

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@5329 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

test/cctest/testcfg.py

@@ -31,15 +31,12 @@ from os.path import join, dirname, exists
 import platform
 import utils
 
-CCTEST_DEBUG_FLAGS = ['--enable-slow-asserts', '--debug-code', '--verify-heap']
-
 
 class CcTestCase(test.TestCase):
 
   def __init__(self, path, executable, mode, raw_name, dependency, context):
-    super(CcTestCase, self).__init__(context, path)
+    super(CcTestCase, self).__init__(context, path, mode)
     self.executable = executable
-    self.mode = mode
     self.raw_name = raw_name
     self.dependency = dependency
 
@@ -54,8 +51,7 @@ class CcTestCase(test.TestCase):
     serialization_file += '_' + self.GetName()
     serialization_option = '--testing_serialization_file=' + serialization_file
     result = [ self.executable, name, serialization_option ]
-    if self.mode == 'debug':
-      result += CCTEST_DEBUG_FLAGS
+    result += self.context.GetVmFlags(self, self.mode)
     return result
 
   def GetCommand(self):

test/es5conform/testcfg.py

@@ -37,9 +37,8 @@ HARNESS_FILES = ['sth.js']
 class ES5ConformTestCase(test.TestCase):
 
   def __init__(self, filename, path, context, root, mode, framework):
-    super(ES5ConformTestCase, self).__init__(context, path)
+    super(ES5ConformTestCase, self).__init__(context, path, mode)
     self.filename = filename
-    self.mode = mode
     self.framework = framework
     self.root = root
 
@@ -55,7 +54,7 @@ class ES5ConformTestCase(test.TestCase):
     return 'FAILED!' in output.stdout
 
   def GetCommand(self):
-    result = [self.context.GetVm(self.mode)]
+    result = self.context.GetVmCommand(self, self.mode)
     result += ['-e', 'var window = this']
     result += self.framework
     result.append(self.filename)

test/message/testcfg.py

@@ -35,11 +35,10 @@ FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
 class MessageTestCase(test.TestCase):
 
   def __init__(self, path, file, expected, mode, context, config):
-    super(MessageTestCase, self).__init__(context, path)
+    super(MessageTestCase, self).__init__(context, path, mode)
     self.file = file
     self.expected = expected
     self.config = config
-    self.mode = mode
 
   def IgnoreLine(self, str):
     """Ignore empty lines and valgrind output."""
@@ -79,7 +78,7 @@ class MessageTestCase(test.TestCase):
     return self.path[-1]
 
   def GetCommand(self):
-    result = [self.config.context.GetVm(self.mode)]
+    result = self.config.context.GetVmCommand(self, self.mode)
     source = open(self.file).read()
     flags_match = FLAGS_PATTERN.search(source)
     if flags_match:

test/mjsunit/testcfg.py

@@ -31,7 +31,6 @@ from os.path import join, dirname, exists
 import re
 import tempfile
 
-MJSUNIT_DEBUG_FLAGS = ['--enable-slow-asserts', '--debug-code', '--verify-heap']
 FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
 FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
 SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
@@ -40,10 +39,9 @@ SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
 class MjsunitTestCase(test.TestCase):
 
   def __init__(self, path, file, mode, context, config):
-    super(MjsunitTestCase, self).__init__(context, path)
+    super(MjsunitTestCase, self).__init__(context, path, mode)
     self.file = file
     self.config = config
-    self.mode = mode
     self.self_script = False
 
   def GetLabel(self):
@@ -53,13 +51,11 @@ class MjsunitTestCase(test.TestCase):
     return self.path[-1]
 
   def GetCommand(self):
-    result = [self.config.context.GetVm(self.mode)]
+    result = self.config.context.GetVmCommand(self, self.mode)
     source = open(self.file).read()
     flags_match = FLAGS_PATTERN.search(source)
     if flags_match:
       result += flags_match.group(1).strip().split()
-    if self.mode == 'debug':
-      result += MJSUNIT_DEBUG_FLAGS
     additional_files = []
     files_match = FILES_PATTERN.search(source);
     # Accept several lines of 'Files:'
@@ -94,8 +90,8 @@ class MjsunitTestCase(test.TestCase):
     self.self_script = self_script
     return self_script
 
-  def Cleanup(self):
-    if self.self_script:
+  def AfterRun(self, result):
+    if self.self_script and (not result.HasPreciousOutput()):
       test.CheckedUnlink(self.self_script)
 
 class MjsunitTestConfiguration(test.TestConfiguration):

test/mozilla/testcfg.py

@@ -57,9 +57,8 @@ TEST_DIRS = """
 class MozillaTestCase(test.TestCase):
 
   def __init__(self, filename, path, context, root, mode, framework):
-    super(MozillaTestCase, self).__init__(context, path)
+    super(MozillaTestCase, self).__init__(context, path, mode)
     self.filename = filename
-    self.mode = mode
     self.framework = framework
     self.root = root
 
@@ -75,8 +74,8 @@ class MozillaTestCase(test.TestCase):
     return 'FAILED!' in output.stdout
 
   def GetCommand(self):
-    result = [self.context.GetVm(self.mode), '--expose-gc',
-              join(self.root, 'mozilla-shell-emulation.js')]
+    result = self.context.GetVmCommand(self, self.mode) + \
+       [ '--expose-gc', join(self.root, 'mozilla-shell-emulation.js') ]
     result += self.framework
     result.append(self.filename)
     return result

test/sputnik/testcfg.py

@@ -36,9 +36,8 @@ import time
 class SputnikTestCase(test.TestCase):
 
   def __init__(self, case, path, context, mode):
-    super(SputnikTestCase, self).__init__(context, path)
+    super(SputnikTestCase, self).__init__(context, path, mode)
     self.case = case
-    self.mode = mode
     self.tmpfile = None
     self.source = None
 
@@ -56,12 +55,13 @@ class SputnikTestCase(test.TestCase):
     self.tmpfile.Write(self.GetSource())
     self.tmpfile.Close()
 
-  def AfterRun(self):
-    self.tmpfile.Dispose()
+  def AfterRun(self, result):
+    # Dispose the temporary file if everything looks okay.
+    if not result.HasPreciousOutput(): self.tmpfile.Dispose()
     self.tmpfile = None
 
   def GetCommand(self):
-    result = [self.context.GetVm(self.mode)]
+    result = self.context.GetVmCommand(self, self.mode)
     result.append(self.tmpfile.name)
     return result
 

tools/test.py

@@ -331,10 +331,11 @@ class CommandOutput(object):
 
 class TestCase(object):
 
-  def __init__(self, context, path):
+  def __init__(self, context, path, mode):
     self.path = path
     self.context = context
     self.duration = None
+    self.mode = mode
 
   def IsNegative(self):
     return False
@@ -355,14 +356,19 @@ class TestCase(object):
 
   def RunCommand(self, command):
     full_command = self.context.processor(command)
-    output = Execute(full_command, self.context, self.context.timeout)
+    output = Execute(full_command,
+                     self.context,
+                     self.context.GetTimeout(self.mode))
     self.Cleanup()
-    return TestOutput(self, full_command, output)
+    return TestOutput(self,
+                      full_command,
+                      output,
+                      self.context.store_unexpected_output)
 
   def BeforeRun(self):
     pass
 
-  def AfterRun(self):
+  def AfterRun(self, result):
     pass
 
   def Run(self):
@@ -370,7 +376,7 @@ class TestCase(object):
     try:
       result = self.RunCommand(self.GetCommand())
     finally:
-      self.AfterRun()
+      self.AfterRun(result)
     return result
 
   def Cleanup(self):
@@ -379,10 +385,11 @@ class TestCase(object):
 
 class TestOutput(object):
 
-  def __init__(self, test, command, output):
+  def __init__(self, test, command, output, store_unexpected_output):
     self.test = test
     self.command = command
     self.output = output
+    self.store_unexpected_output = store_unexpected_output
 
   def UnexpectedOutput(self):
     if self.HasCrashed():
@@ -395,6 +402,9 @@ class TestOutput(object):
       outcome = PASS
     return not outcome in self.test.outcomes
 
+  def HasPreciousOutput(self):
+    return self.UnexpectedOutput() and self.store_unexpected_output
+
   def HasCrashed(self):
     if utils.IsWindows():
       return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code)
@@ -557,6 +567,11 @@ class TestSuite(object):
     return self.name
 
 
+# Use this to run several variants of the tests, e.g.:
+# VARIANT_FLAGS = [[], ['--always_compact', '--noflush_code']]
+VARIANT_FLAGS = [[]]
+
+
 class TestRepository(TestSuite):
 
   def __init__(self, path):
@@ -583,8 +598,12 @@ class TestRepository(TestSuite):
   def GetBuildRequirements(self, path, context):
     return self.GetConfiguration(context).GetBuildRequirements()
 
-  def ListTests(self, current_path, path, context, mode):
-    return self.GetConfiguration(context).ListTests(current_path, path, mode)
+  def AddTestsToList(self, result, current_path, path, context, mode):
+    for v in VARIANT_FLAGS:
+      tests = self.GetConfiguration(context).ListTests(current_path, path, mode)
+      for t in tests: t.variant_flags = v
+      result += tests
+
 
   def GetTestStatus(self, context, sections, defs):
     self.GetConfiguration(context).GetTestStatus(sections, defs)
@@ -611,7 +630,7 @@ class LiteralTestSuite(TestSuite):
       test_name = test.GetName()
       if not name or name.match(test_name):
         full_path = current_path + [test_name]
-        result += test.ListTests(full_path, path, context, mode)
+        test.AddTestsToList(result, full_path, path, context, mode)
     return result
 
   def GetTestStatus(self, context, sections, defs):
@@ -619,12 +638,20 @@ class LiteralTestSuite(TestSuite):
       test.GetTestStatus(context, sections, defs)
 
 
-SUFFIX = {'debug': '_g', 'release': ''}
+SUFFIX = {
+    'debug'   : '_g',
+    'release' : '' }
+FLAGS = {
+    'debug'   : ['--enable-slow-asserts', '--debug-code', '--verify-heap'],
+    'release' : []}
+TIMEOUT_SCALEFACTOR = {
+    'debug'   : 4,
+    'release' : 1 }
 
 
 class Context(object):
 
-  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs):
+  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output):
     self.workspace = workspace
     self.buildspace = buildspace
     self.verbose = verbose
@@ -632,6 +659,7 @@ class Context(object):
     self.timeout = timeout
     self.processor = processor
     self.suppress_dialogs = suppress_dialogs
+    self.store_unexpected_output = store_unexpected_output
 
   def GetVm(self, mode):
     name = self.vm_root + SUFFIX[mode]
@@ -639,6 +667,15 @@ class Context(object):
       name = name + '.exe'
     return name
 
+  def GetVmCommand(self, testcase, mode):
+    return [self.GetVm(mode)] + self.GetVmFlags(testcase, mode)
+
+  def GetVmFlags(self, testcase, mode):
+    return testcase.variant_flags + FLAGS[mode]
+
+  def GetTimeout(self, mode):
+    return self.timeout * TIMEOUT_SCALEFACTOR[mode]
+
 def RunTestCases(cases_to_run, progress, tasks):
   progress = PROGRESS_INDICATORS[progress](cases_to_run)
   return progress.Run(tasks)
@@ -1121,7 +1158,13 @@ def BuildOptions():
         dest="suppress_dialogs", default=True, action="store_true")
   result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests",
         dest="suppress_dialogs", action="store_false")
-  result.add_option("--shell", help="Path to V8 shell", default="shell");
+  result.add_option("--shell", help="Path to V8 shell", default="shell")
+  result.add_option("--store-unexpected-output", 
+      help="Store the temporary JS files from tests that fails",
+      dest="store_unexpected_output", default=True, action="store_true")
+  result.add_option("--no-store-unexpected-output", 
+      help="Deletes the temporary JS files from tests that fails",
+      dest="store_unexpected_output", action="store_false")
   return result
 
 
@@ -1258,11 +1301,13 @@ def Main():
 
   shell = abspath(options.shell)
   buildspace = dirname(shell)
+
   context = Context(workspace, buildspace, VERBOSE,
                     shell,
                     options.timeout,
                     GetSpecialCommandProcessor(options.special_command),
-                    options.suppress_dialogs)
+                    options.suppress_dialogs,
+                    options.store_unexpected_output)
   # First build the required targets
   if not options.no_build:
     reqs = [ ]
@@ -1278,7 +1323,7 @@ def Main():
   # Just return if we are only building the targets for running the tests.
   if options.build_only:
     return 0
-  
+
   # Get status for tests
   sections = [ ]
   defs = { }