test.py 42.7 KB
Newer Older
1 2
#!/usr/bin/env python
#
3
# Copyright 2008 the V8 project authors. All rights reserved.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Google Inc. nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

30

31 32 33
import imp
import optparse
import os
34
from os.path import join, dirname, abspath, basename, isdir, exists
35 36
import platform
import re
37
import signal
38 39
import subprocess
import sys
40
import tempfile
41
import time
42
import threading
43
import utils
44
from Queue import Queue, Empty
45 46 47 48 49 50 51 52 53 54 55 56 57 58


VERBOSE = False


# ---------------------------------------------
# --- P r o g r e s s   I n d i c a t o r s ---
# ---------------------------------------------


class ProgressIndicator(object):

  def __init__(self, cases):
    self.cases = cases
59 60 61
    self.queue = Queue(len(cases))
    for case in cases:
      self.queue.put_nowait(case)
62
    self.succeeded = 0
63 64 65
    self.remaining = len(cases)
    self.total = len(cases)
    self.failed = [ ]
66
    self.crashed = 0
67 68
    self.terminate = False
    self.lock = threading.Lock()
69

70 71 72 73 74 75 76 77 78 79 80
  def PrintFailureHeader(self, test):
    if test.IsNegative():
      negative_marker = '[negative] '
    else:
      negative_marker = ''
    print "=== %(label)s %(negative)s===" % {
      'label': test.GetLabel(),
      'negative': negative_marker
    }
    print "Path: %s" % "/".join(test.path)

81
  def Run(self, tasks):
82
    self.Starting()
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
    threads = []
    # Spawn N-1 threads and then use this thread as the last one.
    # That way -j1 avoids threading altogether which is a nice fallback
    # in case of threading problems.
    for i in xrange(tasks - 1):
      thread = threading.Thread(target=self.RunSingle, args=[])
      threads.append(thread)
      thread.start()
    try:
      self.RunSingle()
      # Wait for the remaining threads
      for thread in threads:
        # Use a timeout so that signals (ctrl-c) will be processed.
        thread.join(timeout=10000000)
    except Exception, e:
      # If there's an exception we schedule an interruption for any
      # remaining threads.
      self.terminate = True
101 102
      # ...and then reraise the exception to bail out
      raise
103 104 105 106 107 108 109 110 111
    self.Done()
    return not self.failed

  def RunSingle(self):
    while not self.terminate:
      try:
        test = self.queue.get_nowait()
      except Empty:
        return
112
      case = test.case
113
      self.lock.acquire()
114
      self.AboutToRun(case)
115 116
      self.lock.release()
      try:
117
        start = time.time()
118
        output = case.Run()
119
        case.duration = (time.time() - start)
120 121
      except BreakNowException:
        self.terminate = True
122 123 124 125 126 127
      except IOError, e:
        assert self.terminate
        return
      if self.terminate:
        return
      self.lock.acquire()
128
      if output.UnexpectedOutput():
129
        self.failed.append(output)
130 131
        if output.HasCrashed():
          self.crashed += 1
132 133 134 135
      else:
        self.succeeded += 1
      self.remaining -= 1
      self.HasRun(output)
136
      self.lock.release()
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157


def EscapeCommand(command):
  parts = []
  for part in command:
    if ' ' in part:
      # Escape spaces.  We may need to escape more characters for this
      # to work properly.
      parts.append('"%s"' % part)
    else:
      parts.append(part)
  return " ".join(parts)


class SimpleProgressIndicator(ProgressIndicator):

  def Starting(self):
    print 'Running %i tests' % len(self.cases)

  def Done(self):
    print
158
    for failed in self.failed:
159
      self.PrintFailureHeader(failed.test)
160 161 162 163 164 165
      if failed.output.stderr:
        print "--- stderr ---"
        print failed.output.stderr.strip()
      if failed.output.stdout:
        print "--- stdout ---"
        print failed.output.stdout.strip()
166
      print "Command: %s" % EscapeCommand(failed.command)
167
      if failed.HasCrashed():
168
        print "--- CRASHED ---"
169 170
      if failed.HasTimedOut():
        print "--- TIMEOUT ---"
171
    if len(self.failed) == 0:
172 173 174 175 176 177
      print "==="
      print "=== All tests succeeded"
      print "==="
    else:
      print
      print "==="
178
      print "=== %i tests failed" % len(self.failed)
179 180
      if self.crashed > 0:
        print "=== %i tests CRASHED" % self.crashed
181 182 183 184 185 186
      print "==="


class VerboseProgressIndicator(SimpleProgressIndicator):

  def AboutToRun(self, case):
187
    print 'Starting %s...' % case.GetLabel()
188 189
    sys.stdout.flush()

190 191
  def HasRun(self, output):
    if output.UnexpectedOutput():
192 193 194 195
      if output.HasCrashed():
        outcome = 'CRASH'
      else:
        outcome = 'FAIL'
196
    else:
197 198
      outcome = 'pass'
    print 'Done running %s: %s' % (output.test.GetLabel(), outcome)
199 200 201 202 203 204 205


class DotsProgressIndicator(SimpleProgressIndicator):

  def AboutToRun(self, case):
    pass

206
  def HasRun(self, output):
207
    total = self.succeeded + len(self.failed)
208 209
    if (total > 1) and (total % 50 == 1):
      sys.stdout.write('\n')
210
    if output.UnexpectedOutput():
211 212 213
      if output.HasCrashed():
        sys.stdout.write('C')
        sys.stdout.flush()
214 215 216
      elif output.HasTimedOut():
        sys.stdout.write('T')
        sys.stdout.flush()
217 218 219
      else:
        sys.stdout.write('F')
        sys.stdout.flush()
220 221 222 223 224 225 226 227 228 229 230 231
    else:
      sys.stdout.write('.')
      sys.stdout.flush()


class CompactProgressIndicator(ProgressIndicator):

  def __init__(self, cases, templates):
    super(CompactProgressIndicator, self).__init__(cases)
    self.templates = templates
    self.last_status_length = 0
    self.start_time = time.time()
232

233 234
  def Starting(self):
    pass
235

236 237
  def Done(self):
    self.PrintProgress('Done')
238

239
  def AboutToRun(self, case):
240 241
    self.PrintProgress(case.GetLabel())

242
  def HasRun(self, output):
243
    if output.UnexpectedOutput():
244 245
      self.ClearLine(self.last_status_length)
      self.PrintFailureHeader(output.test)
246 247 248 249 250 251
      stdout = output.output.stdout.strip()
      if len(stdout):
        print self.templates['stdout'] % stdout
      stderr = output.output.stderr.strip()
      if len(stderr):
        print self.templates['stderr'] % stderr
252
      print "Command: %s" % EscapeCommand(output.command)
253
      if output.HasCrashed():
254
        print "--- CRASHED ---"
255 256
      if output.HasTimedOut():
        print "--- TIMEOUT ---"
257 258 259 260 261 262 263 264 265 266 267 268 269

  def Truncate(self, str, length):
    if length and (len(str) > (length - 3)):
      return str[:(length-3)] + "..."
    else:
      return str

  def PrintProgress(self, name):
    self.ClearLine(self.last_status_length)
    elapsed = time.time() - self.start_time
    status = self.templates['status_line'] % {
      'passed': self.succeeded,
      'remaining': (((self.total - self.remaining) * 100) // self.total),
270
      'failed': len(self.failed),
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289
      'test': name,
      'mins': int(elapsed) / 60,
      'secs': int(elapsed) % 60
    }
    status = self.Truncate(status, 78)
    self.last_status_length = len(status)
    print status,
    sys.stdout.flush()


class ColorProgressIndicator(CompactProgressIndicator):

  def __init__(self, cases):
    templates = {
      'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
      'stdout': "\033[1m%s\033[0m",
      'stderr': "\033[31m%s\033[0m",
    }
    super(ColorProgressIndicator, self).__init__(cases, templates)
290

291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
  def ClearLine(self, last_line_length):
    print "\033[1K\r",


class MonochromeProgressIndicator(CompactProgressIndicator):

  def __init__(self, cases):
    templates = {
      'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
      'stdout': '%s',
      'stderr': '%s',
      'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
      'max_length': 78
    }
    super(MonochromeProgressIndicator, self).__init__(cases, templates)
306

307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
  def ClearLine(self, last_line_length):
    print ("\r" + (" " * last_line_length) + "\r"),


PROGRESS_INDICATORS = {
  'verbose': VerboseProgressIndicator,
  'dots': DotsProgressIndicator,
  'color': ColorProgressIndicator,
  'mono': MonochromeProgressIndicator
}


# -------------------------
# --- F r a m e w o r k ---
# -------------------------

323 324 325 326 327 328
class BreakNowException(Exception):
  def __init__(self, value):
    self.value = value
  def __str__(self):
    return repr(self.value)

329 330 331

class CommandOutput(object):

332
  def __init__(self, exit_code, timed_out, stdout, stderr):
333
    self.exit_code = exit_code
334
    self.timed_out = timed_out
335 336
    self.stdout = stdout
    self.stderr = stderr
337
    self.failed = None
338 339 340 341


class TestCase(object):

342
  def __init__(self, context, path, mode):
343
    self.path = path
344
    self.context = context
345
    self.duration = None
346
    self.mode = mode
347 348 349 350

  def IsNegative(self):
    return False

351 352 353
  def TestsIsolates(self):
    return False

354 355 356
  def CompareTime(self, other):
    return cmp(other.duration, self.duration)

357
  def DidFail(self, output):
358 359 360
    if output.failed is None:
      output.failed = self.IsFailureOutput(output)
    return output.failed
361

362 363
  def IsFailureOutput(self, output):
    return output.exit_code != 0
364

365 366
  def GetSource(self):
    return "(no source available)"
367

368
  def RunCommand(self, command):
369
    full_command = self.context.processor(command)
370 371
    output = Execute(full_command,
                     self.context,
372
                     self.context.GetTimeout(self, self.mode))
373
    self.Cleanup()
374 375 376 377
    return TestOutput(self,
                      full_command,
                      output,
                      self.context.store_unexpected_output)
378

379 380 381
  def BeforeRun(self):
    pass

382
  def AfterRun(self, result):
383 384
    pass

385 386 387
  def GetCustomFlags(self, mode):
    return None

388
  def Run(self):
389
    self.BeforeRun()
390
    result = None
391 392
    try:
      result = self.RunCommand(self.GetCommand())
393 394
    except:
      self.terminate = True
395
      raise BreakNowException("User pressed CTRL+C or IO went wrong")
396
    finally:
397
      self.AfterRun(result)
398
    return result
399

400 401 402
  def Cleanup(self):
    return

403 404 405

class TestOutput(object):

406
  def __init__(self, test, command, output, store_unexpected_output):
407 408 409
    self.test = test
    self.command = command
    self.output = output
410
    self.store_unexpected_output = store_unexpected_output
411

412
  def UnexpectedOutput(self):
413 414
    if self.HasCrashed():
      outcome = CRASH
415 416
    elif self.HasTimedOut():
      outcome = TIMEOUT
417
    elif self.HasFailed():
418 419 420 421 422
      outcome = FAIL
    else:
      outcome = PASS
    return not outcome in self.test.outcomes

423 424 425
  def HasPreciousOutput(self):
    return self.UnexpectedOutput() and self.store_unexpected_output

426
  def HasCrashed(self):
427
    if utils.IsWindows():
428 429
      return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code)
    else:
430 431 432 433 434
      # Timed out tests will have exit_code -signal.SIGTERM.
      if self.output.timed_out:
        return False
      return self.output.exit_code < 0 and \
             self.output.exit_code != -signal.SIGABRT
435

436
  def HasTimedOut(self):
437
    return self.output.timed_out
438

439
  def HasFailed(self):
440
    execution_failed = self.test.DidFail(self.output)
441 442 443 444 445 446
    if self.test.IsNegative():
      return not execution_failed
    else:
      return execution_failed


447
def KillProcessWithID(pid):
448
  if utils.IsWindows():
449 450 451 452 453
    os.popen('taskkill /T /F /PID %d' % pid)
  else:
    os.kill(pid, signal.SIGTERM)


454 455 456
MAX_SLEEP_TIME = 0.1
INITIAL_SLEEP_TIME = 0.0001
SLEEP_TIME_FACTOR = 1.25
457

458 459
SEM_INVALID_VALUE = -1
SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h
460

461 462 463 464
def Win32SetErrorMode(mode):
  prev_error_mode = SEM_INVALID_VALUE
  try:
    import ctypes
465
    prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode)
466 467 468
  except ImportError:
    pass
  return prev_error_mode
469

470 471 472
def RunProcess(context, timeout, args, **rest):
  if context.verbose: print "#", " ".join(args)
  popen_args = args
473
  prev_error_mode = SEM_INVALID_VALUE
474
  if utils.IsWindows():
475
    popen_args = '"' + subprocess.list2cmdline(args) + '"'
476
    if context.suppress_dialogs:
477 478 479
      # Try to change the error mode to avoid dialogs on fatal errors. Don't
      # touch any existing error mode flags by merging the existing error mode.
      # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx.
480 481 482
      error_mode = SEM_NOGPFAULTERRORBOX
      prev_error_mode = Win32SetErrorMode(error_mode)
      Win32SetErrorMode(error_mode | prev_error_mode)
483
  process = subprocess.Popen(
484
    shell = utils.IsWindows(),
485 486
    args = popen_args,
    **rest
487
  )
488
  if utils.IsWindows() and context.suppress_dialogs and prev_error_mode != SEM_INVALID_VALUE:
489
    Win32SetErrorMode(prev_error_mode)
490 491 492 493 494 495 496
  # Compute the end time - if the process crosses this limit we
  # consider it timed out.
  if timeout is None: end_time = None
  else: end_time = time.time() + timeout
  timed_out = False
  # Repeatedly check the exit code from the process in a
  # loop and keep track of whether or not it times out.
497 498 499
  exit_code = None
  sleep_time = INITIAL_SLEEP_TIME
  while exit_code is None:
500 501 502 503 504 505 506 507 508 509 510 511
    if (not end_time is None) and (time.time() >= end_time):
      # Kill the process and wait for it to exit.
      KillProcessWithID(process.pid)
      exit_code = process.wait()
      timed_out = True
    else:
      exit_code = process.poll()
      time.sleep(sleep_time)
      sleep_time = sleep_time * SLEEP_TIME_FACTOR
      if sleep_time > MAX_SLEEP_TIME:
        sleep_time = MAX_SLEEP_TIME
  return (process, exit_code, timed_out)
512 513


514 515 516 517 518
def PrintError(str):
  sys.stderr.write(str)
  sys.stderr.write('\n')


519
def CheckedUnlink(name):
520 521 522 523 524 525 526 527 528 529
  # On Windows, when run with -jN in parallel processes,
  # OS often fails to unlink the temp file. Not sure why.
  # Need to retry.
  # Idea from https://bugs.webkit.org/attachment.cgi?id=75982&action=prettypatch
  retry_count = 0
  while retry_count < 30:
    try:
      os.unlink(name)
      return
    except OSError, e:
530
      retry_count += 1
531 532
      time.sleep(retry_count * 0.1)
  PrintError("os.unlink() " + str(e))
533

534
def Execute(args, context, timeout=None):
535 536
  (fd_out, outname) = tempfile.mkstemp()
  (fd_err, errname) = tempfile.mkstemp()
537
  (process, exit_code, timed_out) = RunProcess(
538
    context,
539
    timeout,
540
    args = args,
541 542
    stdout = fd_out,
    stderr = fd_err,
543
  )
544 545 546 547
  os.close(fd_out)
  os.close(fd_err)
  output = file(outname).read()
  errors = file(errname).read()
548 549
  CheckedUnlink(outname)
  CheckedUnlink(errname)
550
  return CommandOutput(exit_code, timed_out, output, errors)
551 552


553 554
def ExecuteNoCapture(args, context, timeout=None):
  (process, exit_code, timed_out) = RunProcess(
555
    context,
556
    timeout,
557 558
    args = args,
  )
559
  return CommandOutput(exit_code, False, "", "")
560 561 562 563 564 565 566 567 568


def CarCdr(path):
  if len(path) == 0:
    return (None, [ ])
  else:
    return (path[0], path[1:])


569 570 571 572 573 574 575
# Use this to run several variants of the tests, e.g.:
# VARIANT_FLAGS = [[], ['--always_compact', '--noflush_code']]
VARIANT_FLAGS = [[],
                 ['--stress-opt', '--always-opt'],
                 ['--nocrankshaft']]


576 577 578 579 580 581
class TestConfiguration(object):

  def __init__(self, context, root):
    self.context = context
    self.root = root

582 583 584 585 586 587 588 589
  def Contains(self, path, file):
    if len(path) > len(file):
      return False
    for i in xrange(len(path)):
      if not path[i].match(file[i]):
        return False
    return True

590 591 592
  def GetTestStatus(self, sections, defs):
    pass

593 594 595 596 597
  def VariantFlags(self):
    return VARIANT_FLAGS



598 599 600 601 602 603 604 605 606 607 608 609 610

class TestSuite(object):

  def __init__(self, name):
    self.name = name

  def GetName(self):
    return self.name


class TestRepository(TestSuite):

  def __init__(self, path):
611 612 613
    normalized_path = abspath(path)
    super(TestRepository, self).__init__(basename(normalized_path))
    self.path = normalized_path
614 615 616 617 618 619 620 621 622 623
    self.is_loaded = False
    self.config = None

  def GetConfiguration(self, context):
    if self.is_loaded:
      return self.config
    self.is_loaded = True
    file = None
    try:
      (file, pathname, description) = imp.find_module('testcfg', [ self.path ])
624
      module = imp.load_module('testcfg', file, pathname, description)
625 626 627 628 629 630 631 632 633
      self.config = module.GetConfiguration(context, self.path)
    finally:
      if file:
        file.close()
    return self.config

  def GetBuildRequirements(self, path, context):
    return self.GetConfiguration(context).GetBuildRequirements()

634
  def AddTestsToList(self, result, current_path, path, context, mode):
635
    for v in self.GetConfiguration(context).VariantFlags():
636
      tests = self.GetConfiguration(context).ListTests(current_path, path, mode, v)
637 638 639
      for t in tests: t.variant_flags = v
      result += tests

640 641 642
  def GetTestStatus(self, context, sections, defs):
    self.GetConfiguration(context).GetTestStatus(sections, defs)

643 644 645 646 647 648 649 650 651 652 653

class LiteralTestSuite(TestSuite):

  def __init__(self, tests):
    super(LiteralTestSuite, self).__init__('root')
    self.tests = tests

  def GetBuildRequirements(self, path, context):
    (name, rest) = CarCdr(path)
    result = [ ]
    for test in self.tests:
654
      if not name or name.match(test.GetName()):
655 656 657
        result += test.GetBuildRequirements(rest, context)
    return result

658
  def ListTests(self, current_path, path, context, mode, variant_flags):
659 660 661 662
    (name, rest) = CarCdr(path)
    result = [ ]
    for test in self.tests:
      test_name = test.GetName()
663
      if not name or name.match(test_name):
664
        full_path = current_path + [test_name]
665
        test.AddTestsToList(result, full_path, path, context, mode)
666 667
    return result

668 669 670 671
  def GetTestStatus(self, context, sections, defs):
    for test in self.tests:
      test.GetTestStatus(context, sections, defs)

672

673 674 675 676 677 678 679 680 681
SUFFIX = {
    'debug'   : '_g',
    'release' : '' }
FLAGS = {
    'debug'   : ['--enable-slow-asserts', '--debug-code', '--verify-heap'],
    'release' : []}
TIMEOUT_SCALEFACTOR = {
    'debug'   : 4,
    'release' : 1 }
682 683


684 685
class Context(object):

686
  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output):
687 688
    self.workspace = workspace
    self.buildspace = buildspace
689 690 691
    self.verbose = verbose
    self.vm_root = vm
    self.timeout = timeout
692
    self.processor = processor
693
    self.suppress_dialogs = suppress_dialogs
694
    self.store_unexpected_output = store_unexpected_output
695 696

  def GetVm(self, mode):
697 698 699 700
    name = self.vm_root + SUFFIX[mode]
    if utils.IsWindows() and not name.endswith('.exe'):
      name = name + '.exe'
    return name
701

702 703 704 705
  def GetVmCommand(self, testcase, mode):
    return [self.GetVm(mode)] + self.GetVmFlags(testcase, mode)

  def GetVmFlags(self, testcase, mode):
706 707 708 709
    flags = testcase.GetCustomFlags(mode)
    if flags is None:
      flags = FLAGS[mode]
    return testcase.variant_flags + flags
710

711 712 713 714 715 716
  def GetTimeout(self, testcase, mode):
    result = self.timeout * TIMEOUT_SCALEFACTOR[mode]
    if '--stress-opt' in self.GetVmFlags(testcase, mode):
      return result * 2
    else:
      return result
717

718
def RunTestCases(cases_to_run, progress, tasks):
719
  progress = PROGRESS_INDICATORS[progress](cases_to_run)
720 721 722 723 724 725
  result = 0
  try:
    result = progress.Run(tasks)
  except Exception, e:
    print "\n", e
  return result
726 727


728 729 730 731
def BuildRequirements(context, requirements, mode, scons_flags):
  command_line = (['scons', '-Y', context.workspace, 'mode=' + ",".join(mode)]
                  + requirements
                  + scons_flags)
732
  output = ExecuteNoCapture(command_line, context)
733 734 735 736 737 738 739 740 741 742 743 744 745 746
  return output.exit_code == 0


# -------------------------------------------
# --- T e s t   C o n f i g u r a t i o n ---
# -------------------------------------------


SKIP = 'skip'
FAIL = 'fail'
PASS = 'pass'
OKAY = 'okay'
TIMEOUT = 'timeout'
CRASH = 'crash'
747
SLOW = 'slow'
748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771


class Expression(object):
  pass


class Constant(Expression):

  def __init__(self, value):
    self.value = value

  def Evaluate(self, env, defs):
    return self.value


class Variable(Expression):

  def __init__(self, name):
    self.name = name

  def GetOutcomes(self, env, defs):
    if self.name in env: return ListSet([env[self.name]])
    else: return Nothing()

772 773 774
  def Evaluate(self, env, defs):
    return env[self.name]

775 776 777 778 779 780 781 782

class Outcome(Expression):

  def __init__(self, name):
    self.name = name

  def GetOutcomes(self, env, defs):
    if self.name in defs:
783
      return defs[self.name].GetOutcomes(env, defs)
784 785 786 787 788 789 790 791 792 793 794 795
    else:
      return ListSet([self.name])


class Set(object):
  pass


class ListSet(Set):

  def __init__(self, elms):
    self.elms = elms
796

797 798 799 800 801 802 803 804 805 806 807 808
  def __str__(self):
    return "ListSet%s" % str(self.elms)

  def Intersect(self, that):
    if not isinstance(that, ListSet):
      return that.Intersect(self)
    return ListSet([ x for x in self.elms if x in that.elms ])

  def Union(self, that):
    if not isinstance(that, ListSet):
      return that.Union(self)
    return ListSet(self.elms + [ x for x in that.elms if x not in self.elms ])
809

810 811 812 813 814 815 816 817 818 819 820
  def IsEmpty(self):
    return len(self.elms) == 0


class Everything(Set):

  def Intersect(self, that):
    return that

  def Union(self, that):
    return self
821

822 823 824 825 826 827 828 829 830 831 832
  def IsEmpty(self):
    return False


class Nothing(Set):

  def Intersect(self, that):
    return self

  def Union(self, that):
    return that
833

834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885
  def IsEmpty(self):
    return True


class Operation(Expression):

  def __init__(self, left, op, right):
    self.left = left
    self.op = op
    self.right = right

  def Evaluate(self, env, defs):
    if self.op == '||' or self.op == ',':
      return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs)
    elif self.op == 'if':
      return False
    elif self.op == '==':
      inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs))
      return not inter.IsEmpty()
    else:
      assert self.op == '&&'
      return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs)

  def GetOutcomes(self, env, defs):
    if self.op == '||' or self.op == ',':
      return self.left.GetOutcomes(env, defs).Union(self.right.GetOutcomes(env, defs))
    elif self.op == 'if':
      if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs)
      else: return Nothing()
    else:
      assert self.op == '&&'
      return self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs))


def IsAlpha(str):
  for char in str:
    if not (char.isalpha() or char.isdigit() or char == '_'):
      return False
  return True


class Tokenizer(object):
  """A simple string tokenizer that chops expressions into variables,
  parens and operators"""

  def __init__(self, expr):
    self.index = 0
    self.expr = expr
    self.length = len(expr)
    self.tokens = None

  def Current(self, length = 1):
886
    if not self.HasMore(length): return ""
887 888 889 890 891 892 893 894 895 896 897 898
    return self.expr[self.index:self.index+length]

  def HasMore(self, length = 1):
    return self.index < self.length + (length - 1)

  def Advance(self, count = 1):
    self.index = self.index + count

  def AddToken(self, token):
    self.tokens.append(token)

  def SkipSpaces(self):
899
    while self.HasMore() and self.Current().isspace():
900 901 902 903 904 905 906 907 908 909 910
      self.Advance()

  def Tokenize(self):
    self.tokens = [ ]
    while self.HasMore():
      self.SkipSpaces()
      if not self.HasMore():
        return None
      if self.Current() == '(':
        self.AddToken('(')
        self.Advance()
911
      elif self.Current() == ')':
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003
        self.AddToken(')')
        self.Advance()
      elif self.Current() == '$':
        self.AddToken('$')
        self.Advance()
      elif self.Current() == ',':
        self.AddToken(',')
        self.Advance()
      elif IsAlpha(self.Current()):
        buf = ""
        while self.HasMore() and IsAlpha(self.Current()):
          buf += self.Current()
          self.Advance()
        self.AddToken(buf)
      elif self.Current(2) == '&&':
        self.AddToken('&&')
        self.Advance(2)
      elif self.Current(2) == '||':
        self.AddToken('||')
        self.Advance(2)
      elif self.Current(2) == '==':
        self.AddToken('==')
        self.Advance(2)
      else:
        return None
    return self.tokens


class Scanner(object):
  """A simple scanner that can serve out tokens from a given list"""

  def __init__(self, tokens):
    self.tokens = tokens
    self.length = len(tokens)
    self.index = 0

  def HasMore(self):
    return self.index < self.length

  def Current(self):
    return self.tokens[self.index]

  def Advance(self):
    self.index = self.index + 1


def ParseAtomicExpression(scan):
  if scan.Current() == "true":
    scan.Advance()
    return Constant(True)
  elif scan.Current() == "false":
    scan.Advance()
    return Constant(False)
  elif IsAlpha(scan.Current()):
    name = scan.Current()
    scan.Advance()
    return Outcome(name.lower())
  elif scan.Current() == '$':
    scan.Advance()
    if not IsAlpha(scan.Current()):
      return None
    name = scan.Current()
    scan.Advance()
    return Variable(name.lower())
  elif scan.Current() == '(':
    scan.Advance()
    result = ParseLogicalExpression(scan)
    if (not result) or (scan.Current() != ')'):
      return None
    scan.Advance()
    return result
  else:
    return None


BINARIES = ['==']
def ParseOperatorExpression(scan):
  left = ParseAtomicExpression(scan)
  if not left: return None
  while scan.HasMore() and (scan.Current() in BINARIES):
    op = scan.Current()
    scan.Advance()
    right = ParseOperatorExpression(scan)
    if not right:
      return None
    left = Operation(left, op, right)
  return left


def ParseConditionalExpression(scan):
  left = ParseOperatorExpression(scan)
  if not left: return None
1004
  while scan.HasMore() and (scan.Current() == 'if'):
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
    scan.Advance()
    right = ParseOperatorExpression(scan)
    if not right:
      return None
    left=  Operation(left, 'if', right)
  return left


LOGICALS = ["&&", "||", ","]
def ParseLogicalExpression(scan):
  left = ParseConditionalExpression(scan)
  if not left: return None
  while scan.HasMore() and (scan.Current() in LOGICALS):
    op = scan.Current()
    scan.Advance()
    right = ParseConditionalExpression(scan)
    if not right:
      return None
    left = Operation(left, op, right)
  return left


def ParseCondition(expr):
  """Parses a logical expression into an Expression object"""
1029
  tokens = Tokenizer(expr).Tokenize()
1030 1031 1032 1033 1034 1035 1036 1037
  if not tokens:
    print "Malformed expression: '%s'" % expr
    return None
  scan = Scanner(tokens)
  ast = ParseLogicalExpression(scan)
  if not ast:
    print "Malformed expression: '%s'" % expr
    return None
1038 1039 1040
  if scan.HasMore():
    print "Malformed expression: '%s'" % expr
    return None
1041 1042 1043 1044 1045 1046 1047 1048 1049
  return ast


class ClassifiedTest(object):

  def __init__(self, case, outcomes):
    self.case = case
    self.outcomes = outcomes

1050 1051 1052
  def TestsIsolates(self):
    return self.case.TestsIsolates()

1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065

class Configuration(object):
  """The parsed contents of a configuration file"""

  def __init__(self, sections, defs):
    self.sections = sections
    self.defs = defs

  def ClassifyTests(self, cases, env):
    sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)]
    all_rules = reduce(list.__add__, [s.rules for s in sections], [])
    unused_rules = set(all_rules)
    result = [ ]
1066
    all_outcomes = set([])
1067 1068 1069 1070
    for case in cases:
      matches = [ r for r in all_rules if r.Contains(case.path) ]
      outcomes = set([])
      for rule in matches:
1071
        outcomes = outcomes.union(rule.GetOutcomes(env, self.defs))
1072
        unused_rules.discard(rule)
1073 1074 1075
      if not outcomes:
        outcomes = [PASS]
      case.outcomes = outcomes
1076
      all_outcomes = all_outcomes.union(outcomes)
1077
      result.append(ClassifiedTest(case, outcomes))
1078
    return (result, list(unused_rules), all_outcomes)
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096


class Section(object):
  """A section of the configuration file.  Sections are enabled or
  disabled prior to running the tests, based on their conditions"""

  def __init__(self, condition):
    self.condition = condition
    self.rules = [ ]

  def AddRule(self, rule):
    self.rules.append(rule)


class Rule(object):
  """A single rule that specifies the expected outcome for a single
  test."""

1097 1098
  def __init__(self, raw_path, path, value):
    self.raw_path = raw_path
1099 1100 1101 1102 1103 1104 1105
    self.path = path
    self.value = value

  def GetOutcomes(self, env, defs):
    set = self.value.GetOutcomes(env, defs)
    assert isinstance(set, ListSet)
    return set.elms
1106

1107 1108 1109 1110
  def Contains(self, path):
    if len(self.path) > len(path):
      return False
    for i in xrange(len(self.path)):
1111
      if not self.path[i].match(path[i]):
1112 1113 1114 1115 1116
        return False
    return True


HEADER_PATTERN = re.compile(r'\[([^]]+)\]')
1117
RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)')
1118
DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$')
1119
PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$')
1120 1121 1122 1123 1124


def ReadConfigurationInto(path, sections, defs):
  current_section = Section(Constant(True))
  sections.append(current_section)
1125
  prefix = []
1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
  for line in utils.ReadLinesFrom(path):
    header_match = HEADER_PATTERN.match(line)
    if header_match:
      condition_str = header_match.group(1).strip()
      condition = ParseCondition(condition_str)
      new_section = Section(condition)
      sections.append(new_section)
      current_section = new_section
      continue
    rule_match = RULE_PATTERN.match(line)
    if rule_match:
1137 1138
      path = prefix + SplitPath(rule_match.group(1).strip())
      value_str = rule_match.group(2).strip()
1139 1140 1141
      value = ParseCondition(value_str)
      if not value:
        return False
1142
      current_section.AddRule(Rule(rule_match.group(1), path, value))
1143 1144 1145 1146 1147 1148 1149 1150 1151
      continue
    def_match = DEF_PATTERN.match(line)
    if def_match:
      name = def_match.group(1).lower()
      value = ParseCondition(def_match.group(2).strip())
      if not value:
        return False
      defs[name] = value
      continue
1152 1153 1154 1155
    prefix_match = PREFIX_PATTERN.match(line)
    if prefix_match:
      prefix = SplitPath(prefix_match.group(1).strip())
      continue
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
    print "Malformed line: '%s'." % line
    return False
  return True


# ---------------
# --- M a i n ---
# ---------------


1166 1167 1168
ARCH_GUESS = utils.GuessArchitecture()


1169 1170
def BuildOptions():
  result = optparse.OptionParser()
1171 1172
  result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated)",
      default='release')
1173 1174
  result.add_option("-v", "--verbose", help="Verbose output",
      default=False, action="store_true")
1175 1176
  result.add_option("-S", dest="scons_flags", help="Flag to pass through to scons",
      default=[], action="append")
1177 1178 1179 1180 1181
  result.add_option("-p", "--progress",
      help="The style of progress indicator (verbose, dots, color, mono)",
      choices=PROGRESS_INDICATORS.keys(), default="mono")
  result.add_option("--no-build", help="Don't build requirements",
      default=False, action="store_true")
1182 1183
  result.add_option("--build-only", help="Only build requirements, don't run the tests",
      default=False, action="store_true")
1184 1185
  result.add_option("--report", help="Print a summary of the tests to be run",
      default=False, action="store_true")
1186 1187 1188 1189
  result.add_option("-s", "--suite", help="A test suite",
      default=[], action="append")
  result.add_option("-t", "--timeout", help="Timeout in seconds",
      default=60, type="int")
1190
  result.add_option("--arch", help='The architecture to run tests for',
1191
      default='none')
1192 1193
  result.add_option("--snapshot", help="Run the tests with snapshot turned on",
      default=False, action="store_true")
1194 1195
  result.add_option("--simulator", help="Run tests with architecture simulator",
      default='none')
1196
  result.add_option("--special-command", default=None)
1197 1198
  result.add_option("--valgrind", help="Run tests through valgrind",
      default=False, action="store_true")
1199 1200
  result.add_option("--cat", help="Print the source of the tests",
      default=False, action="store_true")
1201 1202
  result.add_option("--warn-unused", help="Report unused rules",
      default=False, action="store_true")
1203 1204
  result.add_option("-j", help="The number of parallel tasks to run",
      default=1, type="int")
1205 1206
  result.add_option("--time", help="Print timing information after running",
      default=False, action="store_true")
1207 1208 1209 1210
  result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for crashing tests",
        dest="suppress_dialogs", default=True, action="store_true")
  result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests",
        dest="suppress_dialogs", action="store_false")
1211
  result.add_option("--shell", help="Path to V8 shell", default="shell")
1212
  result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true")
1213
  result.add_option("--store-unexpected-output",
1214 1215
      help="Store the temporary JS files from tests that fails",
      dest="store_unexpected_output", default=True, action="store_true")
1216
  result.add_option("--no-store-unexpected-output",
1217 1218
      help="Deletes the temporary JS files from tests that fails",
      dest="store_unexpected_output", action="store_false")
1219 1220 1221 1222 1223 1224
  result.add_option("--stress-only",
                    help="Only run tests with --always-opt --stress-opt",
                    default=False, action="store_true")
  result.add_option("--nostress",
                    help="Don't run crankshaft --always-opt --stress-op test",
                    default=False, action="store_true")
1225 1226 1227
  result.add_option("--crankshaft",
                    help="Run with the --crankshaft flag",
                    default=False, action="store_true")
1228 1229 1230 1231 1232 1233
  result.add_option("--shard-count",
                    help="Split testsuites into this number of shards",
                    default=1, type="int")
  result.add_option("--shard-run",
                    help="Run this shard from the split up tests.",
                    default=1, type="int")
1234 1235
  result.add_option("--noprof", help="Disable profiling support",
                    default=False)
1236 1237 1238 1239 1240 1241
  return result


def ProcessOptions(options):
  global VERBOSE
  VERBOSE = options.verbose
1242 1243 1244 1245 1246
  options.mode = options.mode.split(',')
  for mode in options.mode:
    if not mode in ['debug', 'release']:
      print "Unknown mode %s" % mode
      return False
1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
  if options.simulator != 'none':
    # Simulator argument was set. Make sure arch and simulator agree.
    if options.simulator != options.arch:
      if options.arch == 'none':
        options.arch = options.simulator
      else:
        print "Architecture %s does not match sim %s" %(options.arch, options.simulator)
        return False
    # Ensure that the simulator argument is handed down to scons.
    options.scons_flags.append("simulator=" + options.simulator)
  else:
    # If options.arch is not set by the command line and no simulator setting
    # was found, set the arch to the guess.
    if options.arch == 'none':
      options.arch = ARCH_GUESS
1262
    options.scons_flags.append("arch=" + options.arch)
1263 1264
  if options.snapshot:
    options.scons_flags.append("snapshot=on")
1265 1266 1267
  global VARIANT_FLAGS
  if options.stress_only:
    VARIANT_FLAGS = [['--stress-opt', '--always-opt']]
1268 1269 1270 1271 1272 1273 1274
  if options.nostress:
    VARIANT_FLAGS = [[],['--nocrankshaft']]
  if options.crankshaft:
    if options.special_command:
      options.special_command += " --crankshaft"
    else:
      options.special_command = "@--crankshaft"
1275 1276 1277 1278 1279
  if options.shell == "d8":
    if options.special_command:
      options.special_command += " --test"
    else:
      options.special_command = "@--test"
1280 1281 1282
  if options.noprof:
    options.scons_flags.append("prof=off")
    options.scons_flags.append("profilingsupport=off")
1283 1284 1285 1286 1287
  return True


REPORT_TEMPLATE = """\
Total: %(total)i tests
1288 1289 1290 1291 1292
 * %(skipped)4d tests will be skipped
 * %(nocrash)4d tests are expected to be flaky but not crash
 * %(pass)4d tests are expected to pass
 * %(fail_ok)4d tests are expected to fail that we won't fix
 * %(fail)4d tests are expected to fail that we should fix\
1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
"""

def PrintReport(cases):
  def IsFlaky(o):
    return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o)
  def IsFailOk(o):
    return (len(o) == 2) and (FAIL in o) and (OKAY in o)
  unskipped = [c for c in cases if not SKIP in c.outcomes]
  print REPORT_TEMPLATE % {
    'total': len(cases),
    'skipped': len(cases) - len(unskipped),
    'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]),
    'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]),
    'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]),
    'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]])
  }


1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
class Pattern(object):

  def __init__(self, pattern):
    self.pattern = pattern
    self.compiled = None

  def match(self, str):
    if not self.compiled:
      pattern = "^" + self.pattern.replace('*', '.*') + "$"
      self.compiled = re.compile(pattern)
    return self.compiled.match(str)

  def __str__(self):
    return self.pattern


1327 1328
def SplitPath(s):
  stripped = [ c.strip() for c in s.split('/') ]
1329 1330 1331
  return [ Pattern(s) for s in stripped if len(s) > 0 ]


1332 1333 1334 1335 1336 1337 1338
def GetSpecialCommandProcessor(value):
  if (not value) or (value.find('@') == -1):
    def ExpandCommand(args):
      return args
    return ExpandCommand
  else:
    pos = value.find('@')
1339
    import urllib
1340 1341
    prefix = urllib.unquote(value[:pos]).split()
    suffix = urllib.unquote(value[pos+1:]).split()
1342 1343 1344 1345 1346
    def ExpandCommand(args):
      return prefix + args + suffix
    return ExpandCommand


1347
BUILT_IN_TESTS = ['mjsunit', 'cctest', 'message', 'preparser']
1348 1349


1350
def GetSuites(test_root):
1351 1352 1353
  def IsSuite(path):
    return isdir(path) and exists(join(path, 'testcfg.py'))
  return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ]
1354 1355


1356 1357 1358 1359
def FormatTime(d):
  millis = round(d * 1000) % 1000
  return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis)

1360 1361 1362 1363 1364 1365 1366
def ShardTests(tests, options):
  if options.shard_count < 2:
    return tests
  if options.shard_run < 1 or options.shard_run > options.shard_count:
    print "shard-run not a valid number, should be in [1:shard-count]"
    print "defaulting back to running all tests"
    return tests
1367
  count = 0
1368 1369 1370
  shard = []
  for test in tests:
    if count % options.shard_count == options.shard_run - 1:
1371
      shard.append(test)
1372 1373
    count += 1
  return shard
1374

1375 1376 1377 1378 1379 1380 1381 1382
def Main():
  parser = BuildOptions()
  (options, args) = parser.parse_args()
  if not ProcessOptions(options):
    parser.print_help()
    return 1

  workspace = abspath(join(dirname(sys.argv[0]), '..'))
1383 1384
  suites = GetSuites(join(workspace, 'test'))
  repositories = [TestRepository(join(workspace, 'test', name)) for name in suites]
1385 1386
  repositories += [TestRepository(a) for a in options.suite]

1387 1388
  root = LiteralTestSuite(repositories)
  if len(args) == 0:
1389
    paths = [SplitPath(t) for t in BUILT_IN_TESTS]
1390 1391 1392 1393 1394 1395
  else:
    paths = [ ]
    for arg in args:
      path = SplitPath(arg)
      paths.append(path)

1396 1397 1398 1399 1400 1401
  # Check for --valgrind option. If enabled, we overwrite the special
  # command flag with a command that uses the run-valgrind.py script.
  if options.valgrind:
    run_valgrind = join(workspace, "tools", "run-valgrind.py")
    options.special_command = "python -u " + run_valgrind + " @"

1402 1403
  shell = abspath(options.shell)
  buildspace = dirname(shell)
1404

1405
  context = Context(workspace, buildspace, VERBOSE,
1406
                    shell,
1407
                    options.timeout,
1408
                    GetSpecialCommandProcessor(options.special_command),
1409 1410
                    options.suppress_dialogs,
                    options.store_unexpected_output)
1411
  # First build the required targets
1412 1413 1414 1415 1416 1417
  if not options.no_build:
    reqs = [ ]
    for path in paths:
      reqs += root.GetBuildRequirements(path, context)
    reqs = list(set(reqs))
    if len(reqs) > 0:
1418 1419
      if options.j != 1:
        options.scons_flags += ['-j', str(options.j)]
1420
      if not BuildRequirements(context, reqs, options.mode, options.scons_flags):
1421 1422
        return 1

1423 1424 1425
  # Just return if we are only building the targets for running the tests.
  if options.build_only:
    return 0
1426

1427 1428 1429 1430 1431 1432 1433
  # Get status for tests
  sections = [ ]
  defs = { }
  root.GetTestStatus(context, sections, defs)
  config = Configuration(sections, defs)

  # List the tests
1434 1435
  all_cases = [ ]
  all_unused = [ ]
1436
  unclassified_tests = [ ]
1437
  globally_unused_rules = None
1438 1439
  for path in paths:
    for mode in options.mode:
1440
      env = {
1441
        'mode': mode,
1442
        'system': utils.GuessOS(),
1443
        'arch': options.arch,
1444 1445
        'simulator': options.simulator,
        'crankshaft': options.crankshaft
1446
      }
1447
      test_list = root.ListTests([], path, context, mode, [])
1448
      unclassified_tests += test_list
1449 1450 1451 1452 1453
      (cases, unused_rules, all_outcomes) = config.ClassifyTests(test_list, env)
      if globally_unused_rules is None:
        globally_unused_rules = set(unused_rules)
      else:
        globally_unused_rules = globally_unused_rules.intersection(unused_rules)
1454
      all_cases += ShardTests(cases, options)
1455
      all_unused.append(unused_rules)
1456

1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469
  if options.cat:
    visited = set()
    for test in unclassified_tests:
      key = tuple(test.path)
      if key in visited:
        continue
      visited.add(key)
      print "--- begin source: %s ---" % test.GetLabel()
      source = test.GetSource().strip()
      print source
      print "--- end source: %s ---" % test.GetLabel()
    return 0

1470 1471 1472
  if options.warn_unused:
    for rule in globally_unused_rules:
      print "Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path])
1473 1474

  if options.report:
1475 1476
    PrintReport(all_cases)

1477
  result = None
1478 1479 1480
  def DoSkip(case):
    return SKIP in case.outcomes or SLOW in case.outcomes
  cases_to_run = [ c for c in all_cases if not DoSkip(c) ]
1481 1482
  if not options.isolates:
    cases_to_run = [c for c in cases_to_run if not c.TestsIsolates()]
1483
  if len(cases_to_run) == 0:
1484
    print "No tests to run."
1485
    return 0
1486
  else:
1487
    try:
1488
      start = time.time()
1489
      if RunTestCases(cases_to_run, options.progress, options.j):
1490
        result = 0
1491
      else:
1492 1493
        result = 1
      duration = time.time() - start
1494 1495 1496
    except KeyboardInterrupt:
      print "Interrupted"
      return 1
1497

1498 1499 1500 1501 1502
  if options.time:
    # Write the times to stderr to make it easy to separate from the
    # test output.
    print
    sys.stderr.write("--- Total time: %s ---\n" % FormatTime(duration))
1503
    timed_tests = [ t.case for t in cases_to_run if not t.case.duration is None ]
1504 1505 1506 1507 1508 1509 1510 1511 1512
    timed_tests.sort(lambda a, b: a.CompareTime(b))
    index = 1
    for entry in timed_tests[:20]:
      t = FormatTime(entry.duration)
      sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel()))
      index += 1

  return result

1513 1514 1515

if __name__ == '__main__':
  sys.exit(Main())